diff --git a/.gitignore b/.gitignore index 485dee64..be897f86 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .idea +_output \ No newline at end of file diff --git a/OWNERS b/OWNERS index c5d34c19..5788a0a7 100644 --- a/OWNERS +++ b/OWNERS @@ -1,11 +1,16 @@ # See the OWNERS docs: https://go.k8s.io/owners approvers: - - lavalamp - - apelisse - - jennybuckley -reviewers: - - lavalamp - - apelisse - - jennybuckley - jpbetz + - liggitt +emeritus_approvers: + - apelisse # 2025-09-10 + - lavalamp # 2023-05-13 + - jennybuckley # 2021-05-13 +reviewers: + - jefftree + - liggitt + - yliaog + - cici37 + - yongruilin + diff --git a/fieldpath/element.go b/fieldpath/element.go index 280d0ea6..73436912 100644 --- a/fieldpath/element.go +++ b/fieldpath/element.go @@ -18,10 +18,11 @@ package fieldpath import ( "fmt" + "iter" "sort" "strings" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // PathElement describes how to select a child field given a containing object. @@ -47,6 +48,36 @@ type PathElement struct { Index *int } +// FieldNameElement creates a new FieldName PathElement. +func FieldNameElement(name string) PathElement { + return PathElement{FieldName: &name} +} + +// KeyElement creates a new Key PathElement with the key fields. +func KeyElement(fields ...value.Field) PathElement { + l := value.FieldList(fields) + return PathElement{Key: &l} +} + +// KeyElementByFields creates a new Key PathElement from names and values. +// `nameValues` must have an even number of entries, alternating +// names (type must be string) with values (type must be value.Value). If these +// conditions are not met, KeyByFields will panic--it's intended for static +// construction and shouldn't have user-produced values passed to it. +func KeyElementByFields(nameValues ...any) PathElement { + return PathElement{Key: KeyByFields(nameValues...)} +} + +// ValueElement creates a new Value PathElement. +func ValueElement(value value.Value) PathElement { + return PathElement{Value: &value} +} + +// IndexElement creates a new Index PathElement. +func IndexElement(index int) PathElement { + return PathElement{Index: &index} +} + // Less provides an order for path elements. func (e PathElement) Less(rhs PathElement) bool { return e.Compare(rhs) < 0 @@ -156,6 +187,25 @@ func (e PathElement) String() string { } } +// Copy returns a copy of the PathElement. +// This is not a full deep copy as any contained value.Value is not copied. +func (e PathElement) Copy() PathElement { + if e.FieldName != nil { + return PathElement{FieldName: e.FieldName} + } + if e.Key != nil { + c := e.Key.Copy() + return PathElement{Key: &c} + } + if e.Value != nil { + return PathElement{Value: e.Value} + } + if e.Index != nil { + return PathElement{Index: e.Index} + } + return e // zero value +} + // KeyByFields is a helper function which constructs a key for an associative // list type. `nameValues` must have an even number of entries, alternating // names (type must be string) with values (type must be value.Value). If these @@ -193,6 +243,16 @@ func (spe sortedPathElements) Len() int { return len(spe) } func (spe sortedPathElements) Less(i, j int) bool { return spe[i].Less(spe[j]) } func (spe sortedPathElements) Swap(i, j int) { spe[i], spe[j] = spe[j], spe[i] } +// Copy returns a copy of the PathElementSet. +// This is not a full deep copy as any contained value.Value is not copied. +func (s PathElementSet) Copy() PathElementSet { + out := make(sortedPathElements, len(s.members)) + for i := range s.members { + out[i] = s.members[i].Copy() + } + return PathElementSet{members: out} +} + // Insert adds pe to the set. func (s *PathElementSet) Insert(pe PathElement) { loc := sort.Search(len(s.members), func(i int) bool { @@ -315,3 +375,14 @@ func (s *PathElementSet) Iterate(f func(PathElement)) { f(pe) } } + +// All iterates over each PathElement in the set. The order is deterministic. +func (s *PathElementSet) All() iter.Seq[PathElement] { + return func(yield func(element PathElement) bool) { + for _, pe := range s.members { + if !yield(pe) { + return + } + } + } +} diff --git a/fieldpath/element_test.go b/fieldpath/element_test.go index 794bfdee..aba96b14 100644 --- a/fieldpath/element_test.go +++ b/fieldpath/element_test.go @@ -19,7 +19,7 @@ package fieldpath import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) func TestPathElementSet(t *testing.T) { @@ -33,6 +33,10 @@ func TestPathElementSet(t *testing.T) { if !s2.Has(PathElement{}) { t.Errorf("expected to have something: %#v", s2) } + c2 := s2.Copy() + if !c2.Equals(s2) { + t.Errorf("expected copy to equal original: %#v, %#v", s2, c2) + } n1 := "aoeu" n2 := "asdf" @@ -60,6 +64,20 @@ func TestPathElementSet(t *testing.T) { } i++ }) + i = 0 + for pe := range s2.All() { + e, a := expected[i], pe.FieldName + if e == nil || a == nil { + if e != a { + t.Errorf("index %v wanted %#v, got %#v", i, e, a) + } + } else { + if *e != *a { + t.Errorf("index %v wanted %#v, got %#v", i, *e, *a) + } + } + i++ + } } func strptr(s string) *string { return &s } @@ -68,6 +86,9 @@ func valptr(i interface{}) *value.Value { v := value.NewValueInterface(i) return &v } +func val(i interface{}) value.Value { + return value.NewValueInterface(i) +} func TestPathElementLess(t *testing.T) { table := []struct { @@ -84,71 +105,71 @@ func TestPathElementLess(t *testing.T) { eq: true, }, { name: "FieldName-1", - a: PathElement{FieldName: strptr("anteater")}, - b: PathElement{FieldName: strptr("zebra")}, + a: FieldNameElement("anteater"), + b: FieldNameElement("zebra"), }, { name: "FieldName-2", - a: PathElement{FieldName: strptr("bee")}, - b: PathElement{FieldName: strptr("bee")}, + a: FieldNameElement("bee"), + b: FieldNameElement("bee"), eq: true, }, { name: "FieldName-3", - a: PathElement{FieldName: strptr("capybara")}, - b: PathElement{Key: KeyByFields("dog", 3)}, + a: FieldNameElement("capybara"), + b: KeyElementByFields("dog", 3), }, { name: "FieldName-4", - a: PathElement{FieldName: strptr("elephant")}, - b: PathElement{Value: valptr(4)}, + a: FieldNameElement("elephant"), + b: ValueElement(val(4)), }, { name: "FieldName-5", - a: PathElement{FieldName: strptr("falcon")}, - b: PathElement{Index: intptr(5)}, + a: FieldNameElement("falcon"), + b: IndexElement(5), }, { name: "Key-1", - a: PathElement{Key: KeyByFields("goat", 1)}, - b: PathElement{Key: KeyByFields("goat", 1)}, + a: KeyElementByFields("goat", 1), + b: KeyElementByFields("goat", 1), eq: true, }, { name: "Key-2", - a: PathElement{Key: KeyByFields("horse", 1)}, - b: PathElement{Key: KeyByFields("horse", 2)}, + a: KeyElementByFields("horse", 1), + b: KeyElementByFields("horse", 2), }, { name: "Key-3", - a: PathElement{Key: KeyByFields("ibex", 1)}, - b: PathElement{Key: KeyByFields("jay", 1)}, + a: KeyElementByFields("ibex", 1), + b: KeyElementByFields("jay", 1), }, { name: "Key-4", - a: PathElement{Key: KeyByFields("kite", 1)}, - b: PathElement{Key: KeyByFields("kite", 1, "kite-2", 1)}, + a: KeyElementByFields("kite", 1), + b: KeyElementByFields("kite", 1, "kite-2", 1), }, { name: "Key-5", - a: PathElement{Key: KeyByFields("kite", 1)}, - b: PathElement{Value: valptr(1)}, + a: KeyElementByFields("kite", 1), + b: ValueElement(val(1)), }, { name: "Key-6", - a: PathElement{Key: KeyByFields("kite", 1)}, - b: PathElement{Index: intptr(5)}, + a: KeyElementByFields("kite", 1), + b: IndexElement(5), }, { name: "Value-1", - a: PathElement{Value: valptr(1)}, - b: PathElement{Value: valptr(2)}, + a: ValueElement(val(1)), + b: ValueElement(val(2)), }, { name: "Value-2", - a: PathElement{Value: valptr(1)}, - b: PathElement{Value: valptr(1)}, + a: ValueElement(val(1)), + b: ValueElement(val(1)), eq: true, }, { name: "Value-3", - a: PathElement{Value: valptr(1)}, - b: PathElement{Index: intptr(1)}, + a: ValueElement(val(1)), + b: IndexElement(1), }, { name: "Index-1", - a: PathElement{Index: intptr(1)}, - b: PathElement{Index: intptr(2)}, + a: IndexElement(1), + b: IndexElement(2), }, { name: "Index-2", - a: PathElement{Index: intptr(1)}, - b: PathElement{Index: intptr(1)}, + a: IndexElement(1), + b: IndexElement(1), eq: true, }, } diff --git a/fieldpath/fieldpath_test.go b/fieldpath/fieldpath_test.go new file mode 100644 index 00000000..34812d6a --- /dev/null +++ b/fieldpath/fieldpath_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fieldpath_test + +import ( + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/value" +) + +// Type and function aliases to avoid typing... + +type ( + Filter = fieldpath.Filter + Path = fieldpath.Path + PathElement = fieldpath.PathElement + Set = fieldpath.Set + SetNodeMap = fieldpath.SetNodeMap +) + +var ( + KeyByFields = fieldpath.KeyByFields + MakePathOrDie = fieldpath.MakePathOrDie + MakePrefixMatcherOrDie = fieldpath.MakePrefixMatcherOrDie + MatchAnyPathElement = fieldpath.MatchAnyPathElement + NewIncludeMatcherFilter = fieldpath.NewIncludeMatcherFilter + NewSet = fieldpath.NewSet + + // Short names for readable test cases. + _V = value.NewValueInterface + _NS = fieldpath.NewSet + _P = fieldpath.MakePathOrDie +) diff --git a/fieldpath/fromvalue.go b/fieldpath/fromvalue.go index 295d2685..78383e40 100644 --- a/fieldpath/fromvalue.go +++ b/fieldpath/fromvalue.go @@ -17,7 +17,7 @@ limitations under the License. package fieldpath import ( - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // SetFromValue creates a set containing every leaf field mentioned in v. diff --git a/fieldpath/fromvalue_test.go b/fieldpath/fromvalue_test.go index 92de84e3..cd909a50 100644 --- a/fieldpath/fromvalue_test.go +++ b/fieldpath/fromvalue_test.go @@ -19,8 +19,8 @@ package fieldpath import ( "testing" - "gopkg.in/yaml.v2" - "sigs.k8s.io/structured-merge-diff/v5/value" + yaml "go.yaml.in/yaml/v2" + "sigs.k8s.io/structured-merge-diff/v6/value" ) func TestFromValue(t *testing.T) { diff --git a/fieldpath/managers_test.go b/fieldpath/managers_test.go index 432cd590..6984176c 100644 --- a/fieldpath/managers_test.go +++ b/fieldpath/managers_test.go @@ -17,17 +17,10 @@ limitations under the License. package fieldpath_test import ( - "fmt" "reflect" "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" -) - -var ( - // Short names for readable test cases. - _NS = fieldpath.NewSet - _P = fieldpath.MakePathOrDie + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" ) func TestManagersEquals(t *testing.T) { @@ -155,7 +148,7 @@ func TestManagersEquals(t *testing.T) { } for _, test := range tests { - t.Run(fmt.Sprintf(test.name), func(t *testing.T) { + t.Run(test.name, func(t *testing.T) { want := test.out got := test.lhs.Difference(test.rhs) if !reflect.DeepEqual(want, got) { @@ -273,7 +266,7 @@ func TestManagersDifference(t *testing.T) { } for _, test := range tests { - t.Run(fmt.Sprintf(test.name), func(t *testing.T) { + t.Run(test.name, func(t *testing.T) { equal := test.lhs.Equals(test.rhs) if test.equal && !equal { difference := test.lhs.Difference(test.rhs) diff --git a/fieldpath/path.go b/fieldpath/path.go index 0a457fd9..a865ec42 100644 --- a/fieldpath/path.go +++ b/fieldpath/path.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // Path describes how to select a potentially deeply-nested child field given a diff --git a/fieldpath/path_test.go b/fieldpath/path_test.go index 2f50173a..c5eea4f0 100644 --- a/fieldpath/path_test.go +++ b/fieldpath/path_test.go @@ -19,7 +19,7 @@ package fieldpath import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) var ( @@ -50,9 +50,66 @@ func TestPathString(t *testing.T) { _V(false), _V(3.14159), ), `.foo[="b"][=5][=false][=3.14159]`}, + { + name: "simple field", + fp: MakePathOrDie("spec"), + expect: ".spec", + }, + { + name: "app container image", + fp: MakePathOrDie( + "spec", "apps", + KeyByFields("name", "app-🚀"), + "container", "image", + ), + expect: `.spec.apps[name="app-🚀"].container.image`, + }, + { + name: "app port", + fp: MakePathOrDie( + "spec", "apps", + KeyByFields("name", "app-💻"), + "container", "ports", + KeyByFields("name", "port-🔑"), + "containerPort", + ), + expect: ".spec.apps[name=\"app-💻\"].container.ports[name=\"port-🔑\"].containerPort", + }, + { + name: "field with space", + fp: MakePathOrDie("spec", "field with space"), + expect: ".spec.field with space", + }, + { + name: "value with space", + fp: MakePathOrDie( + "spec", "apps", + _V("app with space"), + "container", "image", + ), + expect: `.spec.apps[="app with space"].container.image`, + }, + { + name: "value with quotes", + fp: MakePathOrDie( + "spec", "apps", + _V("app with \"quotes\""), + "container", "image", + ), + expect: ".spec.apps[=\"app with \\\"quotes\\\"\"].container.image", + }, + + { + name: "value with unicode", + fp: MakePathOrDie( + "spec", "apps", + _V("app-with-unicøde"), + "container", "image", + ), + expect: ".spec.apps[=\"app-with-unicøde\"].container.image", + }, } for _, tt := range table { - tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() got := tt.fp.String() diff --git a/fieldpath/pathelementmap.go b/fieldpath/pathelementmap.go index ba4a0399..ff7ee510 100644 --- a/fieldpath/pathelementmap.go +++ b/fieldpath/pathelementmap.go @@ -19,7 +19,7 @@ package fieldpath import ( "sort" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // PathElementValueMap is a map from PathElement to value.Value. @@ -28,20 +28,15 @@ import ( // for PathElementSet and SetNodeMap, so we could probably share the // code. type PathElementValueMap struct { - members sortedPathElementValues + valueMap PathElementMap } func MakePathElementValueMap(size int) PathElementValueMap { return PathElementValueMap{ - members: make(sortedPathElementValues, 0, size), + valueMap: MakePathElementMap(size), } } -type pathElementValue struct { - PathElement PathElement - Value value.Value -} - type sortedPathElementValues []pathElementValue // Implement the sort interface; this would permit bulk creation, which would @@ -53,7 +48,40 @@ func (spev sortedPathElementValues) Less(i, j int) bool { func (spev sortedPathElementValues) Swap(i, j int) { spev[i], spev[j] = spev[j], spev[i] } // Insert adds the pathelement and associated value in the map. +// If insert is called twice with the same PathElement, the value is replaced. func (s *PathElementValueMap) Insert(pe PathElement, v value.Value) { + s.valueMap.Insert(pe, v) +} + +// Get retrieves the value associated with the given PathElement from the map. +// (nil, false) is returned if there is no such PathElement. +func (s *PathElementValueMap) Get(pe PathElement) (value.Value, bool) { + v, ok := s.valueMap.Get(pe) + if !ok { + return nil, false + } + return v.(value.Value), true +} + +// PathElementValueMap is a map from PathElement to interface{}. +type PathElementMap struct { + members sortedPathElementValues +} + +type pathElementValue struct { + PathElement PathElement + Value interface{} +} + +func MakePathElementMap(size int) PathElementMap { + return PathElementMap{ + members: make(sortedPathElementValues, 0, size), + } +} + +// Insert adds the pathelement and associated value in the map. +// If insert is called twice with the same PathElement, the value is replaced. +func (s *PathElementMap) Insert(pe PathElement, v interface{}) { loc := sort.Search(len(s.members), func(i int) bool { return !s.members[i].PathElement.Less(pe) }) @@ -62,6 +90,7 @@ func (s *PathElementValueMap) Insert(pe PathElement, v value.Value) { return } if s.members[loc].PathElement.Equals(pe) { + s.members[loc].Value = v return } s.members = append(s.members, pathElementValue{}) @@ -71,7 +100,7 @@ func (s *PathElementValueMap) Insert(pe PathElement, v value.Value) { // Get retrieves the value associated with the given PathElement from the map. // (nil, false) is returned if there is no such PathElement. -func (s *PathElementValueMap) Get(pe PathElement) (value.Value, bool) { +func (s *PathElementMap) Get(pe PathElement) (interface{}, bool) { loc := sort.Search(len(s.members), func(i int) bool { return !s.members[i].PathElement.Less(pe) }) diff --git a/fieldpath/pathelementmap_test.go b/fieldpath/pathelementmap_test.go index b635771a..8341370b 100644 --- a/fieldpath/pathelementmap_test.go +++ b/fieldpath/pathelementmap_test.go @@ -19,7 +19,7 @@ package fieldpath import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) func TestPathElementValueMap(t *testing.T) { @@ -47,4 +47,11 @@ func TestPathElementValueMap(t *testing.T) { } else if !value.Equals(val, value.NewValueInterface(2)) { t.Fatalf("Unexpected value found: %#v", val) } + + m.Insert(PathElement{FieldName: strptr("carrot")}, value.NewValueInterface("fork")) + if val, ok := m.Get(PathElement{FieldName: strptr("carrot")}); !ok { + t.Fatal("Missing path-element in map") + } else if !value.Equals(val, value.NewValueInterface("fork")) { + t.Fatalf("Unexpected value found: %#v", val) + } } diff --git a/fieldpath/serialize-pe.go b/fieldpath/serialize-pe.go index 2904051a..f4b00c2e 100644 --- a/fieldpath/serialize-pe.go +++ b/fieldpath/serialize-pe.go @@ -24,7 +24,7 @@ import ( "strings" jsoniter "github.com/json-iterator/go" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/value" ) var ErrUnknownPathElementType = errors.New("unknown path element type") @@ -54,6 +54,24 @@ var ( peSepBytes = []byte(peSeparator) ) +// readJSONIter reads a Value from a JSON iterator. +// DO NOT EXPORT +// TODO: eliminate this https://github.com/kubernetes-sigs/structured-merge-diff/issues/202 +func readJSONIter(iter *jsoniter.Iterator) (value.Value, error) { + v := iter.Read() + if iter.Error != nil && iter.Error != io.EOF { + return nil, iter.Error + } + return value.NewValueInterface(v), nil +} + +// writeJSONStream writes a value into a JSON stream. +// DO NOT EXPORT +// TODO: eliminate this https://github.com/kubernetes-sigs/structured-merge-diff/issues/202 +func writeJSONStream(v value.Value, stream *jsoniter.Stream) { + stream.WriteVal(v.Unstructured()) +} + // DeserializePathElement parses a serialized path element func DeserializePathElement(s string) (PathElement, error) { b := []byte(s) @@ -75,7 +93,7 @@ func DeserializePathElement(s string) (PathElement, error) { case peValueSepBytes[0]: iter := readPool.BorrowIterator(b) defer readPool.ReturnIterator(iter) - v, err := value.ReadJSONIter(iter) + v, err := readJSONIter(iter) if err != nil { return PathElement{}, err } @@ -86,7 +104,7 @@ func DeserializePathElement(s string) (PathElement, error) { fields := value.FieldList{} iter.ReadObjectCB(func(iter *jsoniter.Iterator, key string) bool { - v, err := value.ReadJSONIter(iter) + v, err := readJSONIter(iter) if err != nil { iter.Error = err return false @@ -141,14 +159,14 @@ func serializePathElementToWriter(w io.Writer, pe PathElement) error { stream.WriteMore() } stream.WriteObjectField(field.Name) - value.WriteJSONStream(field.Value, stream) + writeJSONStream(field.Value, stream) } stream.WriteObjectEnd() case pe.Value != nil: if _, err := stream.Write(peValueSepBytes); err != nil { return err } - value.WriteJSONStream(*pe.Value, stream) + writeJSONStream(*pe.Value, stream) case pe.Index != nil: if _, err := stream.Write(peIndexSepBytes); err != nil { return err diff --git a/fieldpath/serialize-pe_test.go b/fieldpath/serialize-pe_test.go index 0a6ee8f1..6ba00387 100644 --- a/fieldpath/serialize-pe_test.go +++ b/fieldpath/serialize-pe_test.go @@ -16,7 +16,9 @@ limitations under the License. package fieldpath -import "testing" +import ( + "testing" +) func TestPathElementRoundTrip(t *testing.T) { tests := []string{ @@ -25,7 +27,10 @@ func TestPathElementRoundTrip(t *testing.T) { `f:`, `f:spec`, `f:more-complicated-string`, + `f: string-with-spaces `, + `f:abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789`, `k:{"name":"my-container"}`, + `k:{"name":" name with spaces "}`, `k:{"port":"8080","protocol":"TCP"}`, `k:{"optionalField":null}`, `k:{"jsonField":{"A":1,"B":null,"C":"D","E":{"F":"G"}}}`, @@ -34,6 +39,14 @@ func TestPathElementRoundTrip(t *testing.T) { `v:"some-string"`, `v:1234`, `v:{"some":"json"}`, + `v:{"some":" some with spaces "}`, + `k:{"name":"app-🚀"}`, + `k:{"name":"app-💻"}`, + `k:{"name":"app with-unicøde"}`, + `k:{"name":"你好世界"}`, + `k:{"name":"Привет, мир"}`, + `k:{"name":"नमस्ते दुनिया"}`, + `k:{"name":"👋"}`, } for _, test := range tests { @@ -71,6 +84,7 @@ func TestDeserializePathElementError(t *testing.T) { `v:`, `k:invalid json`, `k:{"name":invalid}`, + `v:{"some":" \x41"}`, // This is an invalid JSON string because \x41 is not a valid escape sequence. } for _, test := range tests { diff --git a/fieldpath/serialize_test.go b/fieldpath/serialize_test.go index a3bcc9e6..d1bd1fc2 100644 --- a/fieldpath/serialize_test.go +++ b/fieldpath/serialize_test.go @@ -14,13 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fieldpath +package fieldpath_test import ( "bytes" "fmt" "strings" "testing" + + "github.com/google/go-cmp/cmp" ) func TestSerializeV1(t *testing.T) { @@ -51,6 +53,7 @@ func TestSerializeV1GoldenData(t *testing.T) { examples := []string{ `{"f:aaa":{},"f:aab":{},"f:aac":{},"f:aad":{},"f:aae":{},"f:aaf":{},"k:{\"name\":\"first\"}":{},"k:{\"name\":\"second\"}":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{},"k:{\"port\":443,\"protocol\":\"udp\"}":{},"v:1":{},"v:2":{},"v:3":{},"v:\"aa\"":{},"v:\"ab\"":{},"v:true":{},"i:1":{},"i:2":{},"i:3":{},"i:4":{}}`, `{"f:aaa":{"k:{\"name\":\"second\"}":{"v:3":{"f:aab":{}}},"v:3":{},"v:true":{}},"f:aab":{"f:aaa":{},"f:aaf":{"k:{\"port\":443,\"protocol\":\"udp\"}":{"k:{\"port\":443,\"protocol\":\"tcp\"}":{}}},"k:{\"name\":\"first\"}":{}},"f:aac":{"f:aaa":{"v:1":{}},"f:aac":{},"v:3":{"k:{\"name\":\"second\"}":{}}},"f:aad":{"f:aac":{"v:1":{}},"f:aaf":{"k:{\"name\":\"first\"}":{"k:{\"name\":\"first\"}":{}}},"i:1":{"i:1":{},"i:3":{"v:true":{}}}},"f:aae":{"f:aae":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{"k:{\"port\":443,\"protocol\":\"udp\"}":{}},"i:4":{"f:aaf":{}}},"f:aaf":{"i:1":{"f:aac":{}},"i:2":{},"i:3":{}},"k:{\"name\":\"first\"}":{"f:aad":{"f:aaf":{}}},"k:{\"port\":443,\"protocol\":\"tcp\"}":{"f:aaa":{"f:aad":{}}},"k:{\"port\":443,\"protocol\":\"udp\"}":{"f:aac":{},"k:{\"name\":\"first\"}":{"i:3":{}},"k:{\"port\":443,\"protocol\":\"udp\"}":{"i:4":{}}},"v:1":{"f:aac":{"i:4":{}},"f:aaf":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{}},"v:2":{"f:aad":{"f:aaf":{}},"i:1":{}},"v:3":{"f:aaa":{},"k:{\"name\":\"first\"}":{},"i:2":{}},"v:\"aa\"":{"f:aab":{"f:aaf":{}},"f:aae":{},"k:{\"name\":\"first\"}":{"f:aad":{}},"i:2":{}},"v:\"ab\"":{"f:aaf":{"i:4":{}},"k:{\"port\":443,\"protocol\":\"tcp\"}":{},"k:{\"port\":443,\"protocol\":\"udp\"}":{},"v:1":{"k:{\"port\":443,\"protocol\":\"udp\"}":{}},"i:1":{"f:aae":{"i:4":{}}}},"v:true":{"k:{\"name\":\"second\"}":{"f:aaa":{}},"i:2":{"k:{\"port\":443,\"protocol\":\"tcp\"}":{}}},"i:1":{"i:3":{"f:aaf":{}}},"i:2":{"f:aae":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{"v:1":{}}},"i:3":{"f:aab":{"v:true":{"v:\"aa\"":{}}},"f:aaf":{},"i:1":{}},"i:4":{"v:\"aa\"":{"f:aab":{"k:{\"name\":\"second\"}":{}}}}}`, + `{"f:spec":{".":{},"f:apps":{".":{},"k:{\"name\":\" app-💻\"}":{".":{},"f:container":{".":{},"f:image":{},"f:name":{},"f:ports":{".":{},"k:{\"name\":\"port 🔑\"}":{".":{},"f:containerPort":{},"f:name":{}}}},"f:name":{}},"k:{\"name\":\" app-🚀\"}":{".":{},"f:container":{".":{},"f:image":{},"f:name":{},"f:ports":{".":{},"k:{\"name\":\"port-✅ \"}":{".":{},"f:containerPort":{},"f:name":{}}}},"f:name":{}}}}}`, } for i, str := range examples { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { @@ -70,6 +73,246 @@ func TestSerializeV1GoldenData(t *testing.T) { } } +func TestDeserializeForValidNonNormalized(t *testing.T) { + testCases := []struct { + nonNormalizedString string + normalizedString string + }{ + { + nonNormalizedString: `{ + "f:aad": {}, + "f:aaa": {}, + "f:aab": {}, + "f:aac": {}, + "f:aae": {}, + "f:aaf": {}, + + + "k:{\"name\":\"first\"}": {}, + "k:{\"name\":\"second\"}": {}, + "k:{\"port\":443,\"protocol\":\"tcp\"}": {}, + "k:{ \"protocol\":\"udp\",\"port\":443}": {}, + "v:1": {}, + + "v:2": {}, + "v: 3": {}, + "v:\"aa\"": {}, + + "v:\"ab\"": {}, + "v:true": {}, + "i:1": {}, + "i:2": {}, + "i:3": {} , + "i:4": {} +}`, + normalizedString: `{"f:aaa":{},"f:aab":{},"f:aac":{},"f:aad":{},"f:aae":{},"f:aaf":{},"k:{\"name\":\"first\"}":{},"k:{\"name\":\"second\"}":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{},"k:{\"port\":443,\"protocol\":\"udp\"}":{},"v:1":{},"v:2":{},"v:3":{},"v:\"aa\"":{},"v:\"ab\"":{},"v:true":{},"i:1":{},"i:2":{},"i:3":{},"i:4":{}}`, + }, { + nonNormalizedString: `{ + "f:aaa": { + "k:{\"name\":\"second\"}": { + "v:3": { + "f:aab": {} + } + }, + "v:3": {}, + "v:true": {} + }, + "f:aab": { + "f:aaa": {}, + "f:aaf": { + "k:{\"port\":443,\"protocol\":\"udp\"}": { + "k:{\"port\":443,\"protocol\":\"tcp\"}": {} + } + }, + "k:{\"name\":\"first\"}": {} + }, + "f:aac": { + "f:aaa": { + "v:1": {} + }, + "f:aac": {}, + "v:3": { + "k:{\"name\":\"second\"}": {} + } + }, + "f:aad": { + "f:aac": { + "v:1": {} + }, + "f:aaf": { + "k:{\"name\":\"first\"}": { + "k:{\"name\":\"first\"}": {} + } + }, + "i:1": { + "i:1": {}, + "i:3": { + "v:true": {} + } + } + }, + "f:aae": { + "f:aae": {}, + "k:{\"port\":443,\"protocol\":\"tcp\"}": { + "k:{\"port\":443,\"protocol\":\"udp\"}": {} + }, + "i:4": { + "f:aaf": {} + } + }, + "f:aaf": { + "i:1": { + "f:aac": {} + }, + "i:3": {}, + "i:2": {} + }, + "k:{\"name\":\"first\"}": { + "f:aad": { + "f:aaf": {} + } + }, + "k:{\"port\":443,\"protocol\":\"tcp\"}": { + "f:aaa": { + "f:aad": {} + } + }, + "k:{\"port\":443,\"protocol\":\"udp\"}": { + "f:aac": {}, + "k:{\"name\":\"first\"}": { + "i:3": {} + }, + "k:{\"port\":443,\"protocol\":\"udp\"}": { + "i:4": {} + } + }, + "v:2": { + "f:aad": { + "f:aaf": {} + }, + "i:1": {} + }, + "v:1": { + "f:aac": { + "i:4": {} + }, + "f:aaf": {}, + "k:{\"port\":443,\"protocol\":\"tcp\"}": {} + }, + "v:3": { + "f:aaa": {}, + "k:{\"name\":\"first\"}": {}, + "i:2": {} + }, + "v:\"aa\"": { + "f:aab": { + "f:aaf": {} + }, + "f:aae": {}, + "k:{\"name\":\"first\"}": { + "f:aad": {} + }, + "i:2": {} + }, + "v:\"ab\"": { + "f:aaf": { + "i:4": {} + }, + "k:{\"port\":443,\"protocol\":\"tcp\"}": {}, + "k:{\"port\":443,\"protocol\":\"udp\"}": {}, + "v:1": { + "k:{\"port\":443,\"protocol\":\"udp\"}": {} + }, + "i:1": { + "f:aae": { + "i:4": {} + } + } + }, + + "v:true": { + "k:{\"name\":\"second\"}": { + "f:aaa": {} + }, + "i:2": { + "k:{\"port\":443,\"protocol\":\"tcp\"}": {} + } + }, + + + "i:1": { + "i:3": { + "f:aaf": {} + } + }, + + "i:3": { + "f:aab": { + "v:true": { + "v:\"aa\"": {} + } + }, + "f:aaf": {}, + "i:1": {} + }, + + "i:2": { + "f:aae": {}, + "k:{\"port\":443,\"protocol\":\"tcp\"}": { + "v:1": {} + } + }, + + "i:4": { + "v:\"aa\"": { + "f:aab": { + "k:{\"name\":\"second\"}": {} + } + } + } +}`, + normalizedString: `{"f:aaa":{"k:{\"name\":\"second\"}":{"v:3":{"f:aab":{}}},"v:3":{},"v:true":{}},"f:aab":{"f:aaa":{},"f:aaf":{"k:{\"port\":443,\"protocol\":\"udp\"}":{"k:{\"port\":443,\"protocol\":\"tcp\"}":{}}},"k:{\"name\":\"first\"}":{}},"f:aac":{"f:aaa":{"v:1":{}},"f:aac":{},"v:3":{"k:{\"name\":\"second\"}":{}}},"f:aad":{"f:aac":{"v:1":{}},"f:aaf":{"k:{\"name\":\"first\"}":{"k:{\"name\":\"first\"}":{}}},"i:1":{"i:1":{},"i:3":{"v:true":{}}}},"f:aae":{"f:aae":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{"k:{\"port\":443,\"protocol\":\"udp\"}":{}},"i:4":{"f:aaf":{}}},"f:aaf":{"i:1":{"f:aac":{}},"i:2":{},"i:3":{}},"k:{\"name\":\"first\"}":{"f:aad":{"f:aaf":{}}},"k:{\"port\":443,\"protocol\":\"tcp\"}":{"f:aaa":{"f:aad":{}}},"k:{\"port\":443,\"protocol\":\"udp\"}":{"f:aac":{},"k:{\"name\":\"first\"}":{"i:3":{}},"k:{\"port\":443,\"protocol\":\"udp\"}":{"i:4":{}}},"v:1":{"f:aac":{"i:4":{}},"f:aaf":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{}},"v:2":{"f:aad":{"f:aaf":{}},"i:1":{}},"v:3":{"f:aaa":{},"k:{\"name\":\"first\"}":{},"i:2":{}},"v:\"aa\"":{"f:aab":{"f:aaf":{}},"f:aae":{},"k:{\"name\":\"first\"}":{"f:aad":{}},"i:2":{}},"v:\"ab\"":{"f:aaf":{"i:4":{}},"k:{\"port\":443,\"protocol\":\"tcp\"}":{},"k:{\"port\":443,\"protocol\":\"udp\"}":{},"v:1":{"k:{\"port\":443,\"protocol\":\"udp\"}":{}},"i:1":{"f:aae":{"i:4":{}}}},"v:true":{"k:{\"name\":\"second\"}":{"f:aaa":{}},"i:2":{"k:{\"port\":443,\"protocol\":\"tcp\"}":{}}},"i:1":{"i:3":{"f:aaf":{}}},"i:2":{"f:aae":{},"k:{\"port\":443,\"protocol\":\"tcp\"}":{"v:1":{}}},"i:3":{"f:aab":{"v:true":{"v:\"aa\"":{}}},"f:aaf":{},"i:1":{}},"i:4":{"v:\"aa\"":{"f:aab":{"k:{\"name\":\"second\"}":{}}}}}`, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { + nonNormSet := NewSet() + err := nonNormSet.FromJSON(strings.NewReader(tc.nonNormalizedString)) + if err != nil { + t.Fatalf("Failed to deserialize non normalized string %s : %v\n%#v", tc.nonNormalizedString, err, nonNormSet) + } + + normSet := NewSet() + err = normSet.FromJSON(strings.NewReader(tc.normalizedString)) + if err != nil { + t.Fatalf("Failed to deserialize non normalized string %s : %v\n%#v", tc.normalizedString, err, normSet) + } + + nonNormSetJson, err := nonNormSet.ToJSON() + if err != nil { + t.Fatalf("Failed to serialize non-normalized set: %v\n%#v", err, nonNormSet) + + } + + normSetJson, err := normSet.ToJSON() + if err != nil { + t.Fatalf("Failed to serialize normalized set: %v\n%#v", err, normSet) + + } + + if diff := cmp.Diff(nonNormSetJson, normSetJson); diff != "" { + t.Errorf("diff should be non empty, diff %s", diff) + } + + // this should exceeds if the diff was "". + if !normSet.Equals(nonNormSet) { + t.Errorf("expected both set to be equal.") + } + + }) + } +} func TestDropUnknown(t *testing.T) { input := `{"f:aaa":{},"r:aab":{}}` expect := `{"f:aaa":{}}` diff --git a/fieldpath/set.go b/fieldpath/set.go index ac7de802..d2d8c8a4 100644 --- a/fieldpath/set.go +++ b/fieldpath/set.go @@ -17,10 +17,14 @@ limitations under the License. package fieldpath import ( + "fmt" + "iter" "sort" "strings" - "sigs.k8s.io/structured-merge-diff/v5/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" + + "sigs.k8s.io/structured-merge-diff/v6/schema" ) // Set identifies a set of fields. @@ -44,6 +48,15 @@ func NewSet(paths ...Path) *Set { return s } +// Copy returns a copy of the Set. +// This is not a full deep copy as any contained value.Value is not copied. +func (s *Set) Copy() *Set { + return &Set{ + Members: s.Members.Copy(), + Children: s.Children.Copy(), + } +} + // Insert adds the field identified by `p` to the set. Important: parent fields // are NOT added to the set; if that is desired, they must be added separately. func (s *Set) Insert(p Path) { @@ -136,6 +149,198 @@ func (s *Set) EnsureNamedFieldsAreMembers(sc *schema.Schema, tr schema.TypeRef) } } +// MakePrefixMatcherOrDie is the same as PrefixMatcher except it panics if parts can't be +// turned into a SetMatcher. +func MakePrefixMatcherOrDie(parts ...interface{}) *SetMatcher { + result, err := PrefixMatcher(parts...) + if err != nil { + panic(err) + } + return result +} + +// PrefixMatcher creates a SetMatcher that matches all field paths prefixed by the given list of matcher path parts. +// The matcher parts may any of: +// +// - PathElementMatcher - for wildcards, `MatchAnyPathElement()` can be used as well. +// - PathElement - for any path element +// - value.FieldList - for listMap keys +// - value.Value - for scalar list elements +// - string - For field names +// - int - for array indices +func PrefixMatcher(parts ...interface{}) (*SetMatcher, error) { + current := MatchAnySet() // match all field path suffixes + for i := len(parts) - 1; i >= 0; i-- { + part := parts[i] + var pattern PathElementMatcher + switch t := part.(type) { + case PathElementMatcher: + // any path matcher, including wildcard + pattern = t + case PathElement: + // any path element + pattern = PathElementMatcher{PathElement: t} + case *value.FieldList: + // a listMap key + if len(*t) == 0 { + return nil, fmt.Errorf("associative list key type path elements must have at least one key (got zero)") + } + pattern = PathElementMatcher{PathElement: PathElement{Key: t}} + case value.Value: + // a scalar or set-type list element + pattern = PathElementMatcher{PathElement: PathElement{Value: &t}} + case string: + // a plain field name + pattern = PathElementMatcher{PathElement: PathElement{FieldName: &t}} + case int: + // a plain list index + pattern = PathElementMatcher{PathElement: PathElement{Index: &t}} + default: + return nil, fmt.Errorf("unexpected type %T", t) + } + current = &SetMatcher{ + members: []*SetMemberMatcher{{ + Path: pattern, + Child: current, + }}, + } + } + return current, nil +} + +// MatchAnyPathElement returns a PathElementMatcher that matches any path element. +func MatchAnyPathElement() PathElementMatcher { + return PathElementMatcher{Wildcard: true} +} + +// MatchAnySet returns a SetMatcher that matches any set. +func MatchAnySet() *SetMatcher { + return &SetMatcher{wildcard: true} +} + +// NewSetMatcher returns a new SetMatcher. +// Wildcard members take precedent over non-wildcard members; +// all non-wildcard members are ignored if there is a wildcard members. +func NewSetMatcher(wildcard bool, members ...*SetMemberMatcher) *SetMatcher { + sort.Sort(sortedMemberMatcher(members)) + return &SetMatcher{wildcard: wildcard, members: members} +} + +// SetMatcher defines a matcher that matches fields in a Set. +// SetMatcher is structured much like a Set but with wildcard support. +type SetMatcher struct { + // wildcard indicates that all members and children are included in the match. + // If set, the members field is ignored. + wildcard bool + // members provides patterns to match the members of a Set. + // Wildcard members are sorted before non-wildcards and take precedent over + // non-wildcard members. + members sortedMemberMatcher +} + +type sortedMemberMatcher []*SetMemberMatcher + +func (s sortedMemberMatcher) Len() int { return len(s) } +func (s sortedMemberMatcher) Less(i, j int) bool { return s[i].Path.Less(s[j].Path) } +func (s sortedMemberMatcher) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s sortedMemberMatcher) Find(p PathElementMatcher) (location int, ok bool) { + return sort.Find(len(s), func(i int) int { + return s[i].Path.Compare(p) + }) +} + +// Merge merges s and s2 and returns a SetMatcher that matches all field paths matched by either s or s2. +// During the merge, members of s and s2 with the same PathElementMatcher merged into a single member +// with the children of each merged by calling this function recursively. +func (s *SetMatcher) Merge(s2 *SetMatcher) *SetMatcher { + if s.wildcard || s2.wildcard { + return NewSetMatcher(true) + } + merged := make(sortedMemberMatcher, len(s.members), len(s.members)+len(s2.members)) + copy(merged, s.members) + for _, m := range s2.members { + if i, ok := s.members.Find(m.Path); ok { + // since merged is a shallow copy, do not modify elements in place + merged[i] = &SetMemberMatcher{ + Path: merged[i].Path, + Child: merged[i].Child.Merge(m.Child), + } + } else { + merged = append(merged, m) + } + } + return NewSetMatcher(false, merged...) // sort happens here +} + +// SetMemberMatcher defines a matcher that matches the members of a Set. +// SetMemberMatcher is structured much like the elements of a SetNodeMap, but +// with wildcard support. +type SetMemberMatcher struct { + // Path provides a matcher to match members of a Set. + // If Path is a wildcard, all members of a Set are included in the match. + // Otherwise, if any Path is Equal to a member of a Set, that member is + // included in the match and the children of that member are matched + // against the Child matcher. + Path PathElementMatcher + + // Child provides a matcher to use for the children of matched members of a Set. + Child *SetMatcher +} + +// PathElementMatcher defined a path matcher for a PathElement. +type PathElementMatcher struct { + // Wildcard indicates that all PathElements are matched by this matcher. + // If set, PathElement is ignored. + Wildcard bool + + // PathElement indicates that a PathElement is matched if it is Equal + // to this PathElement. + PathElement +} + +func (p PathElementMatcher) Equals(p2 PathElementMatcher) bool { + return p.Wildcard != p2.Wildcard && p.PathElement.Equals(p2.PathElement) +} + +func (p PathElementMatcher) Less(p2 PathElementMatcher) bool { + if p.Wildcard && !p2.Wildcard { + return true + } else if p2.Wildcard { + return false + } + return p.PathElement.Less(p2.PathElement) +} + +func (p PathElementMatcher) Compare(p2 PathElementMatcher) int { + if p.Wildcard && !p2.Wildcard { + return -1 + } else if p2.Wildcard { + return 1 + } + return p.PathElement.Compare(p2.PathElement) +} + +// FilterIncludeMatches returns a Set with only the field paths that match. +func (s *Set) FilterIncludeMatches(pattern *SetMatcher) *Set { + if pattern.wildcard { + return s + } + + members := PathElementSet{} + for _, m := range s.Members.members { + for _, pm := range pattern.members { + if pm.Path.Wildcard || pm.Path.PathElement.Equals(m) { + members.Insert(m) + break + } + } + } + return &Set{ + Members: members, + Children: *s.Children.FilterIncludeMatches(pattern), + } +} + // Size returns the number of members of the set. func (s *Set) Size() int { return s.Members.Size() + s.Children.Size() @@ -191,6 +396,15 @@ func (s *Set) Iterate(f func(Path)) { s.iteratePrefix(Path{}, f) } +// All iterates over each Path in the set (preorder DFS). +func (s *Set) All() iter.Seq[Path] { + return func(yield func(Path) bool) { + s.Iterate(func(p Path) { + yield(p) + }) + } +} + func (s *Set) iteratePrefix(prefix Path, f func(Path)) { s.Members.Iterate(func(pe PathElement) { f(append(prefix, pe)) }) s.Children.iteratePrefix(prefix, f) @@ -260,6 +474,16 @@ func (s sortedSetNode) Len() int { return len(s) } func (s sortedSetNode) Less(i, j int) bool { return s[i].pathElement.Less(s[j].pathElement) } func (s sortedSetNode) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +// Copy returns a copy of the SetNodeMap. +// This is not a full deep copy as any contained value.Value is not copied. +func (s *SetNodeMap) Copy() SetNodeMap { + out := make(sortedSetNode, len(s.members)) + for i, v := range s.members { + out[i] = setNode{pathElement: v.pathElement.Copy(), set: v.set.Copy()} + } + return SetNodeMap{members: out} +} + // Descend adds pe to the set if necessary, returning the associated subset. func (s *SetNodeMap) Descend(pe PathElement) *Set { loc := sort.Search(len(s.members), func(i int) bool { @@ -476,6 +700,33 @@ func (s *SetNodeMap) EnsureNamedFieldsAreMembers(sc *schema.Schema, tr schema.Ty } } +// FilterIncludeMatches returns a SetNodeMap with only the field paths that match the matcher. +func (s *SetNodeMap) FilterIncludeMatches(pattern *SetMatcher) *SetNodeMap { + if pattern.wildcard { + return s + } + + var out sortedSetNode + for _, member := range s.members { + for _, c := range pattern.members { + if c.Path.Wildcard || c.Path.PathElement.Equals(member.pathElement) { + childSet := member.set.FilterIncludeMatches(c.Child) + if childSet.Size() > 0 { + out = append(out, setNode{ + pathElement: member.pathElement, + set: childSet, + }) + } + break + } + } + } + + return &SetNodeMap{ + members: out, + } +} + // Iterate calls f for each PathElement in the set. func (s *SetNodeMap) Iterate(f func(PathElement)) { for _, n := range s.members { @@ -483,6 +734,15 @@ func (s *SetNodeMap) Iterate(f func(PathElement)) { } } +// All iterates over each PathElement in the set. +func (s *SetNodeMap) All() iter.Seq[PathElement] { + return func(yield func(PathElement) bool) { + s.Iterate(func(pe PathElement) { + yield(pe) + }) + } +} + func (s *SetNodeMap) iteratePrefix(prefix Path, f func(Path)) { for _, n := range s.members { pe := n.pathElement @@ -503,3 +763,59 @@ func (s *SetNodeMap) Leaves() *SetNodeMap { } return out } + +// Filter defines an interface for excluding field paths from a set. +// NewExcludeSetFilter can be used to create a filter that removes +// specific field paths and all of their children. +// NewIncludeMatcherFilter can be used to create a filter that removes all fields except +// the fields that match a field path matcher. PrefixMatcher and MakePrefixMatcherOrDie +// can be used to define field path patterns. +type Filter interface { + // Filter returns a filtered copy of the set. + Filter(*Set) *Set +} + +// NewExcludeSetFilter returns a filter that removes field paths in the exclude set. +func NewExcludeSetFilter(exclude *Set) Filter { + return excludeFilter{exclude} +} + +// NewExcludeFilterSetMap converts a map of APIVersion to exclude set to a map of APIVersion to exclude filters. +func NewExcludeFilterSetMap(resetFields map[APIVersion]*Set) map[APIVersion]Filter { + result := make(map[APIVersion]Filter) + for k, v := range resetFields { + result[k] = excludeFilter{v} + } + return result +} + +type excludeFilter struct { + excludeSet *Set +} + +func (t excludeFilter) Filter(set *Set) *Set { + return set.RecursiveDifference(t.excludeSet) +} + +// NewIncludeMatcherFilter returns a filter that only includes field paths that match. +// If no matchers are provided, the filter includes all field paths. +// PrefixMatcher and MakePrefixMatcherOrDie can help create basic matcher. +func NewIncludeMatcherFilter(matchers ...*SetMatcher) Filter { + if len(matchers) == 0 { + return includeMatcherFilter{&SetMatcher{wildcard: true}} + } + matcher := matchers[0] + for i := 1; i < len(matchers); i++ { + matcher = matcher.Merge(matchers[i]) + } + + return includeMatcherFilter{matcher} +} + +type includeMatcherFilter struct { + matcher *SetMatcher +} + +func (pf includeMatcherFilter) Filter(set *Set) *Set { + return set.FilterIncludeMatches(pf.matcher) +} diff --git a/fieldpath/set_test.go b/fieldpath/set_test.go index 364c4e78..54131400 100644 --- a/fieldpath/set_test.go +++ b/fieldpath/set_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fieldpath +package fieldpath_test import ( "bytes" @@ -22,8 +22,9 @@ import ( "math/rand" "testing" - "gopkg.in/yaml.v2" - "sigs.k8s.io/structured-merge-diff/v5/schema" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type randomPathAlphabet []PathElement @@ -47,17 +48,38 @@ var randomPathMaker = randomPathAlphabet(MakePathOrDie( "aad", "aae", "aaf", + // All alphabets + "abcdefghijklmnopqrstuvwxyz", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", + // Keys + KeyByFields("name", "привет"), + KeyByFields("name", "你好"), + KeyByFields("name", "こんにちは"), + KeyByFields("name", "안녕하세요"), + KeyByFields("name", "مرحبا"), KeyByFields("name", "first"), KeyByFields("name", "second"), KeyByFields("port", 443, "protocol", "tcp"), KeyByFields("port", 443, "protocol", "udp"), + KeyByFields("key", " value with spaces "), + KeyByFields("lang", "en-US"), + KeyByFields("unicode-key", "unicode-value-🔥"), + // Values _V(1), _V(2), _V(3), _V("aa"), _V("ab"), _V(true), - 1, 2, 3, 4, + _V(0), + _V(-1), + _V(3.14159), + _V("string with spaces"), + _V("string with \"quotes\""), + _V("unicode-string-你好"), + _V(false), + // Indices + 1, 2, 3, 4, 0, 5, 100, 2147483648, )) func BenchmarkFieldSet(b *testing.B) { @@ -231,7 +253,6 @@ func TestSetIterSize(t *testing.T) { ) s2 := NewSet() - addedCount := 0 s1.Iterate(func(p Path) { if s2.Size() != addedCount { @@ -244,6 +265,19 @@ func TestSetIterSize(t *testing.T) { addedCount++ }) + s2 = NewSet() + addedCount = 0 + for p := range s1.All() { + if s2.Size() != addedCount { + t.Errorf("added %v items to set, but size is %v", addedCount, s2.Size()) + } + if addedCount > 0 == s2.Empty() { + t.Errorf("added %v items to set, but s2.Empty() is %v", addedCount, s2.Empty()) + } + s2.Insert(p) + addedCount++ + } + if !s1.Equals(s2) { // No point in using String() if iterate is broken... t.Errorf("Iterate missed something?\n%#v\n%#v", s1, s2) @@ -650,80 +684,227 @@ func TestSetDifference(t *testing.T) { } } -var nestedSchema = func() (*schema.Schema, schema.TypeRef) { - sc := &schema.Schema{} +var nestedSchema = func() (*typed.Parser, string) { name := "type" - err := yaml.Unmarshal([]byte(`types: + parser := mustParse(`types: - name: type map: elementType: namedType: type fields: + - name: keyAStr + type: + scalar: string + - name: keyBInt + type: + scalar: numeric - name: named type: namedType: type - name: list type: list: - elementRelationShip: associative - keys: ["name"] + elementRelationship: associative + keys: ["keyAStr", "keyBInt"] elementType: namedType: type + - name: a + type: + namedType: type - name: value type: scalar: numeric -`), &sc) +`) + return parser, name +} + +var associativeListSchema = func() (*typed.Parser, string) { + name := "type" + parser := mustParse(`types: +- name: type + map: + fields: + - name: values + type: + list: + elementRelationship: associative + keys: ["keyAStr", "keyBInt"] + elementType: + map: + fields: + - name: keyAStr + type: + scalar: string + - name: keyBInt + type: + scalar: numeric + - name: value + type: + scalar: numeric +`) + return parser, name +} + +var oldAssociativeListSchema = func() (*typed.Parser, string) { + name := "type" + // No keyBInt yet! + parser := mustParse(`types: +- name: type + map: + fields: + - name: values + type: + list: + elementRelationship: associative + keys: ["keyAStr"] + elementType: + map: + fields: + - name: keyAStr + type: + scalar: string + - name: value + type: + scalar: numeric +`) + return parser, name +} + +func mustParse(schema typed.YAMLObject) *typed.Parser { + parser, err := typed.NewParser(schema) if err != nil { panic(err) } - return sc, schema.TypeRef{NamedType: &name} + return parser } -var _P = MakePathOrDie - func TestEnsureNamedFieldsAreMembers(t *testing.T) { table := []struct { - set, expected *Set + schemaFn func() (*typed.Parser, string) + newSchemaFn func() (*typed.Parser, string) + value typed.YAMLObject + expectedBefore *Set + expectedAfter *Set }{ { - set: NewSet(_P("named", "named", "value")), - expected: NewSet( + schemaFn: nestedSchema, + value: `{"named": {"named": {"value": 0}}}`, + expectedBefore: NewSet( + _P("named", "named", "value"), + ), + expectedAfter: NewSet( _P("named", "named", "value"), _P("named", "named"), _P("named"), ), }, { - set: NewSet(_P("named", "a", "named", "value"), _P("a", "named", "value"), _P("a", "b", "value")), - expected: NewSet( + schemaFn: nestedSchema, + value: `{"named": {"a": {"named": {"value": 42}}}, "a": {"named": {"value": 1}}}`, + expectedBefore: NewSet( + _P("named", "a", "named", "value"), + _P("a", "named", "value"), + ), + expectedAfter: NewSet( _P("named", "a", "named", "value"), _P("named", "a", "named"), + _P("named", "a"), _P("named"), _P("a", "named", "value"), _P("a", "named"), - _P("a", "b", "value"), + _P("a"), ), }, { - set: NewSet(_P("named", "list", KeyByFields("name", "a"), "named", "a", "value")), - expected: NewSet( - _P("named", "list", KeyByFields("name", "a"), "named", "a", "value"), - _P("named", "list", KeyByFields("name", "a"), "named"), + schemaFn: nestedSchema, + value: `{"named": {"list": [{"keyAStr": "a", "keyBInt": 1, "named": {"value": 0}}]}}`, + expectedBefore: NewSet( + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "keyAStr"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "keyBInt"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "named", "value"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1)), + ), + expectedAfter: NewSet( + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "keyAStr"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "keyBInt"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "named", "value"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1), "named"), + _P("named", "list", KeyByFields("keyAStr", "a", "keyBInt", 1)), _P("named", "list"), _P("named"), ), }, + { + // Generate the value using the old schema to get missing key entries, + // then process with new schema which has keyBInt. + schemaFn: oldAssociativeListSchema, + newSchemaFn: associativeListSchema, + value: `{"values": [{"keyAStr": "a", "value": 0}]}`, + expectedBefore: NewSet( + _P("values", KeyByFields("keyAStr", "a"), "keyAStr"), + _P("values", KeyByFields("keyAStr", "a"), "value"), + _P("values", KeyByFields("keyAStr", "a")), + ), + expectedAfter: NewSet( + _P("values", KeyByFields("keyAStr", "a"), "keyAStr"), + _P("values", KeyByFields("keyAStr", "a"), "value"), + _P("values", KeyByFields("keyAStr", "a")), + _P("values"), + ), + }, + { + // Check that converting the value with the missing key and + // the recent schema doesn't add the missing key. + schemaFn: associativeListSchema, + value: `{"values": [{"keyAStr": "a", "value": 1}]}`, + expectedBefore: NewSet( + _P("values", KeyByFields("keyAStr", "a"), "keyAStr"), + _P("values", KeyByFields("keyAStr", "a"), "value"), + _P("values", KeyByFields("keyAStr", "a")), + ), + expectedAfter: NewSet( + _P("values", KeyByFields("keyAStr", "a"), "keyAStr"), + _P("values", KeyByFields("keyAStr", "a"), "value"), + _P("values", KeyByFields("keyAStr", "a")), + _P("values"), + ), + }, } for _, test := range table { - t.Run(fmt.Sprintf("%v", test.set), func(t *testing.T) { - got := test.set.EnsureNamedFieldsAreMembers(nestedSchema()) - if !got.Equals(test.expected) { - t.Errorf("expected %v, got %v (missing: %v/superfluous: %v)", - test.expected, + t.Run(string(test.value), func(t *testing.T) { + parser, typeName := test.schemaFn() + typeRef := schema.TypeRef{NamedType: &typeName} + typedValue, err := parser.Type(typeName).FromYAML(test.value) + if err != nil { + t.Fatalf("unexpected error parsing test value: %v", err) + } + set, err := typedValue.ToFieldSet() + if err != nil { + t.Fatalf("unexpected error converting test value to set: %v", err) + } + if !set.Equals(test.expectedBefore) { + t.Errorf("expected before EnsureNamedFieldsAreMembers:\n%v\n\ngot:\n%v\n\nmissing:\n%v\n\nsuperfluous:\n\n%v", + test.expectedBefore, + set, + test.expectedAfter.Difference(set), + set.Difference(test.expectedAfter), + ) + } + + schema := &parser.Schema + if test.newSchemaFn != nil { + newParser, _ := test.newSchemaFn() + schema = &newParser.Schema + } + + got := set.EnsureNamedFieldsAreMembers(schema, typeRef) + if !got.Equals(test.expectedAfter) { + t.Errorf("expected after EnsureNamedFieldsAreMembers:\n%v\n\ngot:\n%v\n\nmissing:\n%v\n\nsuperfluous:\n\n%v", + test.expectedAfter, got, - test.expected.Difference(got), - got.Difference(test.expected), + test.expectedAfter.Difference(got), + got.Difference(test.expectedAfter), ) } }) @@ -754,4 +935,271 @@ func TestSetNodeMapIterate(t *testing.T) { t.Errorf("expected to have iterated over %v, but never did", pe) } } + + iteratedElements = make(map[string]bool, toAdd) + for pe := range set.All() { + iteratedElements[pe.String()] = true + } + if len(iteratedElements) != toAdd { + t.Errorf("expected %v elements to be iterated over, got %v", toAdd, len(iteratedElements)) + } + for _, pe := range addedElements { + if _, ok := iteratedElements[pe]; !ok { + t.Errorf("expected to have iterated over %v, but never did", pe) + } + } +} + +func TestFilterByPattern(t *testing.T) { + testCases := []struct { + name string + input *Set + expect *Set + filter Filter + }{ + { + name: "container resize fields: exact match", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "containers"), + MakePathOrDie("spec", "containers", 0), + MakePathOrDie("spec", "containers", 0, "resources"), + MakePathOrDie("spec", "containers", 0, "resources", "limits"), + MakePathOrDie("spec", "containers", 0, "resources", "limits", "cpu"), + MakePathOrDie("spec", "containers", 0, "resources", "requests"), + MakePathOrDie("spec", "containers", 0, "resources", "requests", "cpu"), + MakePathOrDie("spec", "containers", 0, "resources", "claims"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 0), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 0, "name"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 0, "request"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 1), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 1, "name"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 1, "request"), + MakePathOrDie("spec", "containers", 1), + MakePathOrDie("spec", "containers", 1, "resources"), + MakePathOrDie("spec", "containers", 1, "resources", "limits"), + MakePathOrDie("spec", "containers", 1, "resources", "limits", "cpu"), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "containers", MatchAnyPathElement(), "resources")), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "containers"), + MakePathOrDie("spec", "containers", 0), + MakePathOrDie("spec", "containers", 0, "resources"), + MakePathOrDie("spec", "containers", 0, "resources", "limits"), + MakePathOrDie("spec", "containers", 0, "resources", "limits", "cpu"), + MakePathOrDie("spec", "containers", 0, "resources", "requests"), + MakePathOrDie("spec", "containers", 0, "resources", "requests", "cpu"), + MakePathOrDie("spec", "containers", 0, "resources", "claims"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 0), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 0, "name"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 0, "request"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 1), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 1, "name"), + MakePathOrDie("spec", "containers", 0, "resources", "claims", 1, "request"), + MakePathOrDie("spec", "containers", 1), + MakePathOrDie("spec", "containers", 1, "resources"), + MakePathOrDie("spec", "containers", 1, "resources", "limits"), + MakePathOrDie("spec", "containers", 1, "resources", "limits", "cpu"), + ), + }, + { + name: "container resize fields: filter status and metadata", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("status"), + MakePathOrDie("metadata"), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "containers", MatchAnyPathElement(), "resources")), + expect: NewSet( + MakePathOrDie("spec"), + ), + }, + { + name: "container resize fields: filter non-container spec fields", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "volumes"), + MakePathOrDie("spec", "hostNetwork"), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "containers", MatchAnyPathElement(), "resources")), + expect: NewSet( + MakePathOrDie("spec"), + ), + }, + { + name: "container resize fields: filter non-resource container fields", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "containers"), + MakePathOrDie("spec", "containers", 0), + MakePathOrDie("spec", "containers", 0, "image"), + MakePathOrDie("spec", "containers", 0, "workingDir"), + MakePathOrDie("spec", "containers", 0, "resources"), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "containers", MatchAnyPathElement(), "resources")), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "containers"), + MakePathOrDie("spec", "containers", 0), + MakePathOrDie("spec", "containers", 0, "resources"), + ), + }, + { + name: "filter listMap key", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "listMap", + &value.FieldList{ + {Name: "key1", Value: value.NewValueInterface("value1")}, + {Name: "key2", Value: value.NewValueInterface("value2")}, + }), + MakePathOrDie("spec", "listMap", + &value.FieldList{ + {Name: "key1", Value: value.NewValueInterface("value1")}, + {Name: "key2", Value: value.NewValueInterface("value2")}, + }, "field"), + MakePathOrDie("spec", "listMap", + &value.FieldList{ + {Name: "key1", Value: value.NewValueInterface("valueX")}, + {Name: "key2", Value: value.NewValueInterface("valueY")}, + }, "field"), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "listMap", &value.FieldList{ + {Name: "key1", Value: value.NewValueInterface("value1")}, + {Name: "key2", Value: value.NewValueInterface("value2")}, + })), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "listMap", + &value.FieldList{ + {Name: "key1", Value: value.NewValueInterface("value1")}, + {Name: "key2", Value: value.NewValueInterface("value2")}, + }), + MakePathOrDie("spec", "listMap", + &value.FieldList{ + {Name: "key1", Value: value.NewValueInterface("value1")}, + {Name: "key2", Value: value.NewValueInterface("value2")}, + }, "field"), + ), + }, + { + name: "filter value", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "set", value.NewValueInterface("v1")), + MakePathOrDie("spec", "set", value.NewValueInterface("v2")), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "set", value.NewValueInterface("v1"))), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "set", value.NewValueInterface("v1")), + ), + }, + { + name: "filter by index", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "list"), + MakePathOrDie("spec", "list", 0), + MakePathOrDie("spec", "list", 0, "value"), + MakePathOrDie("spec", "list", 1), + MakePathOrDie("spec", "list", 1, "value"), + ), + filter: NewIncludeMatcherFilter(MakePrefixMatcherOrDie("spec", "list", 1, "value")), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "list"), + MakePathOrDie("spec", "list", 1), + MakePathOrDie("spec", "list", 1, "value"), + ), + }, + { + name: "multiple index matchers", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "list"), + MakePathOrDie("spec", "list", 0), + MakePathOrDie("spec", "list", 0, "value"), + MakePathOrDie("spec", "list", 1), + MakePathOrDie("spec", "list", 1, "value"), + MakePathOrDie("spec", "list", 2), + MakePathOrDie("spec", "list", 2, "value"), + ), + filter: NewIncludeMatcherFilter( + MakePrefixMatcherOrDie("spec", "list", 0, "value"), + MakePrefixMatcherOrDie("spec", "list", 1, "value"), + ), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "list"), + MakePathOrDie("spec", "list", 0), + MakePathOrDie("spec", "list", 0, "value"), + MakePathOrDie("spec", "list", 1), + MakePathOrDie("spec", "list", 1, "value"), + ), + }, + { + name: "multiple field matchers", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "f1"), + MakePathOrDie("spec", "f1", "f11"), + MakePathOrDie("spec", "f2"), + MakePathOrDie("spec", "f2", "f21"), + MakePathOrDie("spec", "f3"), + MakePathOrDie("spec", "f3", "f31"), + ), + filter: NewIncludeMatcherFilter( + MakePrefixMatcherOrDie("spec", "f1"), + MakePrefixMatcherOrDie("spec", "f3"), + ), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "f1"), + MakePathOrDie("spec", "f1", "f11"), + MakePathOrDie("spec", "f3"), + MakePathOrDie("spec", "f3", "f31"), + ), + }, + { + name: "wildcard takes precedence", + input: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "list"), + MakePathOrDie("spec", "list", 0), + MakePathOrDie("spec", "list", 0, "f1"), + MakePathOrDie("spec", "list", 0, "f2"), + MakePathOrDie("spec", "list", 1), + MakePathOrDie("spec", "list", 1, "f1"), + MakePathOrDie("spec", "list", 1, "f2"), + MakePathOrDie("spec", "list", 2), + MakePathOrDie("spec", "list", 2, "f1"), + MakePathOrDie("spec", "list", 2, "f2"), + ), + filter: NewIncludeMatcherFilter( + MakePrefixMatcherOrDie("spec", "list", MatchAnyPathElement(), "f1"), // takes precedence + MakePrefixMatcherOrDie("spec", "list", 1, "f2"), // ignored + ), + expect: NewSet( + MakePathOrDie("spec"), + MakePathOrDie("spec", "list"), + MakePathOrDie("spec", "list", 0), + MakePathOrDie("spec", "list", 0, "f1"), + MakePathOrDie("spec", "list", 1), + MakePathOrDie("spec", "list", 1, "f1"), + MakePathOrDie("spec", "list", 2), + MakePathOrDie("spec", "list", 2, "f1"), + ), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filtered := tc.filter.Filter(tc.input) + if !filtered.Equals(tc.expect) { + t.Errorf("Expected:\n%v\n\nbut got:\n%v", tc.expect, filtered) + } + }) + } } diff --git a/go.mod b/go.mod index 5d0ac105..f5343b69 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,15 @@ -module sigs.k8s.io/structured-merge-diff/v5 - -require gopkg.in/yaml.v2 v2.2.8 +module sigs.k8s.io/structured-merge-diff/v6 require ( - github.com/google/gofuzz v1.0.0 + github.com/google/go-cmp v0.5.9 github.com/json-iterator/go v1.1.12 + go.yaml.in/yaml/v2 v2.4.2 + sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016 +) + +require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect ) -go 1.13 +go 1.23 diff --git a/go.sum b/go.sum index bf04c8ea..e120aad1 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -15,7 +16,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016 h1:kXv6kKdoEtedwuqMmkqhbkgvYKeycVbC8+iPCP9j5kQ= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/hack/apidiff.sh b/hack/apidiff.sh new file mode 100755 index 00000000..ce5bb42b --- /dev/null +++ b/hack/apidiff.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# apidiff.sh: Compare public API changes between revisions or directories using Git worktrees. + +set -euo pipefail + +# Usage Information +usage() { + echo "Usage: $0 [-r ] [-t ] [directory ...]" + echo " -t : Report changes in code up to and including this revision." + echo " Default is the current working tree instead of a revision." + echo " -r : Report changes in code added since this revision." + echo " Default is the common base of master and HEAD." + exit 1 +} + +# Default Values +TARGET_REVISION="" # -t: Target revision +REFERENCE_REVISION="" # -r: Reference revision +TARGET_DIR="." # Default directory to compare is current working directory +API_DIFF_TOOL="apidiff" +REF_API_SNAPSHOT="ref.api" +TGT_API_SNAPSHOT="target.api" +WORKTREES=() # Track created worktrees for cleanup + +# Parse Command-Line Arguments +while getopts ":t:r:" opt; do + case ${opt} in + t) TARGET_REVISION="$OPTARG" ;; + r) REFERENCE_REVISION="$OPTARG" ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage ;; + esac +done +shift $((OPTIND - 1)) + +# Remaining arguments are directories +if [ "$#" -ge 1 ]; then + TARGET_DIR="$1" +fi + +# Debug print to see all traps +trap -p + +# Step 1: Create a temporary directory structure under _output +mkdir -p "_output" +TMP_DIR=$(mktemp -d "_output/apidiff.XXXXXX") +TMP_DIR=$(cd "${TMP_DIR}" && pwd) # Convert to absolute path +TEMP_GOBIN="${TMP_DIR}/gobin" +TEMP_WORKTREES="${TMP_DIR}/worktrees" +mkdir -p "${TEMP_GOBIN}" "${TEMP_WORKTREES}" + +# Single trap for cleanup +trap 'cleanup' EXIT + +# shellcheck disable=SC2317 +cleanup() { + # Remove all created worktrees + for worktree in "${WORKTREES[@]}"; do + git worktree remove --force "$worktree" + done + # Remove temporary directory with all contents + rm -rf "${TMP_DIR}" +} + +# Update GOBIN to use temporary location +if ! command -v "${API_DIFF_TOOL}" &> /dev/null; then + echo "Installing apidiff into ${TEMP_GOBIN}" + GOBIN="${TEMP_GOBIN}" go install golang.org/x/exp/cmd/apidiff@latest + # Add GOBIN to PATH + export PATH=$PATH:${TEMP_GOBIN} +fi + +# Set target revision: PULL_PULL_SHA > target > HEAD +if [ -z "${TARGET_REVISION}" ] && [ -n "${PULL_PULL_SHA:-}" ]; then + TARGET_REVISION="${PULL_PULL_SHA}" +elif [ -z "${TARGET_REVISION}" ]; then + TARGET_REVISION="HEAD" +fi + +# Verify target commit exists +TARGET_REVISION="$(git rev-parse --verify "${TARGET_REVISION}")" + +# Try to determine base revision if not explicitly set +if [ -z "${REFERENCE_REVISION}" ]; then + if [ -n "${PULL_BASE_SHA:-}" ]; then + # Use PULL_BASE_SHA directly as the base + REFERENCE_REVISION="${PULL_BASE_SHA}" + else + # Fall back to merge-base with origin/master + if ! REFERENCE_REVISION="$(git merge-base origin/master "${TARGET_REVISION}")"; then + echo "Error: Could not determine base revision. Please configure git remote 'origin' or use -r explicitly." >&2 + exit 1 + fi + fi +fi + +# Verify base commit exists +REFERENCE_REVISION="$(git rev-parse --verify "${REFERENCE_REVISION}")" + +# Step 2: Export API snapshot for the reference revision +REF_WORKTREE="${TEMP_WORKTREES}/ref" +echo "Creating Git worktree for reference revision: ${REFERENCE_REVISION}" +git worktree add "${REF_WORKTREE}" "${REFERENCE_REVISION}" --quiet +WORKTREES+=("${REF_WORKTREE}") +echo "Exporting API snapshot for reference revision..." +pushd "${REF_WORKTREE}" > /dev/null +"${API_DIFF_TOOL}" -m -w "${TMP_DIR}/${REF_API_SNAPSHOT}" "${TARGET_DIR}" +popd > /dev/null + +# Step 3: Export API snapshot for the target revision +TGT_WORKTREE="${TEMP_WORKTREES}/target" +if [ -n "${TARGET_REVISION}" ]; then + echo "Creating Git worktree for target revision: ${TARGET_REVISION}" + git worktree add "${TGT_WORKTREE}" "${TARGET_REVISION}" --quiet + WORKTREES+=("${TGT_WORKTREE}") + TGT_PATH="${TGT_WORKTREE}" +else + # If no target revision specified, compare with current working tree + TGT_PATH="${TARGET_DIR}" +fi + +echo "Exporting API snapshot for target revision..." +pushd "${TGT_PATH}" > /dev/null +"${API_DIFF_TOOL}" -m -w "${TMP_DIR}/${TGT_API_SNAPSHOT}" "${TARGET_DIR}" +popd > /dev/null + +# Step 4: Compare the two API snapshots for incompatible changes +# Step 4: Compare the two API snapshots for changes +echo "Checking for API changes..." +# All changes +all_changes=$("${API_DIFF_TOOL}" -m "${TMP_DIR}/${REF_API_SNAPSHOT}" "${TMP_DIR}/${TGT_API_SNAPSHOT}" 2>&1 | grep -v -e "^Ignoring internal package" || true) +# Incompatible changes +incompatible_changes=$("${API_DIFF_TOOL}" -incompatible -m "${TMP_DIR}/${REF_API_SNAPSHOT}" "${TMP_DIR}/${TGT_API_SNAPSHOT}" 2>&1 | grep -v -e "^Ignoring internal package" || true) + +# Print out results +echo +echo "API compatibility check completed." +res=0 +if [ -n "$incompatible_changes" ]; then + res=1 + echo "Incompatible API changes found!" +else + echo "No incompatible API changes found." +fi +if [ -z "$all_changes" ]; then + echo "No API changes found." +else + echo "All API changes:" + echo "$all_changes" + echo +fi + +exit ${res} \ No newline at end of file diff --git a/internal/cli/operation.go b/internal/cli/operation.go index 84185bbd..e80d5383 100644 --- a/internal/cli/operation.go +++ b/internal/cli/operation.go @@ -21,8 +21,8 @@ import ( "io" "io/ioutil" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type Operation interface { @@ -153,7 +153,7 @@ func (c compare) Execute(w io.Writer) error { // TODO: I think it'd be neat if we actually emitted a machine-readable // format. - _, err = fmt.Fprintf(w, got.String()) + _, err = fmt.Fprint(w, got.String()) return err } diff --git a/internal/cli/options.go b/internal/cli/options.go index ce46053d..66626ffd 100644 --- a/internal/cli/options.go +++ b/internal/cli/options.go @@ -24,7 +24,7 @@ import ( "io/ioutil" "os" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var ( diff --git a/internal/fixture/state.go b/internal/fixture/state.go index 619951b3..efa35480 100644 --- a/internal/fixture/state.go +++ b/internal/fixture/state.go @@ -20,10 +20,11 @@ import ( "bytes" "fmt" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + "github.com/google/go-cmp/cmp" + + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) // For the sake of tests, a parser is something that can retrieve a @@ -130,7 +131,7 @@ func (s *State) UpdateObject(tv *typed.TypedValue, version fieldpath.APIVersion, // Update the current state with the passed in object func (s *State) Update(obj typed.YAMLObject, version fieldpath.APIVersion, manager string) error { - tv, err := s.Parser.Type(string(version)).FromYAML(FixTabsOrDie(obj)) + tv, err := s.Parser.Type(string(version)).FromYAML(FixTabsOrDie(obj), typed.AllowDuplicates) if err != nil { return err } @@ -168,20 +169,47 @@ func (s *State) Apply(obj typed.YAMLObject, version fieldpath.APIVersion, manage // CompareLive takes a YAML string and returns the comparison with the // current live object or an error. -func (s *State) CompareLive(obj typed.YAMLObject, version fieldpath.APIVersion) (*typed.Comparison, error) { +func (s *State) CompareLive(obj typed.YAMLObject, version fieldpath.APIVersion) (string, error) { obj = FixTabsOrDie(obj) if err := s.checkInit(version); err != nil { - return nil, err + return "", err } - tv, err := s.Parser.Type(string(version)).FromYAML(obj) + tv, err := s.Parser.Type(string(version)).FromYAML(obj, typed.AllowDuplicates) if err != nil { - return nil, err + return "", err } live, err := s.Updater.Converter.Convert(s.Live, version) if err != nil { - return nil, err + return "", err + } + tvu := convertMapAnyToMapString(tv.AsValue().Unstructured()) + liveu := convertMapAnyToMapString(live.AsValue().Unstructured()) + return cmp.Diff(tvu, liveu), nil +} + +func convertMapAnyToMapString(obj interface{}) interface{} { + switch m := obj.(type) { + case map[string]interface{}: + out := map[string]interface{}{} + for key, value := range m { + out[key] = convertMapAnyToMapString(value) + } + return out + case map[interface{}]interface{}: + out := map[string]interface{}{} + for key, value := range m { + out[key.(string)] = convertMapAnyToMapString(value) + } + return out + case []interface{}: + out := []interface{}{} + for _, value := range m { + out = append(out, convertMapAnyToMapString(value)) + } + return out + default: + return obj } - return live.Compare(tv) } // dummyConverter doesn't convert, it just returns the same exact object, as long as a version is provided. @@ -433,7 +461,7 @@ func (u Update) run(state *State) error { } func (u Update) preprocess(parser Parser) (Operation, error) { - tv, err := parser.Type(string(u.APIVersion)).FromYAML(FixTabsOrDie(u.Object)) + tv, err := parser.Type(string(u.APIVersion)).FromYAML(FixTabsOrDie(u.Object), typed.AllowDuplicates) if err != nil { return nil, err } @@ -477,7 +505,7 @@ func (cs ChangeParser) run(state *State) error { // Swap the schema in for use with the live object so it merges. // If the schema is incompatible, this will fail validation. - liveWithNewSchema, err := typed.AsTyped(state.Live.AsValue(), cs.Parser.Schema, state.Live.TypeRef()) + liveWithNewSchema, err := typed.AsTyped(state.Live.AsValue(), &cs.Parser.Schema, state.Live.TypeRef()) if err != nil { return err } @@ -507,9 +535,14 @@ type TestCase struct { // Managed, if not nil, is the ManagedFields as expected // after all operations are run. Managed fieldpath.ManagedFields - // Set to true if the test case needs the union behavior enabled. - RequiresUnions bool - // IgnoredFields containing the set to ignore for every version + // ReportInputOnNoop if we don't want to compare the output and + // always return it. + ReturnInputOnNoop bool + // IgnoreFilter filters out ignored fields from a fieldpath.Set. + IgnoreFilter map[fieldpath.APIVersion]fieldpath.Filter + + // IgnoredFields containing the set to ignore for every version. + // IgnoredFields may not be set if IgnoreFilter is set. IgnoredFields map[fieldpath.APIVersion]*fieldpath.Set } @@ -541,13 +574,16 @@ func (tc TestCase) PreprocessOperations(parser Parser) error { // TestWithConverter once and reset the benchmark, to make sure the test case // actually passes.. func (tc TestCase) BenchWithConverter(parser Parser, converter merge.Converter) error { + updaterBuilder := merge.UpdaterBuilder{ + Converter: converter, + IgnoreFilter: tc.IgnoreFilter, + IgnoredFields: tc.IgnoredFields, + ReturnInputOnNoop: tc.ReturnInputOnNoop, + } state := State{ - Updater: &merge.Updater{Converter: converter, IgnoredFields: tc.IgnoredFields}, + Updater: updaterBuilder.BuildUpdater(), Parser: parser, } - if tc.RequiresUnions { - state.Updater.EnableUnionFeature() - } // We currently don't have any test that converts, we can take // care of that later. for i, ops := range tc.Ops { @@ -561,13 +597,16 @@ func (tc TestCase) BenchWithConverter(parser Parser, converter merge.Converter) // TestWithConverter runs the test-case using the given parser and converter. func (tc TestCase) TestWithConverter(parser Parser, converter merge.Converter) error { + updaterBuilder := merge.UpdaterBuilder{ + Converter: converter, + IgnoreFilter: tc.IgnoreFilter, + IgnoredFields: tc.IgnoredFields, + ReturnInputOnNoop: tc.ReturnInputOnNoop, + } state := State{ - Updater: &merge.Updater{Converter: converter, IgnoredFields: tc.IgnoredFields}, + Updater: updaterBuilder.BuildUpdater(), Parser: parser, } - if tc.RequiresUnions { - state.Updater.EnableUnionFeature() - } for i, ops := range tc.Ops { err := ops.run(&state) if err != nil { @@ -581,14 +620,14 @@ func (tc TestCase) TestWithConverter(parser Parser, converter merge.Converter) e if err != nil { return fmt.Errorf("failed to compare live with config: %v", err) } - if !comparison.IsSame() { - return fmt.Errorf("expected live and config to be the same:\n%v\nConfig: %v\n", comparison, value.ToString(state.Live.AsValue())) + if comparison != "" { + return fmt.Errorf("expected live and config to be the same:\n%v\n", comparison) } } if tc.Managed != nil { if diff := state.Managers.Difference(tc.Managed); len(diff) != 0 { - return fmt.Errorf("expected Managers to be:\n%v\ngot:\n%v", tc.Managed, state.Managers) + return fmt.Errorf("expected Managers to be:\n%v\ngot:\n%v\ndiff:\n%v", tc.Managed, state.Managers, diff) } } @@ -599,15 +638,5 @@ func (tc TestCase) TestWithConverter(parser Parser, converter merge.Converter) e } } - if !tc.RequiresUnions { - // Re-run the test with unions on. - tc2 := tc - tc2.RequiresUnions = true - err := tc2.TestWithConverter(parser, converter) - if err != nil { - return fmt.Errorf("fails if unions are on: %v", err) - } - } - return nil } diff --git a/internal/fixture/state_test.go b/internal/fixture/state_test.go index 745681d2..5b8bae7a 100644 --- a/internal/fixture/state_test.go +++ b/internal/fixture/state_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) func TestFixTabs(t *testing.T) { diff --git a/internal/testdata/k8s-deployment.yaml b/internal/testdata/k8s-deployment.yaml index 899ca327..ddacf2e4 100644 --- a/internal/testdata/k8s-deployment.yaml +++ b/internal/testdata/k8s-deployment.yaml @@ -38,7 +38,7 @@ spec: optional: true containers: - name: kubedns - image: k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10 + image: registry.k8s.io/k8s-dns-kube-dns-amd64:1.14.10 resources: # TODO: Set memory limits when we've profiled the container for large # clusters, then set request = limit to keep this container in @@ -89,7 +89,7 @@ spec: - name: kube-dns-config mountPath: /kube-dns-config - name: dnsmasq - image: k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10 + image: registry.k8s.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10 livenessProbe: httpGet: path: /healthcheck/dnsmasq @@ -128,7 +128,7 @@ spec: - name: kube-dns-config mountPath: /etc/k8s/dns/dnsmasq-nanny - name: sidecar - image: k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10 + image: registry.k8s.io/k8s-dns-sidecar-amd64:1.14.10 livenessProbe: httpGet: path: /metrics diff --git a/internal/testdata/node.yaml b/internal/testdata/node.yaml index dd8681ae..b6e8aeb9 100644 --- a/internal/testdata/node.yaml +++ b/internal/testdata/node.yaml @@ -131,8 +131,8 @@ status: - grafana/grafana:4.4.2 sizeBytes: 287008013 - names: - - k8s.gcr.io/node-problem-detector@sha256:f95cab985c26b2f46e9bd43283e0bfa88860c14e0fb0649266babe8b65e9eb2b - - k8s.gcr.io/node-problem-detector:v0.4.1 + - registry.k8s.io/node-problem-detector@sha256:f95cab985c26b2f46e9bd43283e0bfa88860c14e0fb0649266babe8b65e9eb2b + - registry.k8s.io/node-problem-detector:v0.4.1 sizeBytes: 286572743 - names: - grafana/grafana@sha256:7ff7f9b2501a5d55b55ce3f58d21771b1c5af1f2a4ab7dbf11bef7142aae7033 @@ -151,76 +151,76 @@ status: - nginx:1.10.1 sizeBytes: 180708613 - names: - - k8s.gcr.io/fluentd-elasticsearch@sha256:b8c94527b489fb61d3d81ce5ad7f3ddbb7be71e9620a3a36e2bede2f2e487d73 - - k8s.gcr.io/fluentd-elasticsearch:v2.0.4 + - registry.k8s.io/fluentd-elasticsearch@sha256:b8c94527b489fb61d3d81ce5ad7f3ddbb7be71e9620a3a36e2bede2f2e487d73 + - registry.k8s.io/fluentd-elasticsearch:v2.0.4 sizeBytes: 135716379 - names: - nginx@sha256:00be67d6ba53d5318cd91c57771530f5251cfbe028b7be2c4b70526f988cfc9f - nginx:latest sizeBytes: 109357355 - names: - - k8s.gcr.io/kubernetes-dashboard-amd64@sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0 - - k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3 + - registry.k8s.io/kubernetes-dashboard-amd64@sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0 + - registry.k8s.io/kubernetes-dashboard-amd64:v1.8.3 sizeBytes: 102319441 - names: - gcr.io/google_containers/kube-proxy:v1.11.10-gke.5 - - k8s.gcr.io/kube-proxy:v1.11.10-gke.5 + - registry.k8s.io/kube-proxy:v1.11.10-gke.5 sizeBytes: 102279340 - names: - - k8s.gcr.io/event-exporter@sha256:7f9cd7cb04d6959b0aa960727d04fa86759008048c785397b7b0d9dff0007516 - - k8s.gcr.io/event-exporter:v0.2.3 + - registry.k8s.io/event-exporter@sha256:7f9cd7cb04d6959b0aa960727d04fa86759008048c785397b7b0d9dff0007516 + - registry.k8s.io/event-exporter:v0.2.3 sizeBytes: 94171943 - names: - - k8s.gcr.io/prometheus-to-sd@sha256:6c0c742475363d537ff059136e5d5e4ab1f512ee0fd9b7ca42ea48bc309d1662 - - k8s.gcr.io/prometheus-to-sd:v0.3.1 + - registry.k8s.io/prometheus-to-sd@sha256:6c0c742475363d537ff059136e5d5e4ab1f512ee0fd9b7ca42ea48bc309d1662 + - registry.k8s.io/prometheus-to-sd:v0.3.1 sizeBytes: 88077694 - names: - - k8s.gcr.io/fluentd-gcp-scaler@sha256:a5ace7506d393c4ed65eb2cbb6312c64ab357fcea16dff76b9055bc6e498e5ff - - k8s.gcr.io/fluentd-gcp-scaler:0.5.1 + - registry.k8s.io/fluentd-gcp-scaler@sha256:a5ace7506d393c4ed65eb2cbb6312c64ab357fcea16dff76b9055bc6e498e5ff + - registry.k8s.io/fluentd-gcp-scaler:0.5.1 sizeBytes: 86637208 - names: - - k8s.gcr.io/heapster-amd64@sha256:9fae0af136ce0cf4f88393b3670f7139ffc464692060c374d2ae748e13144521 - - k8s.gcr.io/heapster-amd64:v1.6.0-beta.1 + - registry.k8s.io/heapster-amd64@sha256:9fae0af136ce0cf4f88393b3670f7139ffc464692060c374d2ae748e13144521 + - registry.k8s.io/heapster-amd64:v1.6.0-beta.1 sizeBytes: 76016169 - names: - - k8s.gcr.io/ingress-glbc-amd64@sha256:31d36bbd9c44caffa135fc78cf0737266fcf25e3cf0cd1c2fcbfbc4f7309cc52 - - k8s.gcr.io/ingress-glbc-amd64:v1.1.1 + - registry.k8s.io/ingress-glbc-amd64@sha256:31d36bbd9c44caffa135fc78cf0737266fcf25e3cf0cd1c2fcbfbc4f7309cc52 + - registry.k8s.io/ingress-glbc-amd64:v1.1.1 sizeBytes: 67801919 - names: - - k8s.gcr.io/kube-addon-manager@sha256:d53486c3a0b49ebee019932878dc44232735d5622a51dbbdcec7124199020d09 - - k8s.gcr.io/kube-addon-manager:v8.7 + - registry.k8s.io/kube-addon-manager@sha256:d53486c3a0b49ebee019932878dc44232735d5622a51dbbdcec7124199020d09 + - registry.k8s.io/kube-addon-manager:v8.7 sizeBytes: 63322109 - names: - nginx@sha256:4aacdcf186934dcb02f642579314075910f1855590fd3039d8fa4c9f96e48315 - nginx:1.10-alpine sizeBytes: 54042627 - names: - - k8s.gcr.io/cpvpa-amd64@sha256:cfe7b0a11c9c8e18c87b1eb34fef9a7cbb8480a8da11fc2657f78dbf4739f869 - - k8s.gcr.io/cpvpa-amd64:v0.6.0 + - registry.k8s.io/cpvpa-amd64@sha256:cfe7b0a11c9c8e18c87b1eb34fef9a7cbb8480a8da11fc2657f78dbf4739f869 + - registry.k8s.io/cpvpa-amd64:v0.6.0 sizeBytes: 51785854 - names: - - k8s.gcr.io/cluster-proportional-autoscaler-amd64@sha256:003f98d9f411ddfa6ff6d539196355e03ddd69fa4ed38c7ffb8fec6f729afe2d - - k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + - registry.k8s.io/cluster-proportional-autoscaler-amd64@sha256:003f98d9f411ddfa6ff6d539196355e03ddd69fa4ed38c7ffb8fec6f729afe2d + - registry.k8s.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 sizeBytes: 49648481 - names: - - k8s.gcr.io/ip-masq-agent-amd64@sha256:1ffda57d87901bc01324c82ceb2145fe6a0448d3f0dd9cb65aa76a867cd62103 - - k8s.gcr.io/ip-masq-agent-amd64:v2.1.1 + - registry.k8s.io/ip-masq-agent-amd64@sha256:1ffda57d87901bc01324c82ceb2145fe6a0448d3f0dd9cb65aa76a867cd62103 + - registry.k8s.io/ip-masq-agent-amd64:v2.1.1 sizeBytes: 49612505 - names: - - k8s.gcr.io/k8s-dns-kube-dns-amd64@sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8 - - k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10 + - registry.k8s.io/k8s-dns-kube-dns-amd64@sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8 + - registry.k8s.io/k8s-dns-kube-dns-amd64:1.14.10 sizeBytes: 49549457 - names: - - k8s.gcr.io/rescheduler@sha256:156cfbfd05a5a815206fd2eeb6cbdaf1596d71ea4b415d3a6c43071dd7b99450 - - k8s.gcr.io/rescheduler:v0.4.0 + - registry.k8s.io/rescheduler@sha256:156cfbfd05a5a815206fd2eeb6cbdaf1596d71ea4b415d3a6c43071dd7b99450 + - registry.k8s.io/rescheduler:v0.4.0 sizeBytes: 48973149 - names: - - k8s.gcr.io/event-exporter@sha256:16ca66e2b5dc7a1ce6a5aafcb21d0885828b75cdfc08135430480f7ad2364adc - - k8s.gcr.io/event-exporter:v0.2.4 + - registry.k8s.io/event-exporter@sha256:16ca66e2b5dc7a1ce6a5aafcb21d0885828b75cdfc08135430480f7ad2364adc + - registry.k8s.io/event-exporter:v0.2.4 sizeBytes: 47261019 - names: - - k8s.gcr.io/coredns@sha256:db2bf53126ed1c761d5a41f24a1b82a461c85f736ff6e90542e9522be4757848 - - k8s.gcr.io/coredns:1.1.3 + - registry.k8s.io/coredns@sha256:db2bf53126ed1c761d5a41f24a1b82a461c85f736ff6e90542e9522be4757848 + - registry.k8s.io/coredns:1.1.3 sizeBytes: 45587362 - names: - prom/prometheus@sha256:483f4c9d7733699ba79facca9f8bcce1cef1af43dfc3e7c5a1882aa85f53cb74 diff --git a/internal/testdata/schema.yaml b/internal/testdata/schema.yaml index e85de20a..789d6c6e 100644 --- a/internal/testdata/schema.yaml +++ b/internal/testdata/schema.yaml @@ -87,10 +87,10 @@ types: list: elementType: scalar: string + elementRelationship: atomic - name: untyped map: fields: - name: elementRelationship type: scalar: string - diff --git a/merge/conflict.go b/merge/conflict.go index 6df10f33..dea64707 100644 --- a/merge/conflict.go +++ b/merge/conflict.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" ) // Conflict is a conflict on a specific field with the current manager of @@ -112,7 +112,7 @@ func ConflictsFromManagers(sets fieldpath.ManagedFields) Conflicts { set.Set().Iterate(func(p fieldpath.Path) { conflicts = append(conflicts, Conflict{ Manager: manager, - Path: p, + Path: p.Copy(), }) }) } diff --git a/merge/conflict_test.go b/merge/conflict_test.go index 90c0f916..85978420 100644 --- a/merge/conflict_test.go +++ b/merge/conflict_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/value" ) var ( @@ -92,3 +92,22 @@ func TestToSet(t *testing.T) { t.Fatalf("expected\n%v\n, but got\n%v\n", expected, actual) } } + +func TestConflictsFromManagers(t *testing.T) { + got := merge.ConflictsFromManagers(fieldpath.ManagedFields{ + "Bob": fieldpath.NewVersionedSet( + _NS( + _P("obj", "template", "obj", "list", _KBF("name", "a"), "id"), + _P("obj", "template", "obj", "list", _KBF("name", "a"), "key"), + ), + "v1", + false, + ), + }) + wanted := `conflicts with "Bob": +- .obj.template.obj.list[name="a"].id +- .obj.template.obj.list[name="a"].key` + if got.Error() != wanted { + t.Errorf("Got %v, wanted %v", got.Error(), wanted) + } +} diff --git a/merge/deduced_test.go b/merge/deduced_test.go index 6b0b1525..fcd47c1e 100644 --- a/merge/deduced_test.go +++ b/merge/deduced_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" ) func TestDeduced(t *testing.T) { diff --git a/merge/default_keys_test.go b/merge/default_keys_test.go index 3ea881aa..28f81038 100644 --- a/merge/default_keys_test.go +++ b/merge/default_keys_test.go @@ -16,10 +16,10 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) // portListParser sets the default value of key "protocol" to "TCP" @@ -163,6 +163,36 @@ func TestDefaultKeysFlat(t *testing.T) { ), }, }, + "apply_missing_undefaulted_defaulted_key": { + Ops: []Operation{ + Apply{ + Manager: "default", + APIVersion: "v1", + Object: ` + containerPorts: + - protocol: TCP + name: A + `, + }, + }, + APIVersion: "v1", + Object: ` + containerPorts: + - protocol: TCP + name: A + `, + Managed: fieldpath.ManagedFields{ + "default": fieldpath.NewVersionedSet( + _NS( + _P("containerPorts", _KBF("protocol", "TCP")), + _P("containerPorts", _KBF("protocol", "TCP"), "name"), + _P("containerPorts", _KBF("protocol", "TCP"), "protocol"), + ), + "v1", + true, + ), + }, + }, } for name, test := range tests { @@ -176,18 +206,6 @@ func TestDefaultKeysFlat(t *testing.T) { func TestDefaultKeysFlatErrors(t *testing.T) { tests := map[string]TestCase{ - "apply_missing_undefaulted_defaulted_key": { - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - containerPorts: - - protocol: TCP - `, - }, - }, - }, "apply_missing_defaulted_key_ambiguous_A": { Ops: []Operation{ Apply{ diff --git a/merge/duplicates_test.go b/merge/duplicates_test.go new file mode 100644 index 00000000..299da4ec --- /dev/null +++ b/merge/duplicates_test.go @@ -0,0 +1,808 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package merge_test + +import ( + "testing" + + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" +) + +var duplicatesParser = func() Parser { + parser, err := typed.NewParser(`types: +- name: type + map: + fields: + - name: list + type: + namedType: associativeList + - name: unrelated + type: + scalar: numeric + - name: set + type: + namedType: set +- name: associativeList + list: + elementType: + namedType: myElement + elementRelationship: associative + keys: + - name +- name: myElement + map: + fields: + - name: name + type: + scalar: string + - name: value1 + type: + scalar: numeric + - name: value2 + type: + scalar: numeric +- name: set + list: + elementType: + scalar: numeric + elementRelationship: associative +`) + if err != nil { + panic(err) + } + return SameVersionParser{T: parser.Type("type")} +}() + +func TestDuplicates(t *testing.T) { + tests := map[string]TestCase{ + "sets/ownership/duplicates": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + set: [1, 1, 3, 4] + `, + APIVersion: "v1", + }, + }, + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("set"), + _P("set", _V(1)), + _P("set", _V(3)), + _P("set", _V(4)), + ), + "v1", + false, + ), + }, + }, + "sets/ownership/add_duplicate": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + set: [1, 3, 4] + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater-two", + Object: ` + set: [1, 1, 3, 4] + `, + APIVersion: "v1", + }, + }, + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("set"), + _P("set", _V(3)), + _P("set", _V(4)), + ), + "v1", + false, + ), + "updater-two": fieldpath.NewVersionedSet( + _NS( + _P("set", _V(1)), + ), + "v1", + false, + ), + }, + }, + "sets/ownership/remove_duplicate": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + set: [1, 1, 3, 4] + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater-two", + Object: ` + set: [1, 3, 4] + `, + APIVersion: "v1", + }, + }, + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("set"), + _P("set", _V(3)), + _P("set", _V(4)), + ), + "v1", + false, + ), + "updater-two": fieldpath.NewVersionedSet( + _NS( + _P("set", _V(1)), + ), + "v1", + false, + ), + }, + }, + "sets/merging/remove_duplicate": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + set: [1, 1, 3, 4] + `, + APIVersion: "v1", + }, + Apply{ + Manager: "applier", + Object: ` + set: [1] + `, + APIVersion: "v1", + Conflicts: merge.Conflicts{ + {Manager: "updater", Path: _P("set", _V(1))}, + }, + }, + ForceApply{ + Manager: "applier", + Object: ` + set: [1] + `, + APIVersion: "v1", + }, + }, + Object: ` + set: [1, 3, 4] + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("set"), + _P("set", _V(3)), + _P("set", _V(4)), + ), + "v1", + false, + ), + "applier": fieldpath.NewVersionedSet( + _NS( + _P("set", _V(1)), + ), + "v1", + true, + ), + }, + }, + "sets/merging/ignore_duplicate": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + set: [1, 1, 3, 4] + `, + APIVersion: "v1", + }, + Apply{ + Manager: "applier", + Object: ` + set: [5] + `, + APIVersion: "v1", + }, + }, + Object: ` + set: [1, 1, 3, 4, 5] + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("set"), + _P("set", _V(1)), + _P("set", _V(3)), + _P("set", _V(4)), + ), + "v1", + false, + ), + "applier": fieldpath.NewVersionedSet( + _NS( + _P("set", _V(5)), + ), + "v1", + true, + ), + }, + }, + "list/ownership/duplicated_items": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + }, + // `name: a` is only owned once. + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + }, + }, + "list/ownership/change_duplicated_items": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater-two", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 3 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + }, + // `name: a` is only owned once, by actor who changed some of it. + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + "updater-two": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "a")), + ), + "v1", + false, + ), + }, + }, + "list/ownership/change_fields_duplicated_items": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater-two", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + value2: 3 # New field + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + }, + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + "updater-two": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "a")), + ), + "v1", + false, + ), + }, + }, + "list/ownership/add_duplicated_items_different_field": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + list: + - name: a + value1: 1 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater-two", + Object: ` + list: + - name: a + value1: 1 + - name: a + value2: 3 # New field + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + }, + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + "updater-two": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "a")), + ), + "v1", + false, + ), + }, + }, + "list/ownership/add_unrelated_to_list_with_duplicates": { + Ops: []Operation{ + Update{ + Manager: "updater-one", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater-two", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + }, + Managed: fieldpath.ManagedFields{ + "updater-one": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "a")), + ), + "v1", + false, + ), + "updater-two": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + }, + }, + "list/merge/unrelated_with_duplicated_items": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + ForceApply{ + Manager: "applier", + Object: ` + unrelated: 5 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + unrelated: 5 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + "applier": fieldpath.NewVersionedSet( + _NS( + _P("unrelated"), + ), + "v1", + true, + ), + }, + }, + // TODO: Owning the key is a little messed-up. + "list/merge/change_duplicated_item": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Apply{ + Manager: "applier", + Object: ` + list: + - name: a + value1: 3 + `, + APIVersion: "v1", + Conflicts: merge.Conflicts{ + {Manager: "updater", Path: _P("list", _KBF("name", "a"))}, + }, + }, + ForceApply{ + Manager: "applier", + Object: ` + list: + - name: a + value1: 3 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value1: 3 + - name: b + value1: 3 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + "applier": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "a"), "name"), + _P("list", _KBF("name", "a"), "value1"), + ), + "v1", + true, + ), + }, + }, + + "list/merge/unchanged_duplicated_item": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Apply{ + Manager: "applier", + Object: ` + list: + - name: a + value1: 2 + `, + APIVersion: "v1", + Conflicts: merge.Conflicts{ + {Manager: "updater", Path: _P("list", _KBF("name", "a"))}, + }, + }, + ForceApply{ + Manager: "applier", + Object: ` + list: + - name: a + value1: 3 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value1: 3 + - name: b + value1: 3 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + false, + ), + "applier": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "a"), "name"), + _P("list", _KBF("name", "a"), "value1"), + ), + "v1", + true, + ), + }, + }, + "list/merge/change_non_duplicated_item": { + Ops: []Operation{ + Update{ + Manager: "updater", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + ForceApply{ + Manager: "applier", + Object: ` + list: + - name: b + value1: 4 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 4 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "updater": fieldpath.NewVersionedSet( + _NS( + _P("list"), + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + ), + "v1", + false, + ), + "applier": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + true, + ), + }, + }, + "list/merge/apply_update_duplicates_apply_without": { + Ops: []Operation{ + Apply{ + Manager: "applier", + Object: ` + list: + - name: a + value1: 1 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Update{ + Manager: "updater", + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + Apply{ + Manager: "applier", + Object: ` + list: + - name: b + value1: 3 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value1: 1 + - name: a + value1: 2 + - name: b + value1: 3 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "applier": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "b")), + _P("list", _KBF("name", "b"), "name"), + _P("list", _KBF("name", "b"), "value1"), + ), + "v1", + true, + ), + "updater": fieldpath.NewVersionedSet( + _NS( + _P("list", _KBF("name", "a")), + ), + "v1", + false, + ), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if err := test.Test(duplicatesParser); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/merge/extract_apply_test.go b/merge/extract_apply_test.go index caef7564..2ae1df09 100644 --- a/merge/extract_apply_test.go +++ b/merge/extract_apply_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var extractParser = func() Parser { diff --git a/merge/field_level_overrides_test.go b/merge/field_level_overrides_test.go index 2462eba2..d47d0ef7 100644 --- a/merge/field_level_overrides_test.go +++ b/merge/field_level_overrides_test.go @@ -3,10 +3,10 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) func TestFieldLevelOverrides(t *testing.T) { diff --git a/merge/ignore_test.go b/merge/ignore_test.go index 4fe75c19..debbee29 100644 --- a/merge/ignore_test.go +++ b/merge/ignore_test.go @@ -19,10 +19,135 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" ) +func TestIgnoreFilter(t *testing.T) { + tests := map[string]TestCase{ + "update_does_not_own_ignored": { + APIVersion: "v1", + Ops: []Operation{ + Update{ + Manager: "default", + APIVersion: "v1", + Object: ` + numeric: 1 + string: "some string" + `, + }, + }, + Object: ` + numeric: 1 + string: "some string" + `, + Managed: fieldpath.ManagedFields{ + "default": fieldpath.NewVersionedSet( + _NS( + _P("numeric"), + ), + "v1", + false, + ), + }, + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v1": fieldpath.NewExcludeSetFilter(_NS( + _P("string"), + )), + }, + }, + "update_does_not_own_deep_ignored": { + APIVersion: "v1", + Ops: []Operation{ + Update{ + Manager: "default", + APIVersion: "v1", + Object: `{"numeric": 1, "obj": {"string": "foo", "numeric": 2}}`, + }, + }, + Object: `{"numeric": 1, "obj": {"string": "foo", "numeric": 2}}`, + Managed: fieldpath.ManagedFields{ + "default": fieldpath.NewVersionedSet( + _NS( + _P("numeric"), + ), + "v1", + false, + ), + }, + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v1": fieldpath.NewExcludeSetFilter(_NS( + _P("obj"), + )), + }, + }, + "apply_does_not_own_ignored": { + APIVersion: "v1", + Ops: []Operation{ + Apply{ + Manager: "default", + APIVersion: "v1", + Object: ` + numeric: 1 + string: "some string" + `, + }, + }, + Object: ` + numeric: 1 + string: "some string" + `, + Managed: fieldpath.ManagedFields{ + "default": fieldpath.NewVersionedSet( + _NS( + _P("numeric"), + ), + "v1", + true, + ), + }, + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v1": fieldpath.NewExcludeSetFilter(_NS( + _P("string"), + )), + }, + }, + "apply_does_not_own_deep_ignored": { + APIVersion: "v1", + Ops: []Operation{ + Apply{ + Manager: "default", + APIVersion: "v1", + Object: `{"numeric": 1, "obj": {"string": "foo", "numeric": 2}}`, + }, + }, + Object: `{"numeric": 1, "obj": {"string": "foo", "numeric": 2}}`, + Managed: fieldpath.ManagedFields{ + "default": fieldpath.NewVersionedSet( + _NS( + _P("numeric"), + ), + "v1", + true, + ), + }, + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v1": fieldpath.NewExcludeSetFilter(_NS( + _P("obj"), + )), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if err := test.Test(DeducedParser); err != nil { + t.Fatal("Should fail:", err) + } + }) + } +} + func TestIgnoredFields(t *testing.T) { tests := map[string]TestCase{ "update_does_not_own_ignored": { @@ -148,7 +273,7 @@ func TestIgnoredFields(t *testing.T) { } } -func TestIgnoredFieldsUsesVersions(t *testing.T) { +func TestFilteredFieldsUsesVersions(t *testing.T) { tests := map[string]TestCase{ "does_use_ignored_fields_versions": { Ops: []Operation{ @@ -205,19 +330,19 @@ func TestIgnoredFieldsUsesVersions(t *testing.T) { false, ), }, - IgnoredFields: map[fieldpath.APIVersion]*fieldpath.Set{ - "v1": _NS( + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v1": fieldpath.NewExcludeSetFilter(_NS( _P("mapOfMapsRecursive", "c"), - ), - "v2": _NS( + )), + "v2": fieldpath.NewExcludeSetFilter(_NS( _P("mapOfMapsRecursive", "cc"), - ), - "v3": _NS( + )), + "v3": fieldpath.NewExcludeSetFilter(_NS( _P("mapOfMapsRecursive", "ccc"), - ), - "v4": _NS( + )), + "v4": fieldpath.NewExcludeSetFilter(_NS( _P("mapOfMapsRecursive", "cccc"), - ), + )), }, }, "update_does_not_steal_ignored": { @@ -273,10 +398,10 @@ func TestIgnoredFieldsUsesVersions(t *testing.T) { false, ), }, - IgnoredFields: map[fieldpath.APIVersion]*fieldpath.Set{ - "v2": _NS( + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v2": fieldpath.NewExcludeSetFilter(_NS( _P("mapOfMapsRecursive", "c"), - ), + )), }, }, "apply_does_not_steal_ignored": { @@ -332,10 +457,10 @@ func TestIgnoredFieldsUsesVersions(t *testing.T) { false, ), }, - IgnoredFields: map[fieldpath.APIVersion]*fieldpath.Set{ - "v2": _NS( + IgnoreFilter: map[fieldpath.APIVersion]fieldpath.Filter{ + "v2": fieldpath.NewExcludeSetFilter(_NS( _P("mapOfMapsRecursive", "c"), - ), + )), }, }, } diff --git a/merge/key_test.go b/merge/key_test.go index 71387dcd..61969a83 100644 --- a/merge/key_test.go +++ b/merge/key_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var associativeListParser = func() Parser { diff --git a/merge/leaf_test.go b/merge/leaf_test.go index da8299f6..5eccdaf1 100644 --- a/merge/leaf_test.go +++ b/merge/leaf_test.go @@ -19,10 +19,10 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var leafFieldsParser = func() Parser { diff --git a/merge/multiple_appliers_test.go b/merge/multiple_appliers_test.go index ffed1d03..48f54cdb 100644 --- a/merge/multiple_appliers_test.go +++ b/merge/multiple_appliers_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "gopkg.in/yaml.v2" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + yaml "go.yaml.in/yaml/v2" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) func TestMultipleAppliersSet(t *testing.T) { @@ -1061,14 +1061,14 @@ func TestMultipleAppliersNestedType(t *testing.T) { }, }, Object: ` - mapOfMapsRecursive: - a: - c: - d: - e: - f: - g: - `, + mapOfMapsRecursive: + a: {} + c: + d: + e: + f: + g: + `, APIVersion: "v1", Managed: fieldpath.ManagedFields{ "apply-one": fieldpath.NewVersionedSet( @@ -1187,13 +1187,13 @@ func TestMultipleAppliersDeducedType(t *testing.T) { }, }, Object: ` - a: - c: - d: - e: - f: - g: - `, + a: {} + c: + d: + e: + f: + g: + `, APIVersion: "v1", Managed: fieldpath.ManagedFields{ "apply-two": fieldpath.NewVersionedSet( @@ -1945,3 +1945,160 @@ func TestMultipleAppliersReliantFieldsConversions(t *testing.T) { }) } } + +var versionDoesntMatchTypeName = func() Parser { + parser, err := typed.NewParser(`types: +- name: TypeV1 + map: + fields: + - name: field_foo_rely_on_bar + type: + scalar: string + - name: common_field + type: + scalar: string +- name: TypeV2 + map: + fields: + - name: required_field_bar + type: + scalar: string + - name: common_field + type: + scalar: string +`) + if err != nil { + panic(err) + } + return parser +}() + +type versionDoesntMatchTypenameConverter struct{} + +var _ merge.Converter = versionDoesntMatchTypenameConverter{} + +func (r versionDoesntMatchTypenameConverter) Convert(v *typed.TypedValue, version fieldpath.APIVersion) (*typed.TypedValue, error) { + inVersion := fieldpath.APIVersion("") + switch fieldpath.APIVersion(*v.TypeRef().NamedType) { + case "TypeV1": + inVersion = "v1" + case "TypeV2": + inVersion = "v2" + default: + return nil, fmt.Errorf(`Invalid typename: %q, should be one of ["TypeV1", "TypeV2"]`, version) + } + if inVersion == version { + return v, nil + } + if inVersion == version { + return v, nil + } + y, err := yaml.Marshal(v.AsValue().Unstructured()) + if err != nil { + return nil, err + } + inStr := string(y) + var outStr string + var outType string + switch version { + case "v1": + if !strings.Contains(inStr, "required_field_bar") { + return v, fmt.Errorf("missing requried field bar") + } + outType = "TypeV1" + outStr = strings.Replace(inStr, "required_field_bar", "field_foo_rely_on_bar", -1) + case "v2": + outType = "TypeV2" + outStr = strings.Replace(inStr, "field_foo_rely_on_bar", "required_field_bar", -1) + default: + return nil, missingVersionError + } + return versionDoesntMatchTypeName.Type(string(outType)).FromYAML(typed.YAMLObject(outStr)) +} + +func (r versionDoesntMatchTypenameConverter) IsMissingVersionError(err error) bool { + return err == missingVersionError +} + +// This is the same test as TestMultipleAppliersReliantFieldsConversions +// but written without the internal test framework that assumes that +// typenames and versions match. The goal of this test is to make sure +// that no such assumptions are used in the tested code. +func TestVersionDoesntMatchTypename(t *testing.T) { + converter := versionDoesntMatchTypenameConverter{} + updater := &merge.Updater{Converter: converter} + + for i := 0; i < 10; i++ { + // Apply in one version, apply in another, apply in a third. + live, err := versionDoesntMatchTypeName.Type("TypeV2").FromYAML(`{}`) + if err != nil { + t.Fatalf("Failed to parse empty object: %v", err) + } + managers := fieldpath.ManagedFields{} + config, err := versionDoesntMatchTypeName.Type("TypeV2").FromYAML(`{"required_field_bar": "a"}`) + if err != nil { + t.Fatalf("Failed to parse object: %v", err) + } + live, managers, err = updater.Apply(live, config, "v2", managers, "v2_applier", false) + if err != nil { + t.Fatalf("Failed to apply: %v", err) + } + + live, err = converter.Convert(live, "v1") + if err != nil { + t.Fatalf("Failed to convert object to v1: %v", err) + } + config, err = versionDoesntMatchTypeName.Type("TypeV1").FromYAML(`{"common_field": "b"}`) + if err != nil { + t.Fatalf("Failed to parse object: %v", err) + } + + live, managers, err = updater.Apply(live, config, "v1", managers, "v1_applier", false) + if err != nil { + t.Fatalf("Failed to apply: %v", err) + } + + live, err = converter.Convert(live, "v2") + if err != nil { + t.Fatalf("Failed to convert object to v1: %v", err) + } + config, err = versionDoesntMatchTypeName.Type("TypeV2").FromYAML(`{"required_field_bar": "b"}`) + if err != nil { + t.Fatalf("Failed to parse object: %v", err) + } + live, managers, err = updater.Apply(live, config, "v2", managers, "v2_applier", false) + if err != nil { + t.Fatalf("Failed to apply: %v", err) + } + + expectedObject, err := versionDoesntMatchTypeName.Type("TypeV2").FromYAML(`{"required_field_bar": "b", "common_field": "b"}`) + if err != nil { + t.Fatalf("Failed to parse object: %v", err) + } + if comparison, err := live.Compare(expectedObject); err != nil { + t.Fatalf("Failed to compare live with expected: %v", err) + } else if !comparison.IsSame() { + t.Fatalf("Live is different from expected:\n%v", comparison) + } + + expectedManagers := fieldpath.ManagedFields{ + "v2_applier": fieldpath.NewVersionedSet( + _NS( + _P("required_field_bar"), + ), + "v2", + true, + ), + "v1_applier": fieldpath.NewVersionedSet( + _NS( + _P("common_field"), + ), + "v1", + true, + ), + } + if !expectedManagers.Equals(managers) { + t.Fatalf("ManagedFields not as expected:\n%v", managers) + } + } +} diff --git a/merge/nested_test.go b/merge/nested_test.go index e9b18312..2da4d4c9 100644 --- a/merge/nested_test.go +++ b/merge/nested_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var nestedTypeParser = func() Parser { @@ -55,7 +55,7 @@ var nestedTypeParser = func() Parser { scalar: string - name: value type: - scalar: number + scalar: numeric - name: listOfLists list: elementType: @@ -554,8 +554,8 @@ func TestUpdateNestedType(t *testing.T) { }, }, Object: ` - struct: - `, + struct: {} + `, APIVersion: "v1", Managed: fieldpath.ManagedFields{ "default": fieldpath.NewVersionedSet( diff --git a/merge/obsolete_versions_test.go b/merge/obsolete_versions_test.go index b7936325..226a8708 100644 --- a/merge/obsolete_versions_test.go +++ b/merge/obsolete_versions_test.go @@ -20,10 +20,10 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) // specificVersionConverter doesn't convert and return the exact same @@ -127,7 +127,7 @@ func TestApplyObsoleteVersion(t *testing.T) { if err != nil { t.Fatalf("Failed to compare live object: %v", err) } - if !comparison.IsSame() { + if comparison != "" { t.Fatalf("Unexpected object:\n%v", comparison) } } diff --git a/merge/preserve_unknown_test.go b/merge/preserve_unknown_test.go index 121f2a77..f7db382d 100644 --- a/merge/preserve_unknown_test.go +++ b/merge/preserve_unknown_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var preserveUnknownParser = func() Parser { diff --git a/merge/real_test.go b/merge/real_test.go index 2a5a9f01..c8254fce 100644 --- a/merge/real_test.go +++ b/merge/real_test.go @@ -21,8 +21,8 @@ import ( "path/filepath" "testing" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) func testdata(file string) string { @@ -108,8 +108,9 @@ func BenchmarkOperations(b *testing.B) { b.Run(bench.name, func(b *testing.B) { obj := typed.YAMLObject(read(testdata(bench.filename))) tests := []struct { - name string - ops []Operation + name string + returnInputonNoop bool + ops []Operation }{ { name: "Create", @@ -146,6 +147,22 @@ func BenchmarkOperations(b *testing.B) { }, }, }, + { + name: "ApplyTwiceNoCompare", + returnInputonNoop: true, + ops: []Operation{ + Apply{ + Manager: "controller", + APIVersion: "v1", + Object: obj, + }, + Apply{ + Manager: "other-controller", + APIVersion: "v1", + Object: obj, + }, + }, + }, { name: "Update", ops: []Operation{ @@ -180,7 +197,8 @@ func BenchmarkOperations(b *testing.B) { for _, test := range tests { b.Run(test.name, func(b *testing.B) { tc := TestCase{ - Ops: test.ops, + Ops: test.ops, + ReturnInputOnNoop: test.returnInputonNoop, } p := SameVersionParser{T: bench.parseType} tc.PreprocessOperations(p) diff --git a/merge/schema_change_test.go b/merge/schema_change_test.go index be24bb2d..db78ff56 100644 --- a/merge/schema_change_test.go +++ b/merge/schema_change_test.go @@ -19,10 +19,10 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/merge" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var structParser = func() *typed.Parser { @@ -252,3 +252,459 @@ func TestAtomicToGranularSchemaChanges(t *testing.T) { }) } } + +var associativeListParserOld = func() *typed.Parser { + oldParser, err := typed.NewParser(`types: +- name: v1 + map: + fields: + - name: list + type: + namedType: associativeList +- name: associativeList + list: + elementType: + namedType: myElement + elementRelationship: associative + keys: + - name +- name: myElement + map: + fields: + - name: name + type: + scalar: string + - name: value + type: + scalar: numeric +`) + if err != nil { + panic(err) + } + return oldParser +}() + +var associativeListParserNewOptionalKey = func() *typed.Parser { + newParser, err := typed.NewParser(`types: +- name: v1 + map: + fields: + - name: list + type: + namedType: associativeList +- name: associativeList + list: + elementType: + namedType: myElement + elementRelationship: associative + keys: + - name + - id +- name: myElement + map: + fields: + - name: name + type: + scalar: string + - name: id + type: + scalar: numeric + - name: value + type: + scalar: numeric +`) + if err != nil { + panic(err) + } + return newParser +}() + +var associativeListParserNewKeyWithDefault = func() *typed.Parser { + newParser, err := typed.NewParser(`types: +- name: v1 + map: + fields: + - name: list + type: + namedType: associativeList +- name: associativeList + list: + elementType: + namedType: myElement + elementRelationship: associative + keys: + - name + - id +- name: myElement + map: + fields: + - name: name + type: + scalar: string + - name: id + type: + scalar: numeric + default: 1 + - name: value + type: + scalar: numeric +`) + if err != nil { + panic(err) + } + return newParser +}() + +func TestAssociativeListSchemaChanges(t *testing.T) { + tests := map[string]TestCase{ + "new required key with default": { + Ops: []Operation{ + Apply{ + Manager: "one", + Object: ` + list: + - name: a + value: 1 + - name: b + value: 1 + - name: c + value: 1 + `, + APIVersion: "v1", + }, + ChangeParser{Parser: associativeListParserNewKeyWithDefault}, + Apply{ + Manager: "one", + Object: ` + list: + - name: a + value: 2 + - name: b + id: 1 + value: 2 + - name: c + value: 1 + - name: c + id: 2 + value: 2 + - name: c + id: 3 + value: 3 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value: 2 + - name: b + id: 1 + value: 2 + - name: c + value: 1 + - name: c + id: 2 + value: 2 + - name: c + id: 3 + value: 3 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "one": fieldpath.NewVersionedSet(_NS( + _P("list", _KBF("name", "a", "id", float64(1))), + _P("list", _KBF("name", "a", "id", float64(1)), "name"), + _P("list", _KBF("name", "a", "id", float64(1)), "value"), + _P("list", _KBF("name", "b", "id", float64(1))), + _P("list", _KBF("name", "b", "id", float64(1)), "name"), + _P("list", _KBF("name", "b", "id", float64(1)), "id"), + _P("list", _KBF("name", "b", "id", float64(1)), "value"), + _P("list", _KBF("name", "c", "id", float64(1))), + _P("list", _KBF("name", "c", "id", float64(1)), "name"), + _P("list", _KBF("name", "c", "id", float64(1)), "value"), + _P("list", _KBF("name", "c", "id", float64(2))), + _P("list", _KBF("name", "c", "id", float64(2)), "name"), + _P("list", _KBF("name", "c", "id", float64(2)), "id"), + _P("list", _KBF("name", "c", "id", float64(2)), "value"), + _P("list", _KBF("name", "c", "id", float64(3))), + _P("list", _KBF("name", "c", "id", float64(3)), "name"), + _P("list", _KBF("name", "c", "id", float64(3)), "id"), + _P("list", _KBF("name", "c", "id", float64(3)), "value"), + ), "v1", true), + }, + }, + "new optional key": { + Ops: []Operation{ + Apply{ + Manager: "one", + Object: ` + list: + - name: a + value: 1 + `, + APIVersion: "v1", + }, + ChangeParser{Parser: associativeListParserNewOptionalKey}, + Apply{ + Manager: "one", + Object: ` + list: + - name: a + value: 2 + - name: a + id: 1 + value: 1 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value: 2 + - name: a + id: 1 + value: 1 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "one": fieldpath.NewVersionedSet(_NS( + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "a"), "name"), + _P("list", _KBF("name", "a"), "value"), + _P("list", _KBF("name", "a", "id", float64(1))), + _P("list", _KBF("name", "a", "id", float64(1)), "name"), + _P("list", _KBF("name", "a", "id", float64(1)), "id"), + _P("list", _KBF("name", "a", "id", float64(1)), "value"), + ), "v1", true), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if err := test.Test(associativeListParserOld); err != nil { + t.Fatal(err) + } + }) + } +} + +var associativeListParserPromoteKeyBefore = func() *typed.Parser { + p, err := typed.NewParser(`types: +- name: v1 + map: + fields: + - name: list + type: + namedType: associativeList +- name: associativeList + list: + elementType: + namedType: myElement + elementRelationship: associative + keys: + - name +- name: myElement + map: + fields: + - name: name + type: + scalar: string + - name: id + type: + scalar: numeric + - name: value + type: + scalar: numeric +`) + if err != nil { + panic(err) + } + return p +}() + +var associativeListParserPromoteKeyAfter = func() *typed.Parser { + p, err := typed.NewParser(`types: +- name: v1 + map: + fields: + - name: list + type: + namedType: associativeList +- name: associativeList + list: + elementType: + namedType: myElement + elementRelationship: associative + keys: + - name + - id +- name: myElement + map: + fields: + - name: name + type: + scalar: string + - name: id + type: + scalar: numeric + - name: value + type: + scalar: numeric +`) + if err != nil { + panic(err) + } + return p +}() + +func TestPromoteFieldToAssociativeListKey(t *testing.T) { + tests := map[string]TestCase{ + "identical item merges": { + Ops: []Operation{ + Apply{ + Manager: "one", + Object: ` + list: + - name: a + id: 1 + value: 1 + `, + APIVersion: "v1", + }, + ChangeParser{Parser: associativeListParserPromoteKeyAfter}, + Apply{ + Manager: "one", + Object: ` + list: + - name: a + id: 1 + value: 2 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + id: 1 + value: 2 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "one": fieldpath.NewVersionedSet(_NS( + _P("list", _KBF("name", "a", "id", float64(1))), + _P("list", _KBF("name", "a", "id", float64(1)), "name"), + _P("list", _KBF("name", "a", "id", float64(1)), "id"), + _P("list", _KBF("name", "a", "id", float64(1)), "value"), + ), "v1", true), + }, + }, + "distinct item added": { + Ops: []Operation{ + Apply{ + Manager: "one", + Object: ` + list: + - name: a + id: 1 + value: 1 + `, + APIVersion: "v1", + }, + ChangeParser{Parser: associativeListParserPromoteKeyAfter}, + Apply{ + Manager: "one", + Object: ` + list: + - name: a + id: 1 + value: 1 + - name: a + id: 2 + value: 2 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + id: 1 + value: 1 + - name: a + id: 2 + value: 2 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "one": fieldpath.NewVersionedSet(_NS( + _P("list", _KBF("name", "a", "id", float64(1))), + _P("list", _KBF("name", "a", "id", float64(1)), "name"), + _P("list", _KBF("name", "a", "id", float64(1)), "id"), + _P("list", _KBF("name", "a", "id", float64(1)), "value"), + _P("list", _KBF("name", "a", "id", float64(2))), + _P("list", _KBF("name", "a", "id", float64(2)), "name"), + _P("list", _KBF("name", "a", "id", float64(2)), "id"), + _P("list", _KBF("name", "a", "id", float64(2)), "value"), + ), "v1", true), + }, + }, + "item missing new key field is distinct": { + Ops: []Operation{ + Apply{ + Manager: "one", + Object: ` + list: + - name: a + value: 1 + `, + APIVersion: "v1", + }, + ChangeParser{Parser: associativeListParserPromoteKeyAfter}, + Apply{ + Manager: "one", + Object: ` + list: + - name: a + value: 1 + - name: a + id: 2 + value: 2 + `, + APIVersion: "v1", + }, + }, + Object: ` + list: + - name: a + value: 1 + - name: a + id: 2 + value: 2 + `, + APIVersion: "v1", + Managed: fieldpath.ManagedFields{ + "one": fieldpath.NewVersionedSet(_NS( + _P("list", _KBF("name", "a")), + _P("list", _KBF("name", "a"), "name"), + _P("list", _KBF("name", "a"), "value"), + _P("list", _KBF("name", "a", "id", float64(2))), + _P("list", _KBF("name", "a", "id", float64(2)), "name"), + _P("list", _KBF("name", "a", "id", float64(2)), "id"), + _P("list", _KBF("name", "a", "id", float64(2)), "value"), + ), "v1", true), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if err := test.Test(associativeListParserPromoteKeyBefore); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/merge/set_test.go b/merge/set_test.go index b1bfac29..73d7170d 100644 --- a/merge/set_test.go +++ b/merge/set_test.go @@ -19,9 +19,9 @@ package merge_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + . "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) var setFieldsParser = func() Parser { @@ -124,11 +124,11 @@ func TestUpdateSet(t *testing.T) { Object: ` list: - a - - aprime - b + - aprime - c - - cprime - d + - cprime `, APIVersion: "v1", Managed: fieldpath.ManagedFields{ @@ -189,11 +189,11 @@ func TestUpdateSet(t *testing.T) { Object: ` list: - a - - aprime - b + - aprime - c - - cprime - d + - cprime `, APIVersion: "v1", Managed: fieldpath.ManagedFields{ diff --git a/merge/union_test.go b/merge/union_test.go deleted file mode 100644 index d6989568..00000000 --- a/merge/union_test.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package merge_test - -import ( - "testing" - - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - . "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/merge" - "sigs.k8s.io/structured-merge-diff/v5/typed" -) - -var unionFieldsParser = func() Parser { - parser, err := typed.NewParser(`types: -- name: unionFields - map: - fields: - - name: numeric - type: - scalar: numeric - - name: string - type: - scalar: string - - name: type - type: - scalar: string - - name: fieldA - type: - scalar: string - - name: fieldB - type: - scalar: string - unions: - - discriminator: type - deduceInvalidDiscriminator: true - fields: - - fieldName: numeric - discriminatorValue: Numeric - - fieldName: string - discriminatorValue: String - - fields: - - fieldName: fieldA - discriminatorValue: FieldA - - fieldName: fieldB - discriminatorValue: FieldB`) - if err != nil { - panic(err) - } - return SameVersionParser{T: parser.Type("unionFields")} -}() - -func TestUnion(t *testing.T) { - tests := map[string]TestCase{ - "union_apply_owns_discriminator": { - RequiresUnions: true, - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - numeric: 1 - `, - }, - }, - Object: ` - numeric: 1 - type: Numeric - `, - APIVersion: "v1", - Managed: fieldpath.ManagedFields{ - "default": fieldpath.NewVersionedSet( - _NS( - _P("numeric"), _P("type"), - ), - "v1", - false, - ), - }, - }, - "union_apply_without_discriminator_conflict": { - RequiresUnions: true, - Ops: []Operation{ - Update{ - Manager: "controller", - APIVersion: "v1", - Object: ` - string: "some string" - `, - }, - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - numeric: 1 - `, - Conflicts: merge.Conflicts{ - merge.Conflict{Manager: "controller", Path: _P("type")}, - }, - }, - }, - Object: ` - string: "some string" - type: String - `, - APIVersion: "v1", - Managed: fieldpath.ManagedFields{ - "controller": fieldpath.NewVersionedSet( - _NS( - _P("string"), _P("type"), - ), - "v1", - false, - ), - }, - }, - "union_apply_with_null_value": { - RequiresUnions: true, - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - type: Numeric - string: null - numeric: 1 - `, - }, - }, - }, - "union_apply_multiple_unions": { - RequiresUnions: true, - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - string: "some string" - fieldA: "fieldA string" - `, - }, - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - numeric: 0 - fieldB: "fieldB string" - `, - }, - }, - Object: ` - type: Numeric - numeric: 0 - fieldB: "fieldB string" - `, - APIVersion: "v1", - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - if err := test.Test(unionFieldsParser); err != nil { - t.Fatal(err) - } - }) - } -} - -func TestUnionErrors(t *testing.T) { - tests := map[string]TestCase{ - "union_apply_two": { - RequiresUnions: true, - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - numeric: 1 - string: "some string" - `, - }, - }, - }, - "union_apply_two_and_discriminator": { - RequiresUnions: true, - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - type: Numeric - string: "some string" - numeric: 1 - `, - }, - }, - }, - "union_apply_wrong_discriminator": { - RequiresUnions: true, - Ops: []Operation{ - Apply{ - Manager: "default", - APIVersion: "v1", - Object: ` - type: Numeric - string: "some string" - `, - }, - }, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - if test.Test(unionFieldsParser) == nil { - t.Fatal("Should fail") - } - }) - } -} diff --git a/merge/update.go b/merge/update.go index 190d6997..99d722f8 100644 --- a/merge/update.go +++ b/merge/update.go @@ -16,8 +16,9 @@ package merge import ( "fmt" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // Converter is an interface to the conversion logic. The converter @@ -27,19 +28,46 @@ type Converter interface { IsMissingVersionError(error) bool } +// UpdateBuilder allows you to create a new Updater by exposing all of +// the options and setting them once. +type UpdaterBuilder struct { + Converter Converter + IgnoreFilter map[fieldpath.APIVersion]fieldpath.Filter + + // IgnoredFields provides a set of fields to ignore for each + IgnoredFields map[fieldpath.APIVersion]*fieldpath.Set + + // Stop comparing the new object with old object after applying. + // This was initially used to avoid spurious etcd update, but + // since that's vastly inefficient, we've come-up with a better + // way of doing that. Create this flag to stop it. + // Comparing has become more expensive too now that we're not using + // `Compare` but `value.Equals` so this gives an option to avoid it. + ReturnInputOnNoop bool +} + +func (u *UpdaterBuilder) BuildUpdater() *Updater { + return &Updater{ + Converter: u.Converter, + IgnoreFilter: u.IgnoreFilter, + IgnoredFields: u.IgnoredFields, + returnInputOnNoop: u.ReturnInputOnNoop, + } +} + // Updater is the object used to compute updated FieldSets and also // merge the object on Apply. type Updater struct { - Converter Converter + // Deprecated: This will eventually become private. + Converter Converter + + // Deprecated: This will eventually become private. IgnoredFields map[fieldpath.APIVersion]*fieldpath.Set - enableUnions bool -} + // Deprecated: This will eventually become private. + IgnoreFilter map[fieldpath.APIVersion]fieldpath.Filter -// EnableUnionFeature turns on union handling. It is disabled by default until the -// feature is complete. -func (s *Updater) EnableUnionFeature() { - s.enableUnions = true + returnInputOnNoop bool } func (s *Updater) update(oldObject, newObject *typed.TypedValue, version fieldpath.APIVersion, managers fieldpath.ManagedFields, workflow string, force bool) (fieldpath.ManagedFields, *typed.Comparison, error) { @@ -50,8 +78,19 @@ func (s *Updater) update(oldObject, newObject *typed.TypedValue, version fieldpa return nil, nil, fmt.Errorf("failed to compare objects: %v", err) } - versions := map[fieldpath.APIVersion]*typed.Comparison{ - version: compare.ExcludeFields(s.IgnoredFields[version]), + var versions map[fieldpath.APIVersion]*typed.Comparison + + if s.IgnoredFields != nil && s.IgnoreFilter != nil { + return nil, nil, fmt.Errorf("IgnoreFilter and IgnoreFilter may not both be set") + } + if s.IgnoredFields != nil { + versions = map[fieldpath.APIVersion]*typed.Comparison{ + version: compare.ExcludeFields(s.IgnoredFields[version]), + } + } else { + versions = map[fieldpath.APIVersion]*typed.Comparison{ + version: compare.FilterFields(s.IgnoreFilter[version]), + } } for manager, managerSet := range managers { @@ -81,7 +120,12 @@ func (s *Updater) update(oldObject, newObject *typed.TypedValue, version fieldpa if err != nil { return nil, nil, fmt.Errorf("failed to compare objects: %v", err) } - versions[managerSet.APIVersion()] = compare.ExcludeFields(s.IgnoredFields[managerSet.APIVersion()]) + + if s.IgnoredFields != nil { + versions[managerSet.APIVersion()] = compare.ExcludeFields(s.IgnoredFields[managerSet.APIVersion()]) + } else { + versions[managerSet.APIVersion()] = compare.FilterFields(s.IgnoreFilter[managerSet.APIVersion()]) + } } conflictSet := managerSet.Set().Intersection(compare.Modified.Union(compare.Added)) @@ -126,12 +170,6 @@ func (s *Updater) Update(liveObject, newObject *typed.TypedValue, version fieldp if err != nil { return nil, fieldpath.ManagedFields{}, err } - if s.enableUnions { - newObject, err = liveObject.NormalizeUnions(newObject) - if err != nil { - return nil, fieldpath.ManagedFields{}, err - } - } managers, compare, err := s.update(liveObject, newObject, version, managers, manager, true) if err != nil { return nil, fieldpath.ManagedFields{}, err @@ -139,13 +177,23 @@ func (s *Updater) Update(liveObject, newObject *typed.TypedValue, version fieldp if _, ok := managers[manager]; !ok { managers[manager] = fieldpath.NewVersionedSet(fieldpath.NewSet(), version, false) } + set := managers[manager].Set().Difference(compare.Removed).Union(compare.Modified).Union(compare.Added) - ignored := s.IgnoredFields[version] - if ignored == nil { - ignored = fieldpath.NewSet() + if s.IgnoredFields != nil && s.IgnoreFilter != nil { + return nil, nil, fmt.Errorf("IgnoreFilter and IgnoreFilter may not both be set") + } + var ignoreFilter fieldpath.Filter + if s.IgnoredFields != nil { + ignoreFilter = fieldpath.NewExcludeSetFilter(s.IgnoredFields[version]) + } else { + ignoreFilter = s.IgnoreFilter[version] } + if ignoreFilter != nil { + set = ignoreFilter.Filter(set) + } + managers[manager] = fieldpath.NewVersionedSet( - managers[manager].Set().Union(compare.Modified).Union(compare.Added).Difference(compare.Removed).RecursiveDifference(ignored), + set, version, false, ) @@ -157,54 +205,45 @@ func (s *Updater) Update(liveObject, newObject *typed.TypedValue, version fieldp // Apply should be called when Apply is run, given the current object as // well as the configuration that is applied. This will merge the object -// and return it. If the object hasn't changed, nil is returned (the -// managers can still have changed though). +// and return it. func (s *Updater) Apply(liveObject, configObject *typed.TypedValue, version fieldpath.APIVersion, managers fieldpath.ManagedFields, manager string, force bool) (*typed.TypedValue, fieldpath.ManagedFields, error) { var err error managers, err = s.reconcileManagedFieldsWithSchemaChanges(liveObject, managers) if err != nil { return nil, fieldpath.ManagedFields{}, err } - if s.enableUnions { - configObject, err = configObject.NormalizeUnionsApply(configObject) - if err != nil { - return nil, fieldpath.ManagedFields{}, err - } - } newObject, err := liveObject.Merge(configObject) if err != nil { return nil, fieldpath.ManagedFields{}, fmt.Errorf("failed to merge config: %v", err) } - if s.enableUnions { - newObject, err = configObject.NormalizeUnionsApply(newObject) - if err != nil { - return nil, fieldpath.ManagedFields{}, err - } - } lastSet := managers[manager] set, err := configObject.ToFieldSet() if err != nil { return nil, fieldpath.ManagedFields{}, fmt.Errorf("failed to get field set: %v", err) } - ignored := s.IgnoredFields[version] - if ignored != nil { - set = set.RecursiveDifference(ignored) - // TODO: is this correct. If we don't remove from lastSet pruning might remove the fields? - if lastSet != nil { - lastSet.Set().RecursiveDifference(ignored) - } + if s.IgnoredFields != nil && s.IgnoreFilter != nil { + return nil, nil, fmt.Errorf("IgnoreFilter and IgnoreFilter may not both be set") + } + var ignoreFilter fieldpath.Filter + if s.IgnoredFields != nil { + ignoreFilter = fieldpath.NewExcludeSetFilter(s.IgnoredFields[version]) + } else { + ignoreFilter = s.IgnoreFilter[version] + } + if ignoreFilter != nil { + set = ignoreFilter.Filter(set) } managers[manager] = fieldpath.NewVersionedSet(set, version, true) newObject, err = s.prune(newObject, managers, manager, lastSet) if err != nil { return nil, fieldpath.ManagedFields{}, fmt.Errorf("failed to prune fields: %v", err) } - managers, compare, err := s.update(liveObject, newObject, version, managers, manager, force) + managers, _, err = s.update(liveObject, newObject, version, managers, manager, force) if err != nil { return nil, fieldpath.ManagedFields{}, err } - if compare.IsSame() { + if !s.returnInputOnNoop && value.EqualsUsing(value.NewFreelistAllocator(), liveObject.AsValue(), newObject.AsValue()) { newObject = nil } return newObject, managers, nil @@ -218,7 +257,8 @@ func (s *Updater) prune(merged *typed.TypedValue, managers fieldpath.ManagedFiel if lastSet == nil || lastSet.Set().Empty() { return merged, nil } - convertedMerged, err := s.Converter.Convert(merged, lastSet.APIVersion()) + version := lastSet.APIVersion() + convertedMerged, err := s.Converter.Convert(merged, version) if err != nil { if s.Converter.IsMissingVersionError(err) { return merged, nil @@ -228,7 +268,7 @@ func (s *Updater) prune(merged *typed.TypedValue, managers fieldpath.ManagedFiel sc, tr := convertedMerged.Schema(), convertedMerged.TypeRef() pruned := convertedMerged.RemoveItems(lastSet.Set().EnsureNamedFieldsAreMembers(sc, tr)) - pruned, err = s.addBackOwnedItems(convertedMerged, pruned, managers, applyingManager) + pruned, err = s.addBackOwnedItems(convertedMerged, pruned, version, managers, applyingManager) if err != nil { return nil, fmt.Errorf("failed add back owned items: %v", err) } @@ -241,7 +281,7 @@ func (s *Updater) prune(merged *typed.TypedValue, managers fieldpath.ManagedFiel // addBackOwnedItems adds back any fields, list and map items that were removed by prune, // but other appliers or updaters (or the current applier's new config) claim to own. -func (s *Updater) addBackOwnedItems(merged, pruned *typed.TypedValue, managedFields fieldpath.ManagedFields, applyingManager string) (*typed.TypedValue, error) { +func (s *Updater) addBackOwnedItems(merged, pruned *typed.TypedValue, prunedVersion fieldpath.APIVersion, managedFields fieldpath.ManagedFields, applyingManager string) (*typed.TypedValue, error) { var err error managedAtVersion := map[fieldpath.APIVersion]*fieldpath.Set{} for _, managerSet := range managedFields { @@ -252,7 +292,6 @@ func (s *Updater) addBackOwnedItems(merged, pruned *typed.TypedValue, managedFie } // Add back owned items at pruned version first to avoid conversion failure // caused by pruned fields which are required for conversion. - prunedVersion := fieldpath.APIVersion(*pruned.TypeRef().NamedType) if managed, ok := managedAtVersion[prunedVersion]; ok { merged, pruned, err = s.addBackOwnedItemsForVersion(merged, pruned, prunedVersion, managed) if err != nil { diff --git a/schema/elements.go b/schema/elements.go index 34b504df..c8138a65 100644 --- a/schema/elements.go +++ b/schema/elements.go @@ -18,6 +18,7 @@ package schema import ( "sync" + "sync/atomic" ) // Schema is a list of named types. @@ -28,7 +29,7 @@ type Schema struct { Types []TypeDef `yaml:"types,omitempty"` once sync.Once - m map[string]TypeDef + m atomic.Pointer[map[string]TypeDef] lock sync.Mutex // Cached results of resolving type references to atoms. Only stores @@ -39,7 +40,7 @@ type Schema struct { // A TypeSpecifier references a particular type in a schema. type TypeSpecifier struct { Type TypeRef `yaml:"type,omitempty"` - Schema *Schema `yaml:"schema,omitempty"` + Schema Schema `yaml:"schema,omitempty"` } // TypeDef represents a named type in a schema. @@ -73,7 +74,7 @@ type Atom struct { } // Scalar (AKA "primitive") represents a type which has a single value which is -// either numeric, string, or boolean. +// either numeric, string, or boolean, or untyped for any of them. // // TODO: split numeric into float/int? Something even more fine-grained? type Scalar string @@ -82,6 +83,7 @@ const ( Numeric = Scalar("numeric") String = Scalar("string") Boolean = Scalar("boolean") + Untyped = Scalar("untyped") ) // ElementRelationship is an enum of the different possible relationships @@ -143,22 +145,50 @@ type Map struct { ElementRelationship ElementRelationship `yaml:"elementRelationship,omitempty"` once sync.Once - m map[string]StructField + m atomic.Pointer[map[string]StructField] } // FindField is a convenience function that returns the referenced StructField, // if it exists, or (nil, false) if it doesn't. func (m *Map) FindField(name string) (StructField, bool) { m.once.Do(func() { - m.m = make(map[string]StructField, len(m.Fields)) + mm := make(map[string]StructField, len(m.Fields)) for _, field := range m.Fields { - m.m[field.Name] = field + mm[field.Name] = field } + m.m.Store(&mm) }) - sf, ok := m.m[name] + sf, ok := (*m.m.Load())[name] return sf, ok } +// CopyInto clones this instance of Map into dst +// +// If dst is nil this method does nothing. +// If dst is already initialized, overwrites it with this instance. +// Warning: Not thread safe. Only use dst after this function returns. +func (m *Map) CopyInto(dst *Map) { + if dst == nil { + return + } + + // Map type is considered immutable so sharing references + dst.Fields = m.Fields + dst.ElementType = m.ElementType + dst.Unions = m.Unions + dst.ElementRelationship = m.ElementRelationship + + mm := m.m.Load() + if mm != nil { + // If cache is non-nil then the once token had been consumed. + // Must reset token and use it again to ensure same semantics. + dst.once = sync.Once{} + dst.once.Do(func() { + dst.m.Store(mm) + }) + } +} + // UnionFields are mapping between the fields that are part of the union and // their discriminated value. The discriminated value has to be set, and // should not conflict with other discriminated value in the list. @@ -248,12 +278,13 @@ type List struct { // if it exists, or (nil, false) if it doesn't. func (s *Schema) FindNamedType(name string) (TypeDef, bool) { s.once.Do(func() { - s.m = make(map[string]TypeDef, len(s.Types)) + sm := make(map[string]TypeDef, len(s.Types)) for _, t := range s.Types { - s.m[t.Name] = t + sm[t.Name] = t } + s.m.Store(&sm) }) - t, ok := s.m[name] + t, ok := (*s.m.Load())[name] return t, ok } @@ -285,10 +316,12 @@ func (s *Schema) Resolve(tr TypeRef) (Atom, bool) { return s.resolveNoOverrides(tr) } - // Check to see if we have a cached version of this type s.lock.Lock() defer s.lock.Unlock() - s.resolvedTypes = make(map[TypeRef]Atom) + + if s.resolvedTypes == nil { + s.resolvedTypes = make(map[TypeRef]Atom) + } var result Atom var exists bool @@ -300,7 +333,8 @@ func (s *Schema) Resolve(tr TypeRef) (Atom, bool) { // Allow field-level electives to override the referred type's modifiers switch { case result.Map != nil: - mapCopy := *result.Map + mapCopy := Map{} + result.Map.CopyInto(&mapCopy) mapCopy.ElementRelationship = *tr.ElementRelationship result.Map = &mapCopy case result.List != nil: @@ -323,10 +357,11 @@ func (s *Schema) Resolve(tr TypeRef) (Atom, bool) { return result, true } -// Clones this instance of Schema into the other -// If other is nil this method does nothing. -// If other is already initialized, overwrites it with this instance -// Warning: Not thread safe +// CopyInto clones this instance of Schema into dst +// +// If dst is nil this method does nothing. +// If dst is already initialized, overwrites it with this instance. +// Warning: Not thread safe. Only use dst after this function returns. func (s *Schema) CopyInto(dst *Schema) { if dst == nil { return @@ -335,12 +370,13 @@ func (s *Schema) CopyInto(dst *Schema) { // Schema type is considered immutable so sharing references dst.Types = s.Types - if s.m != nil { + sm := s.m.Load() + if sm != nil { // If cache is non-nil then the once token had been consumed. // Must reset token and use it again to ensure same semantics. dst.once = sync.Once{} dst.once.Do(func() { - dst.m = s.m + dst.m.Store(sm) }) } } diff --git a/schema/elements_test.go b/schema/elements_test.go index be802fbf..bb4b27fa 100644 --- a/schema/elements_test.go +++ b/schema/elements_test.go @@ -87,6 +87,48 @@ func TestFindField(t *testing.T) { } } +func testMap() *Map { + return &Map{ + Fields: []StructField{ + { + Name: "a", + Type: TypeRef{NamedType: strptr("aaa")}, + }, { + Name: "b", + Type: TypeRef{NamedType: strptr("bbb")}, + }, { + Name: "c", + Type: TypeRef{NamedType: strptr("ccc")}, + }, + }, + } +} + +func BenchmarkFindFieldCached(b *testing.B) { + m := testMap() + m.FindField("a") + for i := 0; i < b.N; i++ { + m.FindField("a") + } +} + +func BenchmarkFindFieldNew(b *testing.B) { + for i := 0; i < b.N; i++ { + m := testMap() + m.FindField("a") + } +} + +// As a baseline of BenchmarkFindFieldNew +func BenchmarkMakeMap(b *testing.B) { + var m *Map + for i := 0; i < b.N; i++ { + m = testMap() + } + b.StopTimer() + b.Log(m) // prevent dead code elimination +} + func TestResolve(t *testing.T) { existing := "existing" notExisting := "not-existing" @@ -151,7 +193,8 @@ func TestCopyInto(t *testing.T) { {"existingNamedType", []TypeDef{{Name: existing, Atom: a}}, TypeRef{NamedType: &existing}}, } - for _, tt := range tests { + for i := range tests { + tt := tests[i] t.Run(tt.testName, func(t *testing.T) { t.Parallel() s := Schema{ @@ -176,3 +219,27 @@ func TestCopyInto(t *testing.T) { }) } } + +func TestMapCopyInto(t *testing.T) { + s := Map{ + Fields: []StructField{ + { + Name: "a", + Type: TypeRef{NamedType: strptr("aaa")}, + }, + }, + } + theCopy := Map{} + + assert := func(sf StructField, ok bool) { + if !ok || *sf.Type.NamedType != "aaa" { + t.Error("expected NamedType aaa not found") + } + } + + go func() { + s.CopyInto(&theCopy) + assert(theCopy.FindField("a")) + }() + assert(s.FindField("a")) +} diff --git a/schema/equals_test.go b/schema/equals_test.go index 0994a771..7acfa644 100644 --- a/schema/equals_test.go +++ b/schema/equals_test.go @@ -22,47 +22,47 @@ import ( "testing" "testing/quick" - fuzz "github.com/google/gofuzz" + "sigs.k8s.io/randfill" ) -func fuzzInterface(i *interface{}, c fuzz.Continue) { +func fuzzInterface(i *interface{}, c randfill.Continue) { m := map[string]string{} - c.Fuzz(&m) + c.Fill(&m) *i = &m } func (*Schema) Generate(rand *rand.Rand, size int) reflect.Value { - s := &Schema{} - f := fuzz.New().RandSource(rand).MaxDepth(4) - f.Fuzz(s) - return reflect.ValueOf(s) + s := Schema{} + f := randfill.New().RandSource(rand).MaxDepth(4) + f.Fill(&s) + return reflect.ValueOf(&s) } -func (Map) Generate(rand *rand.Rand, size int) reflect.Value { +func (*Map) Generate(rand *rand.Rand, size int) reflect.Value { m := Map{} - f := fuzz.New().RandSource(rand).MaxDepth(4).Funcs(fuzzInterface) - f.Fuzz(&m) - return reflect.ValueOf(m) + f := randfill.New().RandSource(rand).MaxDepth(4).Funcs(fuzzInterface) + f.Fill(&m) + return reflect.ValueOf(&m) } func (TypeDef) Generate(rand *rand.Rand, size int) reflect.Value { td := TypeDef{} - f := fuzz.New().RandSource(rand).MaxDepth(4) - f.Fuzz(&td) + f := randfill.New().RandSource(rand).MaxDepth(4) + f.Fill(&td) return reflect.ValueOf(td) } func (Atom) Generate(rand *rand.Rand, size int) reflect.Value { a := Atom{} - f := fuzz.New().RandSource(rand).MaxDepth(4) - f.Fuzz(&a) + f := randfill.New().RandSource(rand).MaxDepth(4) + f.Fill(&a) return reflect.ValueOf(a) } func (StructField) Generate(rand *rand.Rand, size int) reflect.Value { a := StructField{} - f := fuzz.New().RandSource(rand).MaxDepth(4).Funcs(fuzzInterface) - f.Fuzz(&a) + f := randfill.New().RandSource(rand).MaxDepth(4).Funcs(fuzzInterface) + f.Fill(&a) return reflect.ValueOf(a) } @@ -109,8 +109,8 @@ func TestEquals(t *testing.T) { y.Map = x.Map return x.Equals(&y) == reflect.DeepEqual(x, y) }, - func(x Map) bool { - if !x.Equals(&x) { + func(x *Map) bool { + if !x.Equals(x) { return false } var y Map @@ -118,7 +118,7 @@ func TestEquals(t *testing.T) { y.ElementRelationship = x.ElementRelationship y.Fields = x.Fields y.Unions = x.Unions - return x.Equals(&y) == reflect.DeepEqual(&x, &y) + return x.Equals(&y) == reflect.DeepEqual(x, &y) }, func(x Union) bool { if !x.Equals(&x) { diff --git a/schema/schemaschema.go b/schema/schemaschema.go index 7d64d130..6eb6c36d 100644 --- a/schema/schemaschema.go +++ b/schema/schemaschema.go @@ -110,7 +110,7 @@ var SchemaSchemaYAML = `types: scalar: string - name: deduceInvalidDiscriminator type: - scalar: bool + scalar: boolean - name: fields type: list: @@ -145,6 +145,7 @@ var SchemaSchemaYAML = `types: list: elementType: scalar: string + elementRelationship: atomic - name: untyped map: fields: diff --git a/smd/main.go b/smd/main.go index 17693f8b..b8b29813 100644 --- a/smd/main.go +++ b/smd/main.go @@ -22,7 +22,7 @@ import ( "flag" "log" - "sigs.k8s.io/structured-merge-diff/v5/internal/cli" + "sigs.k8s.io/structured-merge-diff/v6/internal/cli" ) func main() { diff --git a/typed/compare.go b/typed/compare.go new file mode 100644 index 00000000..488251f6 --- /dev/null +++ b/typed/compare.go @@ -0,0 +1,470 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package typed + +import ( + "fmt" + "strings" + + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" +) + +// Comparison is the return value of a TypedValue.Compare() operation. +// +// No field will appear in more than one of the three fieldsets. If all of the +// fieldsets are empty, then the objects must have been equal. +type Comparison struct { + // Removed contains any fields removed by rhs (the right-hand-side + // object in the comparison). + Removed *fieldpath.Set + // Modified contains fields present in both objects but different. + Modified *fieldpath.Set + // Added contains any fields added by rhs. + Added *fieldpath.Set +} + +// IsSame returns true if the comparison returned no changes (the two +// compared objects are similar). +func (c *Comparison) IsSame() bool { + return c.Removed.Empty() && c.Modified.Empty() && c.Added.Empty() +} + +// String returns a human readable version of the comparison. +func (c *Comparison) String() string { + bld := strings.Builder{} + if !c.Modified.Empty() { + bld.WriteString(fmt.Sprintf("- Modified Fields:\n%v\n", c.Modified)) + } + if !c.Added.Empty() { + bld.WriteString(fmt.Sprintf("- Added Fields:\n%v\n", c.Added)) + } + if !c.Removed.Empty() { + bld.WriteString(fmt.Sprintf("- Removed Fields:\n%v\n", c.Removed)) + } + return bld.String() +} + +// ExcludeFields fields from the compare recursively removes the fields +// from the entire comparison +func (c *Comparison) ExcludeFields(fields *fieldpath.Set) *Comparison { + if fields == nil || fields.Empty() { + return c + } + c.Removed = c.Removed.RecursiveDifference(fields) + c.Modified = c.Modified.RecursiveDifference(fields) + c.Added = c.Added.RecursiveDifference(fields) + return c +} + +func (c *Comparison) FilterFields(filter fieldpath.Filter) *Comparison { + if filter == nil { + return c + } + c.Removed = filter.Filter(c.Removed) + c.Modified = filter.Filter(c.Modified) + c.Added = filter.Filter(c.Added) + return c +} + +type compareWalker struct { + lhs value.Value + rhs value.Value + schema *schema.Schema + typeRef schema.TypeRef + + // Current path that we are comparing + path fieldpath.Path + + // Resulting comparison. + comparison *Comparison + + // internal housekeeping--don't set when constructing. + inLeaf bool // Set to true if we're in a "big leaf"--atomic map/list + + // Allocate only as many walkers as needed for the depth by storing them here. + spareWalkers *[]*compareWalker + + allocator value.Allocator +} + +// compare compares stuff. +func (w *compareWalker) compare(prefixFn func() string) (errs ValidationErrors) { + if w.lhs == nil && w.rhs == nil { + // check this condidition here instead of everywhere below. + return errorf("at least one of lhs and rhs must be provided") + } + a, ok := w.schema.Resolve(w.typeRef) + if !ok { + return errorf("schema error: no type found matching: %v", *w.typeRef.NamedType) + } + + alhs := deduceAtom(a, w.lhs) + arhs := deduceAtom(a, w.rhs) + + // deduceAtom does not fix the type for nil values + // nil is a wildcard and will accept whatever form the other operand takes + if w.rhs == nil { + errs = append(errs, handleAtom(alhs, w.typeRef, w)...) + } else if w.lhs == nil || alhs.Equals(&arhs) { + errs = append(errs, handleAtom(arhs, w.typeRef, w)...) + } else { + w2 := *w + errs = append(errs, handleAtom(alhs, w.typeRef, &w2)...) + errs = append(errs, handleAtom(arhs, w.typeRef, w)...) + } + + if !w.inLeaf { + if w.lhs == nil { + w.comparison.Added.Insert(w.path) + } else if w.rhs == nil { + w.comparison.Removed.Insert(w.path) + } + } + return errs.WithLazyPrefix(prefixFn) +} + +// doLeaf should be called on leaves before descending into children, if there +// will be a descent. It modifies w.inLeaf. +func (w *compareWalker) doLeaf() { + if w.inLeaf { + // We're in a "big leaf", an atomic map or list. Ignore + // subsequent leaves. + return + } + w.inLeaf = true + + // We don't recurse into leaf fields for merging. + if w.lhs == nil { + w.comparison.Added.Insert(w.path) + } else if w.rhs == nil { + w.comparison.Removed.Insert(w.path) + } else if !value.EqualsUsing(w.allocator, w.rhs, w.lhs) { + // TODO: Equality is not sufficient for this. + // Need to implement equality check on the value type. + w.comparison.Modified.Insert(w.path) + } +} + +func (w *compareWalker) doScalar(t *schema.Scalar) ValidationErrors { + // Make sure at least one side is a valid scalar. + lerrs := validateScalar(t, w.lhs, "lhs: ") + rerrs := validateScalar(t, w.rhs, "rhs: ") + if len(lerrs) > 0 && len(rerrs) > 0 { + return append(lerrs, rerrs...) + } + + // All scalars are leaf fields. + w.doLeaf() + + return nil +} + +func (w *compareWalker) prepareDescent(pe fieldpath.PathElement, tr schema.TypeRef, cmp *Comparison) *compareWalker { + if w.spareWalkers == nil { + // first descent. + w.spareWalkers = &[]*compareWalker{} + } + var w2 *compareWalker + if n := len(*w.spareWalkers); n > 0 { + w2, *w.spareWalkers = (*w.spareWalkers)[n-1], (*w.spareWalkers)[:n-1] + } else { + w2 = &compareWalker{} + } + *w2 = *w + w2.typeRef = tr + w2.path = append(w2.path, pe) + w2.lhs = nil + w2.rhs = nil + w2.comparison = cmp + return w2 +} + +func (w *compareWalker) finishDescent(w2 *compareWalker) { + // if the descent caused a realloc, ensure that we reuse the buffer + // for the next sibling. + w.path = w2.path[:len(w2.path)-1] + *w.spareWalkers = append(*w.spareWalkers, w2) +} + +func (w *compareWalker) derefMap(prefix string, v value.Value) (value.Map, ValidationErrors) { + if v == nil { + return nil, nil + } + m, err := mapValue(w.allocator, v) + if err != nil { + return nil, errorf("%v: %v", prefix, err) + } + return m, nil +} + +func (w *compareWalker) visitListItems(t *schema.List, lhs, rhs value.List) (errs ValidationErrors) { + rLen := 0 + if rhs != nil { + rLen = rhs.Length() + } + lLen := 0 + if lhs != nil { + lLen = lhs.Length() + } + + maxLength := rLen + if lLen > maxLength { + maxLength = lLen + } + // Contains all the unique PEs between lhs and rhs, exactly once. + // Order doesn't matter since we're just tracking ownership in a set. + allPEs := make([]fieldpath.PathElement, 0, maxLength) + + // Gather all the elements from lhs, indexed by PE, in a list for duplicates. + lValues := fieldpath.MakePathElementMap(lLen) + for i := 0; i < lLen; i++ { + child := lhs.At(i) + pe, err := listItemToPathElement(w.allocator, w.schema, t, child) + if err != nil { + errs = append(errs, errorf("element %v: %v", i, err.Error())...) + // If we can't construct the path element, we can't + // even report errors deeper in the schema, so bail on + // this element. + continue + } + + if v, found := lValues.Get(pe); found { + list := v.([]value.Value) + lValues.Insert(pe, append(list, child)) + } else { + lValues.Insert(pe, []value.Value{child}) + allPEs = append(allPEs, pe) + } + } + + // Gather all the elements from rhs, indexed by PE, in a list for duplicates. + rValues := fieldpath.MakePathElementMap(rLen) + for i := 0; i < rLen; i++ { + rValue := rhs.At(i) + pe, err := listItemToPathElement(w.allocator, w.schema, t, rValue) + if err != nil { + errs = append(errs, errorf("element %v: %v", i, err.Error())...) + // If we can't construct the path element, we can't + // even report errors deeper in the schema, so bail on + // this element. + continue + } + if v, found := rValues.Get(pe); found { + list := v.([]value.Value) + rValues.Insert(pe, append(list, rValue)) + } else { + rValues.Insert(pe, []value.Value{rValue}) + if _, found := lValues.Get(pe); !found { + allPEs = append(allPEs, pe) + } + } + } + + for _, pe := range allPEs { + lList := []value.Value(nil) + if l, ok := lValues.Get(pe); ok { + lList = l.([]value.Value) + } + rList := []value.Value(nil) + if l, ok := rValues.Get(pe); ok { + rList = l.([]value.Value) + } + + switch { + case len(lList) == 0 && len(rList) == 0: + // We shouldn't be here anyway. + return + // Normal use-case: + // We have no duplicates for this PE, compare items one-to-one. + case len(lList) <= 1 && len(rList) <= 1: + lValue := value.Value(nil) + if len(lList) != 0 { + lValue = lList[0] + } + rValue := value.Value(nil) + if len(rList) != 0 { + rValue = rList[0] + } + errs = append(errs, w.compareListItem(t, pe, lValue, rValue)...) + // Duplicates before & after use-case: + // Compare the duplicates lists as if they were atomic, mark modified if they changed. + case len(lList) >= 2 && len(rList) >= 2: + listEqual := func(lList, rList []value.Value) bool { + if len(lList) != len(rList) { + return false + } + for i := range lList { + if !value.Equals(lList[i], rList[i]) { + return false + } + } + return true + } + if !listEqual(lList, rList) { + w.comparison.Modified.Insert(append(w.path, pe)) + } + // Duplicates before & not anymore use-case: + // Rcursively add new non-duplicate items, Remove duplicate marker, + case len(lList) >= 2: + if len(rList) != 0 { + errs = append(errs, w.compareListItem(t, pe, nil, rList[0])...) + } + w.comparison.Removed.Insert(append(w.path, pe)) + // New duplicates use-case: + // Recursively remove old non-duplicate items, add duplicate marker. + case len(rList) >= 2: + if len(lList) != 0 { + errs = append(errs, w.compareListItem(t, pe, lList[0], nil)...) + } + w.comparison.Added.Insert(append(w.path, pe)) + } + } + + return +} + +func (w *compareWalker) indexListPathElements(t *schema.List, list value.List) ([]fieldpath.PathElement, fieldpath.PathElementValueMap, ValidationErrors) { + var errs ValidationErrors + length := 0 + if list != nil { + length = list.Length() + } + observed := fieldpath.MakePathElementValueMap(length) + pes := make([]fieldpath.PathElement, 0, length) + for i := 0; i < length; i++ { + child := list.At(i) + pe, err := listItemToPathElement(w.allocator, w.schema, t, child) + if err != nil { + errs = append(errs, errorf("element %v: %v", i, err.Error())...) + // If we can't construct the path element, we can't + // even report errors deeper in the schema, so bail on + // this element. + continue + } + // Ignore repeated occurences of `pe`. + if _, found := observed.Get(pe); found { + continue + } + observed.Insert(pe, child) + pes = append(pes, pe) + } + return pes, observed, errs +} + +func (w *compareWalker) compareListItem(t *schema.List, pe fieldpath.PathElement, lChild, rChild value.Value) ValidationErrors { + w2 := w.prepareDescent(pe, t.ElementType, w.comparison) + w2.lhs = lChild + w2.rhs = rChild + errs := w2.compare(pe.String) + w.finishDescent(w2) + return errs +} + +func (w *compareWalker) derefList(prefix string, v value.Value) (value.List, ValidationErrors) { + if v == nil { + return nil, nil + } + l, err := listValue(w.allocator, v) + if err != nil { + return nil, errorf("%v: %v", prefix, err) + } + return l, nil +} + +func (w *compareWalker) doList(t *schema.List) (errs ValidationErrors) { + lhs, _ := w.derefList("lhs: ", w.lhs) + if lhs != nil { + defer w.allocator.Free(lhs) + } + rhs, _ := w.derefList("rhs: ", w.rhs) + if rhs != nil { + defer w.allocator.Free(rhs) + } + + // If both lhs and rhs are empty/null, treat it as a + // leaf: this helps preserve the empty/null + // distinction. + emptyPromoteToLeaf := (lhs == nil || lhs.Length() == 0) && (rhs == nil || rhs.Length() == 0) + + if t.ElementRelationship == schema.Atomic || emptyPromoteToLeaf { + w.doLeaf() + return nil + } + + if lhs == nil && rhs == nil { + return nil + } + + errs = w.visitListItems(t, lhs, rhs) + + return errs +} + +func (w *compareWalker) visitMapItem(t *schema.Map, out map[string]interface{}, key string, lhs, rhs value.Value) (errs ValidationErrors) { + fieldType := t.ElementType + if sf, ok := t.FindField(key); ok { + fieldType = sf.Type + } + pe := fieldpath.PathElement{FieldName: &key} + w2 := w.prepareDescent(pe, fieldType, w.comparison) + w2.lhs = lhs + w2.rhs = rhs + errs = append(errs, w2.compare(pe.String)...) + w.finishDescent(w2) + return errs +} + +func (w *compareWalker) visitMapItems(t *schema.Map, lhs, rhs value.Map) (errs ValidationErrors) { + out := map[string]interface{}{} + + value.MapZipUsing(w.allocator, lhs, rhs, value.Unordered, func(key string, lhsValue, rhsValue value.Value) bool { + errs = append(errs, w.visitMapItem(t, out, key, lhsValue, rhsValue)...) + return true + }) + + return errs +} + +func (w *compareWalker) doMap(t *schema.Map) (errs ValidationErrors) { + lhs, _ := w.derefMap("lhs: ", w.lhs) + if lhs != nil { + defer w.allocator.Free(lhs) + } + rhs, _ := w.derefMap("rhs: ", w.rhs) + if rhs != nil { + defer w.allocator.Free(rhs) + } + // If both lhs and rhs are empty/null, treat it as a + // leaf: this helps preserve the empty/null + // distinction. + emptyPromoteToLeaf := (lhs == nil || lhs.Empty()) && (rhs == nil || rhs.Empty()) + + if t.ElementRelationship == schema.Atomic || emptyPromoteToLeaf { + w.doLeaf() + return nil + } + + if lhs == nil && rhs == nil { + return nil + } + + errs = append(errs, w.visitMapItems(t, lhs, rhs)...) + + return errs +} diff --git a/typed/comparison_test.go b/typed/comparison_test.go index 1334a5ca..73c93622 100644 --- a/typed/comparison_test.go +++ b/typed/comparison_test.go @@ -3,8 +3,8 @@ package typed_test import ( "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) func TestComparisonExcludeFields(t *testing.T) { diff --git a/typed/deduced_test.go b/typed/deduced_test.go index 670b70c5..2d737a2e 100644 --- a/typed/deduced_test.go +++ b/typed/deduced_test.go @@ -20,8 +20,8 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) func TestValidateDeducedType(t *testing.T) { diff --git a/typed/helpers.go b/typed/helpers.go index f58cd67a..8a9c0b50 100644 --- a/typed/helpers.go +++ b/typed/helpers.go @@ -21,9 +21,9 @@ import ( "fmt" "strings" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // ValidationError reports an error about a particular field @@ -197,7 +197,7 @@ func getAssociativeKeyDefault(s *schema.Schema, list *schema.List, fieldName str return field.Default, nil } -func keyedAssociativeListItemToPathElement(a value.Allocator, s *schema.Schema, list *schema.List, index int, child value.Value) (fieldpath.PathElement, error) { +func keyedAssociativeListItemToPathElement(a value.Allocator, s *schema.Schema, list *schema.List, child value.Value) (fieldpath.PathElement, error) { pe := fieldpath.PathElement{} if child.IsNull() { // null entries are illegal. @@ -217,15 +217,22 @@ func keyedAssociativeListItemToPathElement(a value.Allocator, s *schema.Schema, } else if def != nil { keyMap = append(keyMap, value.Field{Name: fieldName, Value: value.NewValueInterface(def)}) } else { - return pe, fmt.Errorf("associative list with keys has an element that omits key field %q (and doesn't have default value)", fieldName) + // Don't add the key to the key field list. + // A key field list where it is set then represents a different entry + // in the associate list. } } + + if len(list.Keys) > 0 && len(keyMap) == 0 { + return pe, fmt.Errorf("associative list with keys has an element that omits all key fields %q (and doesn't have default values for any key fields)", list.Keys) + } + keyMap.Sort() pe.Key = &keyMap return pe, nil } -func setItemToPathElement(list *schema.List, index int, child value.Value) (fieldpath.PathElement, error) { +func setItemToPathElement(child value.Value) (fieldpath.PathElement, error) { pe := fieldpath.PathElement{} switch { case child.IsMap(): @@ -245,16 +252,15 @@ func setItemToPathElement(list *schema.List, index int, child value.Value) (fiel } } -func listItemToPathElement(a value.Allocator, s *schema.Schema, list *schema.List, index int, child value.Value) (fieldpath.PathElement, error) { - if list.ElementRelationship == schema.Associative { - if len(list.Keys) > 0 { - return keyedAssociativeListItemToPathElement(a, s, list, index, child) - } +func listItemToPathElement(a value.Allocator, s *schema.Schema, list *schema.List, child value.Value) (fieldpath.PathElement, error) { + if list.ElementRelationship != schema.Associative { + return fieldpath.PathElement{}, errors.New("invalid indexing of non-associative list") + } - // If there's no keys, then we must be a set of primitives. - return setItemToPathElement(list, index, child) + if len(list.Keys) > 0 { + return keyedAssociativeListItemToPathElement(a, s, list, child) } - // Use the index as a key for atomic lists. - return fieldpath.PathElement{Index: &index}, nil + // If there's no keys, then we must be a set of primitives. + return setItemToPathElement(child) } diff --git a/typed/helpers_test.go b/typed/helpers_test.go index dfc431d5..2b0d1c4f 100644 --- a/typed/helpers_test.go +++ b/typed/helpers_test.go @@ -4,8 +4,8 @@ import ( "strings" "testing" - "sigs.k8s.io/structured-merge-diff/v5/internal/fixture" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/internal/fixture" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) func TestInvalidOverride(t *testing.T) { diff --git a/typed/merge.go b/typed/merge.go index bcfa2472..f8ca9aba 100644 --- a/typed/merge.go +++ b/typed/merge.go @@ -17,9 +17,9 @@ limitations under the License. package typed import ( - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type mergingWalker struct { @@ -113,11 +113,12 @@ func (w *mergingWalker) doLeaf() { w.rule(w) } -func (w *mergingWalker) doScalar(t *schema.Scalar) (errs ValidationErrors) { - errs = append(errs, validateScalar(t, w.lhs, "lhs: ")...) - errs = append(errs, validateScalar(t, w.rhs, "rhs: ")...) - if len(errs) > 0 { - return errs +func (w *mergingWalker) doScalar(t *schema.Scalar) ValidationErrors { + // Make sure at least one side is a valid scalar. + lerrs := validateScalar(t, w.lhs, "lhs: ") + rerrs := validateScalar(t, w.rhs, "rhs: ") + if len(lerrs) > 0 && len(rerrs) > 0 { + return append(lerrs, rerrs...) } // All scalars are leaf fields. @@ -179,14 +180,18 @@ func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (err } out := make([]interface{}, 0, outLen) - rhsOrder, observedRHS, rhsErrs := w.indexListPathElements(t, rhs) + rhsPEs, observedRHS, rhsErrs := w.indexListPathElements(t, rhs, false) errs = append(errs, rhsErrs...) - lhsOrder, observedLHS, lhsErrs := w.indexListPathElements(t, lhs) + lhsPEs, observedLHS, lhsErrs := w.indexListPathElements(t, lhs, true) errs = append(errs, lhsErrs...) + if len(errs) != 0 { + return errs + } + sharedOrder := make([]*fieldpath.PathElement, 0, rLen) - for i := range rhsOrder { - pe := &rhsOrder[i] + for i := range rhsPEs { + pe := &rhsPEs[i] if _, ok := observedLHS.Get(*pe); ok { sharedOrder = append(sharedOrder, pe) } @@ -198,13 +203,15 @@ func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (err sharedOrder = sharedOrder[1:] } - lLen, rLen = len(lhsOrder), len(rhsOrder) + mergedRHS := fieldpath.MakePathElementMap(len(rhsPEs)) + lLen, rLen = len(lhsPEs), len(rhsPEs) for lI, rI := 0, 0; lI < lLen || rI < rLen; { if lI < lLen && rI < rLen { - pe := lhsOrder[lI] - if pe.Equals(rhsOrder[rI]) { + pe := lhsPEs[lI] + if pe.Equals(rhsPEs[rI]) { // merge LHS & RHS items - lChild, _ := observedLHS.Get(pe) + mergedRHS.Insert(pe, struct{}{}) + lChild, _ := observedLHS.Get(pe) // may be nil if the PE is duplicaated. rChild, _ := observedRHS.Get(pe) mergeOut, errs := w.mergeListItem(t, pe, lChild, rChild) errs = append(errs, errs...) @@ -221,17 +228,17 @@ func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (err } continue } - if _, ok := observedRHS.Get(pe); ok && nextShared != nil && !nextShared.Equals(lhsOrder[lI]) { + if _, ok := observedRHS.Get(pe); ok && nextShared != nil && !nextShared.Equals(lhsPEs[lI]) { // shared item, but not the one we want in this round lI++ continue } } if lI < lLen { - pe := lhsOrder[lI] + pe := lhsPEs[lI] if _, ok := observedRHS.Get(pe); !ok { - // take LHS item - lChild, _ := observedLHS.Get(pe) + // take LHS item using At to make sure we get the right item (observed may not contain the right item). + lChild := lhs.AtUsing(w.allocator, lI) mergeOut, errs := w.mergeListItem(t, pe, lChild, nil) errs = append(errs, errs...) if mergeOut != nil { @@ -239,12 +246,16 @@ func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (err } lI++ continue + } else if _, ok := mergedRHS.Get(pe); ok { + // we've already merged it with RHS, we don't want to duplicate it, skip it. + lI++ } } if rI < rLen { // Take the RHS item, merge with matching LHS item if possible - pe := rhsOrder[rI] - lChild, _ := observedLHS.Get(pe) // may be nil + pe := rhsPEs[rI] + mergedRHS.Insert(pe, struct{}{}) + lChild, _ := observedLHS.Get(pe) // may be nil if absent or duplicaated. rChild, _ := observedRHS.Get(pe) mergeOut, errs := w.mergeListItem(t, pe, lChild, rChild) errs = append(errs, errs...) @@ -271,7 +282,7 @@ func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (err return errs } -func (w *mergingWalker) indexListPathElements(t *schema.List, list value.List) ([]fieldpath.PathElement, fieldpath.PathElementValueMap, ValidationErrors) { +func (w *mergingWalker) indexListPathElements(t *schema.List, list value.List, allowDuplicates bool) ([]fieldpath.PathElement, fieldpath.PathElementValueMap, ValidationErrors) { var errs ValidationErrors length := 0 if list != nil { @@ -281,7 +292,7 @@ func (w *mergingWalker) indexListPathElements(t *schema.List, list value.List) ( pes := make([]fieldpath.PathElement, 0, length) for i := 0; i < length; i++ { child := list.At(i) - pe, err := listItemToPathElement(w.allocator, w.schema, t, i, child) + pe, err := listItemToPathElement(w.allocator, w.schema, t, child) if err != nil { errs = append(errs, errorf("element %v: %v", i, err.Error())...) // If we can't construct the path element, we can't @@ -289,11 +300,15 @@ func (w *mergingWalker) indexListPathElements(t *schema.List, list value.List) ( // this element. continue } - if _, found := observed.Get(pe); found { + if _, found := observed.Get(pe); found && !allowDuplicates { errs = append(errs, errorf("duplicate entries for key %v", pe.String())...) continue + } else if !found { + observed.Insert(pe, child) + } else { + // Duplicated items are not merged with the new value, make them nil. + observed.Insert(pe, value.NewValueInterface(nil)) } - observed.Insert(pe, child) pes = append(pes, pe) } return pes, observed, errs diff --git a/typed/merge_test.go b/typed/merge_test.go index 5e62cd4d..c799e32f 100644 --- a/typed/merge_test.go +++ b/typed/merge_test.go @@ -20,8 +20,8 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type mergeTestCase struct { @@ -328,6 +328,34 @@ var mergeCases = []mergeTestCase{{ `{"setStr":["a","b","c","d","e","f","g","h","i","j"]}`, `{"setStr":["1","b","2","h","3","e","4","k","l"]}`, `{"setStr":["a","1","b","c","d","f","g","2","h","i","j","3","e","4","k","l"]}`, + }, { // We have a duplicate in LHS + `{"setStr":["a","b","b"]}`, + `{"setStr":["c"]}`, + `{"setStr":["a","b","b","c"]}`, + }, { // We have a duplicate in LHS. + `{"setStr":["a","b","b"]}`, + `{"setStr":["b"]}`, + `{"setStr":["a","b"]}`, + }, { // We have a duplicate in LHS. + `{"setStr":["a","b","b"]}`, + `{"setStr":["a"]}`, + `{"setStr":["a","b","b"]}`, + }, { // We have a duplicate in LHS. + `{"setStr":["a","b","c","d","e","c"]}`, + `{"setStr":["1","b","2","e","d"]}`, + `{"setStr":["a","1","b","c","2","e","c","d"]}`, + }, { // We have a duplicate in LHS, also present in RHS, keep only one. + `{"setStr":["a","b","c","d","e","c"]}`, + `{"setStr":["1","b","2","c","e","d"]}`, + `{"setStr":["a","1","b","2","c","e","d"]}`, + }, { // We have 2 duplicates in LHS, one is replaced. + `{"setStr":["a","a","b","b"]}`, + `{"setStr":["b","c","d"]}`, + `{"setStr":["a","a","b","c","d"]}`, + }, { // We have 2 duplicates in LHS, and nothing on the right + `{"setStr":["a","a","b","b"]}`, + `{"setStr":[]}`, + `{"setStr":["a","a","b","b"]}`, }, { `{"setBool":[true]}`, `{"setBool":[false]}`, @@ -336,6 +364,22 @@ var mergeCases = []mergeTestCase{{ `{"setNumeric":[1,2,3.14159]}`, `{"setNumeric":[1,2,3]}`, `{"setNumeric":[1,2,3.14159,3]}`, + }, { + `{"setStr":["c","a","g","f","c","a"]}`, + `{"setStr":["c","f","a","g"]}`, + `{"setStr":["c","f","a","g"]}`, + }, { + `{"setNumeric":[1,2,3.14159,1,2]}`, + `{"setNumeric":[1,2,3]}`, + `{"setNumeric":[1,2,3.14159,3]}`, + }, { + `{"setBool":[true,false,true]}`, + `{"setBool":[false]}`, + `{"setBool":[true,false,true]}`, + }, { + `{"setBool":[true,false,true]}`, + `{"setBool":[true]}`, + `{"setBool":[true, false]}`, }, }, }, { @@ -423,6 +467,18 @@ var mergeCases = []mergeTestCase{{ `{"atomicList":["a","a","a"]}`, `{"atomicList":["a","a"]}`, `{"atomicList":["a","a"]}`, + }, { + `{"list":[{"key":"a","id":1,"bv":true},{"key":"b","id":2},{"key":"a","id":1,"bv":false,"nv":2}]}`, + `{"list":[{"key":"a","id":1,"nv":3},{"key":"c","id":3},{"key":"b","id":2}]}`, + `{"list":[{"key":"a","id":1,"nv":3},{"key":"c","id":3},{"key":"b","id":2}]}`, + }, { + `{"list":[{"key":"a","id":1,"nv":1},{"key":"a","id":1,"nv":2}]}`, + `{"list":[]}`, + `{"list":[{"key":"a","id":1,"nv":1},{"key":"a","id":1,"nv":2}]}`, + }, { + `{"list":[{"key":"a","id":1,"nv":1},{"key":"a","id":1,"nv":2}]}`, + `{}`, + `{"list":[{"key":"a","id":1,"nv":1},{"key":"a","id":1,"nv":2}]}`, }}, }} @@ -437,18 +493,16 @@ func (tt mergeTestCase) test(t *testing.T) { t.Run(fmt.Sprintf("%v-valid-%v", tt.name, i), func(t *testing.T) { t.Parallel() pt := parser.Type(tt.rootTypeName) - - lhs, err := pt.FromYAML(triplet.lhs) + // Former object can have duplicates in sets. + lhs, err := pt.FromYAML(triplet.lhs, typed.AllowDuplicates) if err != nil { t.Fatalf("unable to parser/validate lhs yaml: %v\n%v", err, triplet.lhs) } - rhs, err := pt.FromYAML(triplet.rhs) if err != nil { t.Fatalf("unable to parser/validate rhs yaml: %v\n%v", err, triplet.rhs) } - - out, err := pt.FromYAML(triplet.out) + out, err := pt.FromYAML(triplet.out, typed.AllowDuplicates) if err != nil { t.Fatalf("unable to parser/validate out yaml: %v\n%v", err, triplet.out) } diff --git a/typed/parser.go b/typed/parser.go index 83af2d10..c46e69f2 100644 --- a/typed/parser.go +++ b/typed/parser.go @@ -19,9 +19,9 @@ package typed import ( "fmt" - yaml "gopkg.in/yaml.v2" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + yaml "go.yaml.in/yaml/v2" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) // YAMLObject is an object encoded in YAML. @@ -29,14 +29,12 @@ type YAMLObject string // Parser implements YAMLParser and allows introspecting the schema. type Parser struct { - Schema *schema.Schema + Schema schema.Schema } // create builds an unvalidated parser. func create(s YAMLObject) (*Parser, error) { - p := Parser{ - Schema: &schema.Schema{}, - } + p := Parser{} err := yaml.Unmarshal([]byte(s), &p.Schema) return &p, err } @@ -76,7 +74,7 @@ func (p *Parser) TypeNames() (names []string) { // errors are deferred until a further function is called. func (p *Parser) Type(name string) ParseableType { return ParseableType{ - Schema: p.Schema, + Schema: &p.Schema, TypeRef: schema.TypeRef{NamedType: &name}, } } @@ -95,13 +93,13 @@ func (p ParseableType) IsValid() bool { // FromYAML parses a yaml string into an object with the current schema // and the type "typename" or an error if validation fails. -func (p ParseableType) FromYAML(object YAMLObject) (*TypedValue, error) { +func (p ParseableType) FromYAML(object YAMLObject, opts ...ValidationOptions) (*TypedValue, error) { var v interface{} err := yaml.Unmarshal([]byte(object), &v) if err != nil { return nil, err } - return AsTyped(value.NewValueInterface(v), p.Schema, p.TypeRef) + return AsTyped(value.NewValueInterface(v), p.Schema, p.TypeRef, opts...) } // FromUnstructured converts a go "interface{}" type, typically an @@ -110,8 +108,8 @@ func (p ParseableType) FromYAML(object YAMLObject) (*TypedValue, error) { // The provided interface{} must be one of: map[string]interface{}, // map[interface{}]interface{}, []interface{}, int types, float types, // string or boolean. Nested interface{} must also be one of these types. -func (p ParseableType) FromUnstructured(in interface{}) (*TypedValue, error) { - return AsTyped(value.NewValueInterface(in), p.Schema, p.TypeRef) +func (p ParseableType) FromUnstructured(in interface{}, opts ...ValidationOptions) (*TypedValue, error) { + return AsTyped(value.NewValueInterface(in), p.Schema, p.TypeRef, opts...) } // FromStructured converts a go "interface{}" type, typically an structured object in @@ -119,12 +117,12 @@ func (p ParseableType) FromUnstructured(in interface{}) (*TypedValue, error) { // schema validation. The provided "interface{}" value must be a pointer so that the // value can be modified via reflection. The provided "interface{}" may contain structs // and types that are converted to Values by the jsonMarshaler interface. -func (p ParseableType) FromStructured(in interface{}) (*TypedValue, error) { +func (p ParseableType) FromStructured(in interface{}, opts ...ValidationOptions) (*TypedValue, error) { v, err := value.NewValueReflect(in) if err != nil { return nil, fmt.Errorf("error creating struct value reflector: %v", err) } - return AsTyped(v, p.Schema, p.TypeRef) + return AsTyped(v, p.Schema, p.TypeRef, opts...) } // DeducedParseableType is a ParseableType that deduces the type from diff --git a/typed/parser_test.go b/typed/parser_test.go index d874fc08..9ef51cb6 100644 --- a/typed/parser_test.go +++ b/typed/parser_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - yaml "gopkg.in/yaml.v2" - "sigs.k8s.io/structured-merge-diff/v5/typed" + yaml "go.yaml.in/yaml/v2" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) func testdata(file string) string { diff --git a/typed/reconcile_schema.go b/typed/reconcile_schema.go index 4f9d2bca..9b20e54a 100644 --- a/typed/reconcile_schema.go +++ b/typed/reconcile_schema.go @@ -20,8 +20,8 @@ import ( "fmt" "sync" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" ) var fmPool = sync.Pool{ diff --git a/typed/reconcile_schema_test.go b/typed/reconcile_schema_test.go index c244b8b1..9f0ace86 100644 --- a/typed/reconcile_schema_test.go +++ b/typed/reconcile_schema_test.go @@ -20,8 +20,8 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) type reconcileTestCase struct { diff --git a/typed/remove.go b/typed/remove.go index 020b0fbc..0db1734f 100644 --- a/typed/remove.go +++ b/typed/remove.go @@ -14,9 +14,9 @@ limitations under the License. package typed import ( - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type removingWalker struct { @@ -58,6 +58,10 @@ func (w *removingWalker) doList(t *schema.List) (errs ValidationErrors) { defer w.allocator.Free(l) // If list is null or empty just return if l == nil || l.Length() == 0 { + // For extraction, we just return the value as is (which is nil or empty). For extraction the difference matters. + if w.shouldExtract { + w.out = w.value.Unstructured() + } return nil } @@ -71,33 +75,50 @@ func (w *removingWalker) doList(t *schema.List) (errs ValidationErrors) { } var newItems []interface{} + hadMatches := false iter := l.RangeUsing(w.allocator) defer w.allocator.Free(iter) for iter.Next() { - i, item := iter.Item() + _, item := iter.Item() // Ignore error because we have already validated this list - pe, _ := listItemToPathElement(w.allocator, w.schema, t, i, item) + pe, _ := listItemToPathElement(w.allocator, w.schema, t, item) path, _ := fieldpath.MakePath(pe) // save items on the path when we shouldExtract // but ignore them when we are removing (i.e. !w.shouldExtract) - if w.toRemove.Has(path) { - if w.shouldExtract { - newItems = append(newItems, removeItemsWithSchema(item, w.toRemove, w.schema, t.ElementType, w.shouldExtract).Unstructured()) - } else { - continue + isExactPathMatch := w.toRemove.Has(path) + isPrefixMatch := !w.toRemove.WithPrefix(pe).Empty() + if w.shouldExtract { + if isPrefixMatch { + item = removeItemsWithSchema(item, w.toRemove.WithPrefix(pe), w.schema, t.ElementType, w.shouldExtract) + } + if isExactPathMatch || isPrefixMatch { + newItems = append(newItems, item.Unstructured()) } - } - if subset := w.toRemove.WithPrefix(pe); !subset.Empty() { - item = removeItemsWithSchema(item, subset, w.schema, t.ElementType, w.shouldExtract) } else { - // don't save items not on the path when we shouldExtract. - if w.shouldExtract { + if isExactPathMatch { continue } + if isPrefixMatch { + // Removing nested items within this list item and preserve if it becomes empty + hadMatches = true + wasMap := item.IsMap() + wasList := item.IsList() + item = removeItemsWithSchema(item, w.toRemove.WithPrefix(pe), w.schema, t.ElementType, w.shouldExtract) + // If item returned null but we're removing items within the structure(not the item itself), + // preserve the empty container structure + if item.IsNull() && !w.shouldExtract { + if wasMap { + item = value.NewValueInterface(map[string]interface{}{}) + } else if wasList { + item = value.NewValueInterface([]interface{}{}) + } + } + } + newItems = append(newItems, item.Unstructured()) } - newItems = append(newItems, item.Unstructured()) } - if len(newItems) > 0 { + // Preserve empty lists (non-nil) instead of converting to null when items were matched and removed + if len(newItems) > 0 || (hadMatches && !w.shouldExtract) { w.out = newItems } return nil @@ -113,6 +134,10 @@ func (w *removingWalker) doMap(t *schema.Map) ValidationErrors { } // If map is null or empty just return if m == nil || m.Empty() { + // For extraction, we just return the value as is (which is nil or empty). For extraction the difference matters. + if w.shouldExtract { + w.out = w.value.Unstructured() + } return nil } @@ -131,6 +156,7 @@ func (w *removingWalker) doMap(t *schema.Map) ValidationErrors { } newMap := map[string]interface{}{} + hadMatches := false m.Iterate(func(k string, val value.Value) bool { pe := fieldpath.PathElement{FieldName: &k} path, _ := fieldpath.MakePath(pe) @@ -148,7 +174,19 @@ func (w *removingWalker) doMap(t *schema.Map) ValidationErrors { return true } if subset := w.toRemove.WithPrefix(pe); !subset.Empty() { + hadMatches = true + wasMap := val.IsMap() + wasList := val.IsList() val = removeItemsWithSchema(val, subset, w.schema, fieldType, w.shouldExtract) + // If val returned null but we're removing items within the structure (not the field itself), + // preserve the empty container structure + if val.IsNull() && !w.shouldExtract { + if wasMap { + val = value.NewValueInterface(map[string]interface{}{}) + } else if wasList { + val = value.NewValueInterface([]interface{}{}) + } + } } else { // don't save values not on the path when we shouldExtract. if w.shouldExtract { @@ -158,7 +196,8 @@ func (w *removingWalker) doMap(t *schema.Map) ValidationErrors { newMap[k] = val.Unstructured() return true }) - if len(newMap) > 0 { + // Preserve empty maps (non-nil) instead of converting to null when items were matched and removed + if len(newMap) > 0 || (hadMatches && !w.shouldExtract) { w.out = newMap } return nil diff --git a/typed/remove_test.go b/typed/remove_test.go index ae71b664..76850652 100644 --- a/typed/remove_test.go +++ b/typed/remove_test.go @@ -20,9 +20,9 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type removeTestCase struct { @@ -107,6 +107,9 @@ var associativeAndAtomicSchema = `types: - name: atomicMap type: namedType: myAtomicMap + - name: atomicElementList + type: + namedType: myAtomicElementList - name: myList list: elementType: @@ -117,9 +120,20 @@ var associativeAndAtomicSchema = `types: - id - name: myAtomicMap map: + fields: + - name: id + type: + scalar: string elementType: scalar: string elementRelationship: atomic +- name: myAtomicElementList + list: + elementType: + namedType: myAtomicMap + elementRelationship: associative + keys: + - id - name: mySequence list: elementType: @@ -148,27 +162,6 @@ var associativeAndAtomicSchema = `types: elementType: scalar: string ` -var atomicTypesSchema = `types: -- name: myRoot - map: - fields: - - name: atomicMap - type: - namedType: myAtomicMap - - name: atomicList - type: - namedType: mySequence -- name: myAtomicMap - map: - elementType: - scalar: string - elementRelationship: atomic -- name: mySequence - list: - elementType: - scalar: string - elementRelationship: atomic -` var nestedTypesSchema = `types: - name: type @@ -288,7 +281,7 @@ var removeCases = []removeTestCase{{ quadruplets: []removeQuadruplet{{ `{"setBool":[false]}`, _NS(_P("setBool", _V(false))), - `{"setBool":null}`, + `{"setBool":[]}`, `{"setBool":[false]}`, }, { `{"setBool":[false]}`, @@ -678,7 +671,7 @@ var removeCases = []removeTestCase{{ _NS( _P("mapOfMapsRecursive", "a"), ), - `{"mapOfMapsRecursive"}`, + `{"mapOfMapsRecursive":{}}`, `{"mapOfMapsRecursive": {"a":null}}`, }, { // second-level map @@ -686,7 +679,7 @@ var removeCases = []removeTestCase{{ _NS( _P("mapOfMapsRecursive", "a", "b"), ), - `{"mapOfMapsRecursive":{"a":null}}`, + `{"mapOfMapsRecursive":{"a":{}}}`, `{"mapOfMapsRecursive": {"a":{"b":null}}}`, }, { // third-level map @@ -694,8 +687,40 @@ var removeCases = []removeTestCase{{ _NS( _P("mapOfMapsRecursive", "a", "b", "c"), ), - `{"mapOfMapsRecursive":{"a":{"b":null}}}`, + `{"mapOfMapsRecursive":{"a":{"b":{}}}}`, `{"mapOfMapsRecursive": {"a":{"b":{"c":null}}}}`, + }, { + // empty list + `{"listOfLists": []}`, + _NS( + _P("listOfLists"), + ), + ``, + `{"listOfLists": []}`, + }, { + // null list + `{"listOfLists": null}`, + _NS( + _P("listOfLists"), + ), + ``, + `{"listOfLists": null}`, + }, { + // empty map + `{"mapOfMaps": {}}`, + _NS( + _P("mapOfMaps"), + ), + ``, + `{"mapOfMaps": {}}`, + }, { + // nil map + `{"mapOfMaps": null}`, + _NS( + _P("mapOfMaps"), + ), + ``, + `{"mapOfMaps": null}`, }}, }} @@ -906,3 +931,124 @@ func TestReversibleExtract(t *testing.T) { }) } } + +type extractWithKeysTestCase struct { + name string + rootTypeName string + schema typed.YAMLObject + triplets []extractTriplet +} + +type extractTriplet struct { + object typed.YAMLObject + set *fieldpath.Set + wantOutput interface{} +} + +var extractWithKeysCases = []extractWithKeysTestCase{{ + name: "associativeAndAtomicSchema", + rootTypeName: "myRoot", + schema: typed.YAMLObject(associativeAndAtomicSchema), + triplets: []extractTriplet{ + { + // extract with all key fields included + object: `{"list":[{"key":"nginx","id":1,"nv":2}]}`, + set: _NS( + _P("list", _KBF("key", "nginx", "id", 1), "key"), + _P("list", _KBF("key", "nginx", "id", 1), "id"), + ), + wantOutput: typed.YAMLObject(`{"list":[{"key":"nginx","id":1}]}`), + }, + { + // extract no key field included + object: `{"list":[{"key":"nginx","id":1,"nv":2}]}`, + set: _NS( + _P("list", _KBF("key", "nginx", "id", 1), "nv"), + ), + wantOutput: typed.YAMLObject(`{"list":[{"key":"nginx","id":1, "nv":2}]}`), + }, + { + // extract with partial keys included + object: `{"list":[{"key":"nginx","id":1,"nv":2}]}`, + set: _NS( + _P("list", _KBF("key", "nginx", "id", 1), "nv"), + _P("list", _KBF("key", "nginx", "id", 1), "id"), + ), + wantOutput: typed.YAMLObject(`{"list":[{"key":"nginx","id":1, "nv":2}]}`), + }, + { + // extract with null field value + object: `{"list":[{"key":"nginx","id":1,"nv":2}]}`, + set: _NS( + _P("list", _KBF("key", "nginx", "id", 1), "value"), + ), + wantOutput: map[string]interface{}{ + "list": []interface{}{nil}, + }, + }, + { + // extract atomic element from associative list + object: `{"atomicElementList":[{"id":"test-123","data":"value","extra":"field"}]}`, + set: _NS( + _P("atomicElementList", _KBF("id", "test-123")), + ), + wantOutput: typed.YAMLObject(`{"atomicElementList":[{"id":"test-123","data":"value","extra":"field"}]}`), + }, + }, +}} + +func (tt extractWithKeysTestCase) test(t *testing.T) { + parser, err := typed.NewParser(tt.schema) + if err != nil { + t.Fatalf("failed to create schema: %v", err) + } + pt := parser.Type(tt.rootTypeName) + + for i, triplet := range tt.triplets { + triplet := triplet + t.Run(fmt.Sprintf("%v-valid-%v", tt.name, i), func(t *testing.T) { + t.Parallel() + // source typedValue obj + tv, err := pt.FromYAML(triplet.object) + if err != nil { + t.Fatal(err) + } + gotExtracted := tv.ExtractItems(triplet.set, typed.WithAppendKeyFields()) + + switch triplet.wantOutput.(type) { + case typed.YAMLObject: + wantOut, err := pt.FromYAML(triplet.wantOutput.(typed.YAMLObject)) + if err != nil { + t.Fatalf("unable to parser/validate removeOutput yaml: %v\n%v", err, triplet.wantOutput) + } + + if !value.Equals(gotExtracted.AsValue(), wantOut.AsValue()) { + t.Errorf("ExtractItems expected\n%v\nbut got\n%v\n", + value.ToString(wantOut.AsValue()), value.ToString(gotExtracted.AsValue()), + ) + } + default: + // The extracted result + wantOut := value.NewValueInterface(triplet.wantOutput) + if !value.Equals(gotExtracted.AsValue(), wantOut) { + t.Errorf("ExtractItems expected\n%v\nbut got\n%v\n", + value.ToString(wantOut), value.ToString(gotExtracted.AsValue()), + ) + } + } + }) + } +} + +// TestExtractWithKeys ensures that when you extract +// items from an object with the AppendKeyField option, +// the key fields are also included in the output. +func TestExtractWithKeys(t *testing.T) { + for _, tt := range extractWithKeysCases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tt.test(t) + }) + } +} diff --git a/typed/symdiff_test.go b/typed/symdiff_test.go index b5e69a4d..50e7d425 100644 --- a/typed/symdiff_test.go +++ b/typed/symdiff_test.go @@ -20,8 +20,8 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) type symdiffTestCase struct { @@ -596,6 +596,24 @@ var symdiffCases = []symdiffTestCase{{ removed: _NS(), modified: _NS(), added: _NS(_P("setStr", _V("c"))), + }, { + lhs: `{"setStr":["a"]}`, + rhs: `{"setStr":["a","b","b"]}`, + removed: _NS(), + modified: _NS(), + added: _NS(_P("setStr", _V("b"))), + }, { + lhs: `{"setStr":["a","b"]}`, + rhs: `{"setStr":["a","b","b"]}`, + removed: _NS(_P("setStr", _V("b"))), + modified: _NS(), + added: _NS(_P("setStr", _V("b"))), + }, { + lhs: `{"setStr":["b","b"]}`, + rhs: `{"setStr":["a","b","b"]}`, + removed: _NS(), + modified: _NS(), + added: _NS(_P("setStr", _V("a"))), }, { lhs: `{"setStr":["a","b","c"]}`, rhs: `{"setStr":[]}`, @@ -618,6 +636,28 @@ var symdiffCases = []symdiffTestCase{{ removed: _NS(_P("setNumeric", _V(3.14159))), modified: _NS(), added: _NS(_P("setNumeric", _V(3))), + }, { + lhs: `{"setStr":["a","b","b","c","a"]}`, + rhs: `{"setStr":[]}`, + removed: _NS( + _P("setStr", _V("a")), + _P("setStr", _V("b")), + _P("setStr", _V("c")), + ), + modified: _NS(), + added: _NS(), + }, { + lhs: `{"setBool":[true,true]}`, + rhs: `{"setBool":[false]}`, + removed: _NS(_P("setBool", _V(true))), + modified: _NS(), + added: _NS(_P("setBool", _V(false))), + }, { + lhs: `{"setNumeric":[1,2,2,3.14159,1]}`, + rhs: `{"setNumeric":[1,2,3]}`, + removed: _NS(_P("setNumeric", _V(1)), _P("setNumeric", _V(2)), _P("setNumeric", _V(3.14159))), + modified: _NS(), + added: _NS(_P("setNumeric", _V(1)), _P("setNumeric", _V(2)), _P("setNumeric", _V(3))), }}, }, { name: "associative list", @@ -743,6 +783,48 @@ var symdiffCases = []symdiffTestCase{{ removed: _NS(), modified: _NS(_P("atomicList")), added: _NS(), + }, { + lhs: `{"list":[{"key":"a","id":1,"nv":2},{"key":"b","id":1},{"key":"a","id":1,"bv":true}]}`, + rhs: `{"list":[{"key":"a","id":1},{"key":"a","id":2}]}`, + removed: _NS( + _P("list", _KBF("key", "b", "id", 1)), + _P("list", _KBF("key", "b", "id", 1), "key"), + _P("list", _KBF("key", "b", "id", 1), "id"), + _P("list", _KBF("key", "a", "id", 1)), + ), + modified: _NS(), + added: _NS( + _P("list", _KBF("key", "a", "id", 1)), + _P("list", _KBF("key", "a", "id", 1), "key"), + _P("list", _KBF("key", "a", "id", 1), "id"), + _P("list", _KBF("key", "a", "id", 2)), + _P("list", _KBF("key", "a", "id", 2), "key"), + _P("list", _KBF("key", "a", "id", 2), "id"), + ), + }, { + lhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true}]}`, + rhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true,"nv":1}]}`, + removed: _NS(), + modified: _NS(_P("list", _KBF("key", "a", "id", 1))), + added: _NS(), + }, { + lhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true}]}`, + rhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true}]}`, + removed: _NS(), + modified: _NS(), + added: _NS(), + }, { + lhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true}]}`, + rhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1,"bv":true},{"key":"a","id":1,"bv":true}]}`, + removed: _NS(), + modified: _NS(), + added: _NS(_P("list", _KBF("key", "b", "id", 1), "bv")), + }, { + lhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true}]}`, + rhs: `{"list":[{"key":"a","id":1},{"key":"b","id":1},{"key":"a","id":1,"bv":true}],"atomicList":["unrelated"]}`, + removed: _NS(), + modified: _NS(), + added: _NS(_P("atomicList")), }}, }} @@ -757,11 +839,11 @@ func (tt symdiffTestCase) test(t *testing.T) { t.Parallel() pt := parser.Type(tt.rootTypeName) - tvLHS, err := pt.FromYAML(quint.lhs) + tvLHS, err := pt.FromYAML(quint.lhs, typed.AllowDuplicates) if err != nil { t.Fatalf("failed to parse lhs: %v", err) } - tvRHS, err := pt.FromYAML(quint.rhs) + tvRHS, err := pt.FromYAML(quint.rhs, typed.AllowDuplicates) if err != nil { t.Fatalf("failed to parse rhs: %v", err) } diff --git a/typed/tofieldset.go b/typed/tofieldset.go index 015f1110..a52e342e 100644 --- a/typed/tofieldset.go +++ b/typed/tofieldset.go @@ -19,9 +19,9 @@ package typed import ( "sync" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) var tPool = sync.Pool{ @@ -94,9 +94,31 @@ func (v *toFieldSetWalker) doScalar(t *schema.Scalar) ValidationErrors { } func (v *toFieldSetWalker) visitListItems(t *schema.List, list value.List) (errs ValidationErrors) { + // Keeps track of the PEs we've seen + seen := fieldpath.MakePathElementSet(list.Length()) + // Keeps tracks of the PEs we've counted as duplicates + duplicates := fieldpath.MakePathElementSet(list.Length()) for i := 0; i < list.Length(); i++ { child := list.At(i) - pe, _ := listItemToPathElement(v.allocator, v.schema, t, i, child) + pe, _ := listItemToPathElement(v.allocator, v.schema, t, child) + if seen.Has(pe) { + if duplicates.Has(pe) { + // do nothing + } else { + v.set.Insert(append(v.path, pe)) + duplicates.Insert(pe) + } + } else { + seen.Insert(pe) + } + } + + for i := 0; i < list.Length(); i++ { + child := list.At(i) + pe, _ := listItemToPathElement(v.allocator, v.schema, t, child) + if duplicates.Has(pe) { + continue + } v2 := v.prepareDescent(pe, t.ElementType) v2.value = child errs = append(errs, v2.toFieldSet()...) diff --git a/typed/toset_test.go b/typed/toset_test.go index 554ba901..8b3e9da9 100644 --- a/typed/toset_test.go +++ b/typed/toset_test.go @@ -20,9 +20,9 @@ import ( "fmt" "testing" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/typed" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/typed" + "sigs.k8s.io/structured-merge-diff/v6/value" ) type objSetPair struct { @@ -155,7 +155,12 @@ var fieldsetCases = []fieldsetTestCase{{ _P("setStr", _V("b")), _P("setStr", _V("c")), )}, - {`{"setBool":[true,false]}`, _NS( + {`{"setStr":["a","b","c","a","b","c","c"]}`, _NS( + _P("setStr", _V("a")), + _P("setStr", _V("b")), + _P("setStr", _V("c")), + )}, + {`{"setBool":[true,false,true]}`, _NS( _P("setBool", _V(true)), _P("setBool", _V(false)), )}, @@ -244,6 +249,16 @@ var fieldsetCases = []fieldsetTestCase{{ _P("list", _KBF("key", "b", "id", 1), "key"), _P("list", _KBF("key", "b", "id", 1), "id"), )}, + {`{"list":[{"key":"a","id":1,"nv":2},{"key":"a","id":2,"nv":3},{"key":"b","id":1},{"key":"a","id":2,"bv":true}]}`, _NS( + _P("list", _KBF("key", "a", "id", 1)), + _P("list", _KBF("key", "a", "id", 1), "key"), + _P("list", _KBF("key", "a", "id", 1), "id"), + _P("list", _KBF("key", "a", "id", 1), "nv"), + _P("list", _KBF("key", "a", "id", 2)), + _P("list", _KBF("key", "b", "id", 1)), + _P("list", _KBF("key", "b", "id", 1), "key"), + _P("list", _KBF("key", "b", "id", 1), "id"), + )}, {`{"atomicList":["a","a","a"]}`, _NS(_P("atomicList"))}, }, }} @@ -257,9 +272,9 @@ func (tt fieldsetTestCase) test(t *testing.T) { v := v t.Run(fmt.Sprintf("%v-%v", tt.name, i), func(t *testing.T) { t.Parallel() - tv, err := parser.Type(tt.rootTypeName).FromYAML(v.object) + tv, err := parser.Type(tt.rootTypeName).FromYAML(v.object, typed.AllowDuplicates) if err != nil { - t.Errorf("failed to parse object: %v", err) + t.Fatalf("failed to parse object: %v", err) } fs, err := tv.ToFieldSet() if err != nil { diff --git a/typed/typed.go b/typed/typed.go index b66fccb2..0f9968fd 100644 --- a/typed/typed.go +++ b/typed/typed.go @@ -17,25 +17,46 @@ limitations under the License. package typed import ( - "fmt" - "strings" "sync" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) +// ValidationOptions is the list of all the options available when running the validation. +type ValidationOptions int + +const ( + // AllowDuplicates means that sets and associative lists can have duplicate similar items. + AllowDuplicates ValidationOptions = iota +) + +// extractItemsOptions is the options available when extracting items. +type extractItemsOptions struct { + appendKeyFields bool +} + +type ExtractItemsOption func(*extractItemsOptions) + +// WithAppendKeyFields configures ExtractItems to include key fields. +// It is exported for use in configuring ExtractItems. +func WithAppendKeyFields() ExtractItemsOption { + return func(opts *extractItemsOptions) { + opts.appendKeyFields = true + } +} + // AsTyped accepts a value and a type and returns a TypedValue. 'v' must have // type 'typeName' in the schema. An error is returned if the v doesn't conform // to the schema. -func AsTyped(v value.Value, s *schema.Schema, typeRef schema.TypeRef) (*TypedValue, error) { +func AsTyped(v value.Value, s *schema.Schema, typeRef schema.TypeRef, opts ...ValidationOptions) (*TypedValue, error) { tv := &TypedValue{ value: v, typeRef: typeRef, schema: s, } - if err := tv.Validate(); err != nil { + if err := tv.Validate(opts...); err != nil { return nil, err } return tv, nil @@ -45,6 +66,10 @@ func AsTyped(v value.Value, s *schema.Schema, typeRef schema.TypeRef) (*TypedVal // conforms to the schema, for cases where that has already been checked or // where you're going to call a method that validates as a side-effect (like // ToFieldSet). +// +// Deprecated: This function was initially created because validation +// was expensive. Now that this has been solved, objects should always +// be created as validated, using `AsTyped`. func AsTypedUnvalidated(v value.Value, s *schema.Schema, typeRef schema.TypeRef) *TypedValue { tv := &TypedValue{ value: v, @@ -77,8 +102,14 @@ func (tv TypedValue) Schema() *schema.Schema { } // Validate returns an error with a list of every spec violation. -func (tv TypedValue) Validate() error { +func (tv TypedValue) Validate(opts ...ValidationOptions) error { w := tv.walker() + for _, opt := range opts { + switch opt { + case AllowDuplicates: + w.allowDuplicates = true + } + } defer w.finished() if errs := w.validate(nil); len(errs) != 0 { return errs @@ -113,6 +144,10 @@ func (tv TypedValue) Merge(pso *TypedValue) (*TypedValue, error) { return merge(&tv, pso, ruleKeepRHS, nil) } +var cmpwPool = sync.Pool{ + New: func() interface{} { return &compareWalker{} }, +} + // Compare compares the two objects. See the comments on the `Comparison` // struct for details on the return value. // @@ -120,33 +155,44 @@ func (tv TypedValue) Merge(pso *TypedValue) (*TypedValue, error) { // match), or an error will be returned. Validation errors will be returned if // the objects don't conform to the schema. func (tv TypedValue) Compare(rhs *TypedValue) (c *Comparison, err error) { - c = &Comparison{ + lhs := tv + if lhs.schema != rhs.schema { + return nil, errorf("expected objects with types from the same schema") + } + if !lhs.typeRef.Equals(&rhs.typeRef) { + return nil, errorf("expected objects of the same type, but got %v and %v", lhs.typeRef, rhs.typeRef) + } + + cmpw := cmpwPool.Get().(*compareWalker) + defer func() { + cmpw.lhs = nil + cmpw.rhs = nil + cmpw.schema = nil + cmpw.typeRef = schema.TypeRef{} + cmpw.comparison = nil + cmpw.inLeaf = false + + cmpwPool.Put(cmpw) + }() + + cmpw.lhs = lhs.value + cmpw.rhs = rhs.value + cmpw.schema = lhs.schema + cmpw.typeRef = lhs.typeRef + cmpw.comparison = &Comparison{ Removed: fieldpath.NewSet(), Modified: fieldpath.NewSet(), Added: fieldpath.NewSet(), } - _, err = merge(&tv, rhs, func(w *mergingWalker) { - if w.lhs == nil { - c.Added.Insert(w.path) - } else if w.rhs == nil { - c.Removed.Insert(w.path) - } else if !value.Equals(w.rhs, w.lhs) { - // TODO: Equality is not sufficient for this. - // Need to implement equality check on the value type. - c.Modified.Insert(w.path) - } - }, func(w *mergingWalker) { - if w.lhs == nil { - c.Added.Insert(w.path) - } else if w.rhs == nil { - c.Removed.Insert(w.path) - } - }) - if err != nil { - return nil, err + if cmpw.allocator == nil { + cmpw.allocator = value.NewFreelistAllocator() } - return c, nil + errs := cmpw.compare(nil) + if len(errs) > 0 { + return nil, errs + } + return cmpw.comparison, nil } // RemoveItems removes each provided list or map item from the value. @@ -156,66 +202,39 @@ func (tv TypedValue) RemoveItems(items *fieldpath.Set) *TypedValue { } // ExtractItems returns a value with only the provided list or map items extracted from the value. -func (tv TypedValue) ExtractItems(items *fieldpath.Set) *TypedValue { - tv.value = removeItemsWithSchema(tv.value, items, tv.schema, tv.typeRef, true) - return &tv -} - -// NormalizeUnions takes the new object and normalizes the union: -// - If discriminator changed to non-nil, and a new field has been added -// that doesn't match, an error is returned, -// - If discriminator hasn't changed and two fields or more are set, an -// error is returned, -// - If discriminator changed to non-nil, all other fields but the -// discriminated one will be cleared, -// - Otherwise, If only one field is left, update discriminator to that value. -// -// Please note: union behavior isn't finalized yet and this is still experimental. -func (tv TypedValue) NormalizeUnions(new *TypedValue) (*TypedValue, error) { - var errs ValidationErrors - var normalizeFn = func(w *mergingWalker) { - if w.rhs != nil { - v := w.rhs.Unstructured() - w.out = &v - } - if err := normalizeUnions(w); err != nil { - errs = append(errs, errorf(err.Error())...) - } +func (tv TypedValue) ExtractItems(items *fieldpath.Set, opts ...ExtractItemsOption) *TypedValue { + options := &extractItemsOptions{} + for _, opt := range opts { + opt(options) } - out, mergeErrs := merge(&tv, new, func(w *mergingWalker) {}, normalizeFn) - if mergeErrs != nil { - errs = append(errs, mergeErrs.(ValidationErrors)...) - } - if len(errs) > 0 { - return nil, errs - } - return out, nil -} - -// NormalizeUnionsApply specifically normalize unions on apply. It -// validates that the applied union is correct (there should be no -// ambiguity there), and clear the fields according to the sent intent. -// -// Please note: union behavior isn't finalized yet and this is still experimental. -func (tv TypedValue) NormalizeUnionsApply(new *TypedValue) (*TypedValue, error) { - var errs ValidationErrors - var normalizeFn = func(w *mergingWalker) { - if w.rhs != nil { - v := w.rhs.Unstructured() - w.out = &v + if options.appendKeyFields { + tvPathSet, err := tv.ToFieldSet() + if err == nil { + keyFieldPathSet := fieldpath.NewSet() + items.Iterate(func(path fieldpath.Path) { + if !tvPathSet.Has(path) { + return + } + for i, pe := range path { + if pe.Key == nil { + continue + } + for _, keyField := range *pe.Key { + keyName := keyField.Name + // Create a new slice with the same elements as path[:i+1], but set its capacity to len(path[:i+1]). + // This ensures that appending to keyFieldPath creates a new underlying array, avoiding accidental + // modification of the original slice (path). + keyFieldPath := append(path[:i+1:i+1], fieldpath.PathElement{FieldName: &keyName}) + keyFieldPathSet.Insert(keyFieldPath) + } + } + }) + items = items.Union(keyFieldPathSet) } - if err := normalizeUnionsApply(w); err != nil { - errs = append(errs, errorf(err.Error())...) - } - } - out, mergeErrs := merge(&tv, new, func(w *mergingWalker) {}, normalizeFn) - if mergeErrs != nil { - errs = append(errs, mergeErrs.(ValidationErrors)...) } - if len(errs) > 0 { - return nil, errs - } - return out, nil + + tv.value = removeItemsWithSchema(tv.value, items, tv.schema, tv.typeRef, true) + return &tv } func (tv TypedValue) Empty() *TypedValue { @@ -273,50 +292,3 @@ func merge(lhs, rhs *TypedValue, rule, postRule mergeRule) (*TypedValue, error) } return out, nil } - -// Comparison is the return value of a TypedValue.Compare() operation. -// -// No field will appear in more than one of the three fieldsets. If all of the -// fieldsets are empty, then the objects must have been equal. -type Comparison struct { - // Removed contains any fields removed by rhs (the right-hand-side - // object in the comparison). - Removed *fieldpath.Set - // Modified contains fields present in both objects but different. - Modified *fieldpath.Set - // Added contains any fields added by rhs. - Added *fieldpath.Set -} - -// IsSame returns true if the comparison returned no changes (the two -// compared objects are similar). -func (c *Comparison) IsSame() bool { - return c.Removed.Empty() && c.Modified.Empty() && c.Added.Empty() -} - -// String returns a human readable version of the comparison. -func (c *Comparison) String() string { - bld := strings.Builder{} - if !c.Modified.Empty() { - bld.WriteString(fmt.Sprintf("- Modified Fields:\n%v\n", c.Modified)) - } - if !c.Added.Empty() { - bld.WriteString(fmt.Sprintf("- Added Fields:\n%v\n", c.Added)) - } - if !c.Removed.Empty() { - bld.WriteString(fmt.Sprintf("- Removed Fields:\n%v\n", c.Removed)) - } - return bld.String() -} - -// ExcludeFields fields from the compare recursively removes the fields -// from the entire comparison -func (c *Comparison) ExcludeFields(fields *fieldpath.Set) *Comparison { - if fields == nil || fields.Empty() { - return c - } - c.Removed = c.Removed.RecursiveDifference(fields) - c.Modified = c.Modified.RecursiveDifference(fields) - c.Added = c.Added.RecursiveDifference(fields) - return c -} diff --git a/typed/union.go b/typed/union.go deleted file mode 100644 index ee04bf22..00000000 --- a/typed/union.go +++ /dev/null @@ -1,276 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package typed - -import ( - "fmt" - "strings" - - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" -) - -func normalizeUnions(w *mergingWalker) error { - atom, found := w.schema.Resolve(w.typeRef) - if !found { - panic(fmt.Sprintf("Unable to resolve schema in normalize union: %v/%v", w.schema, w.typeRef)) - } - // Unions can only be in structures, and the struct must not have been removed - if atom.Map == nil || w.out == nil { - return nil - } - - var old value.Map - if w.lhs != nil && !w.lhs.IsNull() { - old = w.lhs.AsMap() - } - for _, union := range atom.Map.Unions { - if err := newUnion(&union).Normalize(old, w.rhs.AsMap(), value.NewValueInterface(*w.out).AsMap()); err != nil { - return err - } - } - return nil -} - -func normalizeUnionsApply(w *mergingWalker) error { - atom, found := w.schema.Resolve(w.typeRef) - if !found { - panic(fmt.Sprintf("Unable to resolve schema in normalize union: %v/%v", w.schema, w.typeRef)) - } - // Unions can only be in structures, and the struct must not have been removed - if atom.Map == nil || w.out == nil { - return nil - } - - var old value.Map - if w.lhs != nil && !w.lhs.IsNull() { - old = w.lhs.AsMap() - } - - for _, union := range atom.Map.Unions { - out := value.NewValueInterface(*w.out) - if err := newUnion(&union).NormalizeApply(old, w.rhs.AsMap(), out.AsMap()); err != nil { - return err - } - *w.out = out.Unstructured() - } - return nil -} - -type discriminated string -type field string - -type discriminatedNames struct { - f2d map[field]discriminated - d2f map[discriminated]field -} - -func newDiscriminatedName(f2d map[field]discriminated) discriminatedNames { - d2f := map[discriminated]field{} - for key, value := range f2d { - d2f[value] = key - } - return discriminatedNames{ - f2d: f2d, - d2f: d2f, - } -} - -func (dn discriminatedNames) toField(d discriminated) field { - if f, ok := dn.d2f[d]; ok { - return f - } - return field(d) -} - -func (dn discriminatedNames) toDiscriminated(f field) discriminated { - if d, ok := dn.f2d[f]; ok { - return d - } - return discriminated(f) -} - -type discriminator struct { - name string -} - -func (d *discriminator) Set(m value.Map, v discriminated) { - if d == nil { - return - } - m.Set(d.name, value.NewValueInterface(string(v))) -} - -func (d *discriminator) Get(m value.Map) discriminated { - if d == nil || m == nil { - return "" - } - val, ok := m.Get(d.name) - if !ok { - return "" - } - if !val.IsString() { - return "" - } - return discriminated(val.AsString()) -} - -type fieldsSet map[field]struct{} - -// newFieldsSet returns a map of the fields that are part of the union and are set -// in the given map. -func newFieldsSet(m value.Map, fields []field) fieldsSet { - if m == nil { - return nil - } - set := fieldsSet{} - for _, f := range fields { - if subField, ok := m.Get(string(f)); ok && !subField.IsNull() { - set.Add(f) - } - } - return set -} - -func (fs fieldsSet) Add(f field) { - if fs == nil { - fs = map[field]struct{}{} - } - fs[f] = struct{}{} -} - -func (fs fieldsSet) One() *field { - for f := range fs { - return &f - } - return nil -} - -func (fs fieldsSet) Has(f field) bool { - _, ok := fs[f] - return ok -} - -func (fs fieldsSet) List() []field { - fields := []field{} - for f := range fs { - fields = append(fields, f) - } - return fields -} - -func (fs fieldsSet) Difference(o fieldsSet) fieldsSet { - n := fieldsSet{} - for f := range fs { - if !o.Has(f) { - n.Add(f) - } - } - return n -} - -func (fs fieldsSet) String() string { - s := []string{} - for k := range fs { - s = append(s, string(k)) - } - return strings.Join(s, ", ") -} - -type union struct { - deduceInvalidDiscriminator bool - d *discriminator - dn discriminatedNames - f []field -} - -func newUnion(su *schema.Union) *union { - u := &union{} - if su.Discriminator != nil { - u.d = &discriminator{name: *su.Discriminator} - } - f2d := map[field]discriminated{} - for _, f := range su.Fields { - u.f = append(u.f, field(f.FieldName)) - f2d[field(f.FieldName)] = discriminated(f.DiscriminatorValue) - } - u.dn = newDiscriminatedName(f2d) - u.deduceInvalidDiscriminator = su.DeduceInvalidDiscriminator - return u -} - -// clear removes all the fields in map that are part of the union, but -// the one we decided to keep. -func (u *union) clear(m value.Map, f field) { - for _, fieldName := range u.f { - if field(fieldName) != f { - m.Delete(string(fieldName)) - } - } -} - -func (u *union) Normalize(old, new, out value.Map) error { - os := newFieldsSet(old, u.f) - ns := newFieldsSet(new, u.f) - diff := ns.Difference(os) - - if u.d.Get(old) != u.d.Get(new) && u.d.Get(new) != "" { - if len(diff) == 1 && u.d.Get(new) != u.dn.toDiscriminated(*diff.One()) { - return fmt.Errorf("discriminator (%v) and field changed (%v) don't match", u.d.Get(new), diff.One()) - } - if len(diff) > 1 { - return fmt.Errorf("multiple new fields added: %v", diff) - } - u.clear(out, u.dn.toField(u.d.Get(new))) - return nil - } - - if len(ns) > 1 { - return fmt.Errorf("multiple fields set without discriminator change: %v", ns) - } - - // Set discriminiator if it needs to be deduced. - if u.deduceInvalidDiscriminator && len(ns) == 1 { - u.d.Set(out, u.dn.toDiscriminated(*ns.One())) - } - - return nil -} - -func (u *union) NormalizeApply(applied, merged, out value.Map) error { - as := newFieldsSet(applied, u.f) - if len(as) > 1 { - return fmt.Errorf("more than one field of union applied: %v", as) - } - if len(as) == 0 { - // None is set, just leave. - return nil - } - // We have exactly one, discriminiator must match if set - if u.d.Get(applied) != "" && u.d.Get(applied) != u.dn.toDiscriminated(*as.One()) { - return fmt.Errorf("applied discriminator (%v) doesn't match applied field (%v)", u.d.Get(applied), *as.One()) - } - - // Update discriminiator if needed - if u.deduceInvalidDiscriminator { - u.d.Set(out, u.dn.toDiscriminated(*as.One())) - } - // Clear others fields. - u.clear(out, *as.One()) - - return nil -} diff --git a/typed/union_test.go b/typed/union_test.go deleted file mode 100644 index 09f7335d..00000000 --- a/typed/union_test.go +++ /dev/null @@ -1,326 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package typed_test - -import ( - "testing" - - "sigs.k8s.io/structured-merge-diff/v5/typed" -) - -var unionParser = func() typed.ParseableType { - parser, err := typed.NewParser(`types: -- name: union - map: - fields: - - name: discriminator - type: - scalar: string - - name: one - type: - scalar: numeric - - name: two - type: - scalar: numeric - - name: three - type: - scalar: numeric - - name: letter - type: - scalar: string - - name: a - type: - scalar: numeric - - name: b - type: - scalar: numeric - unions: - - discriminator: discriminator - deduceInvalidDiscriminator: true - fields: - - fieldName: one - discriminatorValue: One - - fieldName: two - discriminatorValue: TWO - - fieldName: three - discriminatorValue: three - - discriminator: letter - fields: - - fieldName: a - discriminatorValue: A - - fieldName: b - discriminatorValue: b`) - if err != nil { - panic(err) - } - return parser.Type("union") -}() - -func TestNormalizeUnions(t *testing.T) { - tests := []struct { - name string - old typed.YAMLObject - new typed.YAMLObject - out typed.YAMLObject - }{ - { - name: "nothing changed, add discriminator", - old: `{"one": 1}`, - new: `{"one": 1}`, - out: `{"one": 1, "discriminator": "One"}`, - }, - { - name: "nothing changed, non-deduced", - old: `{"a": 1}`, - new: `{"a": 1}`, - out: `{"a": 1}`, - }, - { - name: "proper union update, setting discriminator", - old: `{"one": 1}`, - new: `{"two": 1}`, - out: `{"two": 1, "discriminator": "TWO"}`, - }, - { - name: "proper union update, non-deduced", - old: `{"a": 1}`, - new: `{"b": 1}`, - out: `{"b": 1}`, - }, - { - name: "proper union update from not-set, setting discriminator", - old: `{}`, - new: `{"two": 1}`, - out: `{"two": 1, "discriminator": "TWO"}`, - }, - { - name: "proper union update from not-set, non-deduced", - old: `{}`, - new: `{"b": 1}`, - out: `{"b": 1}`, - }, - { - name: "remove union, with discriminator", - old: `{"one": 1}`, - new: `{}`, - out: `{}`, - }, - { - name: "remove union and discriminator", - old: `{"one": 1, "discriminator": "One"}`, - new: `{}`, - out: `{}`, - }, - { - name: "remove union, not discriminator", - old: `{"one": 1, "discriminator": "One"}`, - new: `{"discriminator": "One"}`, - out: `{"discriminator": "One"}`, - }, - { - name: "remove union, not discriminator, non-deduced", - old: `{"a": 1, "letter": "A"}`, - new: `{"letter": "A"}`, - out: `{"letter": "A"}`, - }, - { - name: "change discriminator, nothing else", - old: `{"discriminator": "One"}`, - new: `{"discriminator": "random"}`, - out: `{"discriminator": "random"}`, - }, - { - name: "change discriminator, nothing else, non-deduced", - old: `{"letter": "A"}`, - new: `{"letter": "b"}`, - out: `{"letter": "b"}`, - }, - { - name: "change discriminator, nothing else, it drops other field", - old: `{"discriminator": "One", "one": 1}`, - new: `{"discriminator": "random", "one": 1}`, - out: `{"discriminator": "random"}`, - }, - { - name: "change discriminator, nothing else, it drops other field, non-deduced", - old: `{"letter": "A", "a": 1}`, - new: `{"letter": "b", "a": 1}`, - out: `{"letter": "b"}`, - }, - { - name: "remove discriminator, nothing else", - old: `{"discriminator": "One", "one": 1}`, - new: `{"one": 1}`, - out: `{"one": 1, "discriminator": "One"}`, - }, - { - name: "remove discriminator, nothing else, non-deduced", - old: `{"letter": "A", "a": 1}`, - new: `{"a": 1}`, - out: `{"a": 1}`, - }, - { - name: "remove discriminator, add new field", - old: `{"discriminator": "One", "one": 1}`, - new: `{"two": 1}`, - out: `{"two": 1, "discriminator": "TWO"}`, - }, - { - name: "remove discriminator, add new field, non-deduced", - old: `{"letter": "A", "a": 1}`, - new: `{"b": 1}`, - out: `{"b": 1}`, - }, - { - name: "both fields removed", - old: `{"one": 1, "two": 1}`, - new: `{}`, - out: `{}`, - }, - { - name: "one field removed", - old: `{"one": 1, "two": 1}`, - new: `{"one": 1}`, - out: `{"one": 1, "discriminator": "One"}`, - }, - { - name: "one field removed, non-deduced", - old: `{"a": 1, "b": 1}`, - new: `{"a": 1}`, - out: `{"a": 1}`, - }, - // These use-cases shouldn't happen: - { - name: "one field removed, discriminator unchanged", - old: `{"one": 1, "two": 1, "discriminator": "TWO"}`, - new: `{"one": 1, "discriminator": "TWO"}`, - out: `{"one": 1, "discriminator": "One"}`, - }, - { - name: "one field removed, discriminator unchanged, non-deduced", - old: `{"a": 1, "b": 1, "letter": "b"}`, - new: `{"a": 1, "letter": "b"}`, - out: `{"a": 1, "letter": "b"}`, - }, - { - name: "one field removed, discriminator added", - old: `{"two": 2, "one": 1}`, - new: `{"one": 1, "discriminator": "TWO"}`, - out: `{"discriminator": "TWO"}`, - }, - { - name: "one field removed, discriminator added, non-deduced", - old: `{"b": 2, "a": 1}`, - new: `{"a": 1, "letter": "b"}`, - out: `{"letter": "b"}`, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - old, err := unionParser.FromYAML(test.old) - if err != nil { - t.Fatalf("Failed to parse old object: %v", err) - } - new, err := unionParser.FromYAML(test.new) - if err != nil { - t.Fatalf("failed to parse new object: %v", err) - } - out, err := unionParser.FromYAML(test.out) - if err != nil { - t.Fatalf("failed to parse out object: %v", err) - } - got, err := old.NormalizeUnions(new) - if err != nil { - t.Fatalf("failed to normalize unions: %v", err) - } - comparison, err := out.Compare(got) - if err != nil { - t.Fatalf("failed to compare result and expected: %v", err) - } - if !comparison.IsSame() { - t.Errorf("Result is different from expected:\n%v", comparison) - } - }) - } -} - -func TestNormalizeUnionError(t *testing.T) { - tests := []struct { - name string - old typed.YAMLObject - new typed.YAMLObject - }{ - { - name: "dumb client update, no discriminator", - old: `{"one": 1}`, - new: `{"one": 2, "two": 1}`, - }, - { - name: "new object has three of same union set", - old: `{"one": 1}`, - new: `{"one": 2, "two": 1, "three": 3}`, - }, - { - name: "dumb client doesn't update discriminator", - old: `{"one": 1, "discriminator": "One"}`, - new: `{"one": 2, "two": 1, "discriminator": "One"}`, - }, - { - name: "client sends new field that and discriminator change", - old: `{}`, - new: `{"one": 1, "discriminator": "Two"}`, - }, - { - name: "client sends new fields that don't match discriminator change", - old: `{}`, - new: `{"one": 1, "two": 1, "discriminator": "One"}`, - }, - { - name: "old object has two of same union set", - old: `{"one": 1, "two": 2}`, - new: `{"one": 2, "two": 1}`, - }, - { - name: "old object has two of same union, but we add third", - old: `{"discriminator": "One", "one": 1, "two": 1}`, - new: `{"discriminator": "One", "one": 1, "two": 1, "three": 1}`, - }, - { - name: "one field removed, 2 left, discriminator unchanged", - old: `{"one": 1, "two": 1, "three": 1, "discriminator": "TWO"}`, - new: `{"one": 1, "two": 1, "discriminator": "TWO"}`, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - old, err := unionParser.FromYAML(test.old) - if err != nil { - t.Fatalf("Failed to parse old object: %v", err) - } - new, err := unionParser.FromYAML(test.new) - if err != nil { - t.Fatalf("failed to parse new object: %v", err) - } - _, err = old.NormalizeUnions(new) - if err == nil { - t.Fatal("Normalization should have failed, but hasn't.") - } - }) - } -} diff --git a/typed/validate.go b/typed/validate.go index b7332fc7..3371f87b 100644 --- a/typed/validate.go +++ b/typed/validate.go @@ -19,9 +19,9 @@ package typed import ( "sync" - "sigs.k8s.io/structured-merge-diff/v5/fieldpath" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/value" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/value" ) var vPool = sync.Pool{ @@ -33,6 +33,7 @@ func (tv TypedValue) walker() *validatingObjectWalker { v.value = tv.value v.schema = tv.schema v.typeRef = tv.typeRef + v.allowDuplicates = false if v.allocator == nil { v.allocator = value.NewFreelistAllocator() } @@ -49,6 +50,9 @@ type validatingObjectWalker struct { value value.Value schema *schema.Schema typeRef schema.TypeRef + // If set to true, duplicates will be allowed in + // associativeLists/sets. + allowDuplicates bool // Allocate only as many walkers as needed for the depth by storing them here. spareWalkers *[]*validatingObjectWalker @@ -102,6 +106,12 @@ func validateScalar(t *schema.Scalar, v value.Value, prefix string) (errs Valida if !v.IsBool() { return errorf("%vexpected boolean, got %v", prefix, v) } + case schema.Untyped: + if !v.IsFloat() && !v.IsInt() && !v.IsString() && !v.IsBool() { + return errorf("%vexpected any scalar, got %v", prefix, v) + } + default: + return errorf("%vunexpected scalar type in schema: %v", prefix, *t) } return nil } @@ -123,7 +133,7 @@ func (v *validatingObjectWalker) visitListItems(t *schema.List, list value.List) pe.Index = &i } else { var err error - pe, err = listItemToPathElement(v.allocator, v.schema, t, i, child) + pe, err = listItemToPathElement(v.allocator, v.schema, t, child) if err != nil { errs = append(errs, errorf("element %v: %v", i, err.Error())...) // If we can't construct the path element, we can't @@ -131,7 +141,7 @@ func (v *validatingObjectWalker) visitListItems(t *schema.List, list value.List) // this element. return } - if observedKeys.Has(pe) { + if observedKeys.Has(pe) && !v.allowDuplicates { errs = append(errs, errorf("duplicate entries for key %v", pe.String())...) } observedKeys.Insert(pe) @@ -147,7 +157,7 @@ func (v *validatingObjectWalker) visitListItems(t *schema.List, list value.List) func (v *validatingObjectWalker) doList(t *schema.List) (errs ValidationErrors) { list, err := listValue(v.allocator, v.value) if err != nil { - return errorf(err.Error()) + return errorf("%v", err) } if list == nil { @@ -183,7 +193,7 @@ func (v *validatingObjectWalker) visitMapItems(t *schema.Map, m value.Map) (errs func (v *validatingObjectWalker) doMap(t *schema.Map) (errs ValidationErrors) { m, err := mapValue(v.allocator, v.value) if err != nil { - return errorf(err.Error()) + return errorf("%v", err) } if m == nil { return nil diff --git a/typed/validate_test.go b/typed/validate_test.go index ac54e4c8..92ba1416 100644 --- a/typed/validate_test.go +++ b/typed/validate_test.go @@ -21,8 +21,8 @@ import ( "strings" "testing" - "sigs.k8s.io/structured-merge-diff/v5/schema" - "sigs.k8s.io/structured-merge-diff/v5/typed" + "sigs.k8s.io/structured-merge-diff/v6/schema" + "sigs.k8s.io/structured-merge-diff/v6/typed" ) type validationTestCase struct { @@ -31,6 +31,8 @@ type validationTestCase struct { schema typed.YAMLObject validObjects []typed.YAMLObject invalidObjects []typed.YAMLObject + // duplicatesObjects are valid with AllowDuplicates validation, invalid otherwise. + duplicatesObjects []typed.YAMLObject } var validationCases = []validationTestCase{{ @@ -63,7 +65,6 @@ var validationCases = []validationTestCase{{ `{"key":"foo","value":null}`, `{"key":"foo"}`, `{"key":"foo","value":true}`, - `{"key":"foo","value":true}`, `{"key":null}`, }, invalidObjects: []typed.YAMLObject{ @@ -136,28 +137,27 @@ var validationCases = []validationTestCase{{ `{"bool":"aoeu"}`, `{"bool":{"a":1}}`, `{"bool":["foo"]}`, - `{"setStr":["a","a"]}`, - `{"setBool":[true,false,true]}`, - `{"setNumeric":[1,2,3,3.14159,1]}`, `{"setStr":[1]}`, `{"setStr":[true]}`, `{"setStr":[1.5]}`, `{"setStr":[null]}`, `{"setStr":[{}]}`, `{"setStr":[[]]}`, - `{"setBool":[true,false,true]}`, `{"setBool":[1]}`, `{"setBool":[1.5]}`, `{"setBool":[null]}`, `{"setBool":[{}]}`, `{"setBool":[[]]}`, `{"setBool":["a"]}`, - `{"setNumeric":[1,2,3,3.14159,1]}`, `{"setNumeric":[null]}`, `{"setNumeric":[true]}`, `{"setNumeric":["a"]}`, `{"setNumeric":[[]]}`, `{"setNumeric":[{}]}`, + }, duplicatesObjects: []typed.YAMLObject{ + `{"setStr":["a","a"]}`, + `{"setBool":[true,false,true]}`, + `{"setNumeric":[1,2,3,3.14159,1]}`, }, }, { name: "associative list", @@ -213,6 +213,7 @@ var validationCases = []validationTestCase{{ `{"list":[{"key":"a","id":1,"value":{"a":"a"}}]}`, `{"list":[{"key":"a","id":1},{"key":"a","id":2},{"key":"b","id":1}]}`, `{"atomicList":["a","a","a"]}`, + `{"list":[{"key":"x","value":{"a":"a"},"bv":true,"nv":3.14}]}`, }, invalidObjects: []typed.YAMLObject{ `{"key":true,"value":1}`, @@ -232,9 +233,10 @@ var validationCases = []validationTestCase{{ `{"list":[{}]}`, `{"list":[{"value":{"a":"a"},"bv":true,"nv":3.14}]}`, `{"list":[{"key":"a","id":1,"value":{"a":1}}]}`, - `{"list":[{"key":"a","id":1},{"key":"a","id":1}]}`, `{"list":[{"key":"a","id":1,"value":{"a":"a"},"bv":"true","nv":3.14}]}`, `{"list":[{"key":"a","id":1,"value":{"a":"a"},"bv":true,"nv":false}]}`, + }, duplicatesObjects: []typed.YAMLObject{ + `{"list":[{"key":"a","id":1},{"key":"a","id":1}]}`, }, }} @@ -262,13 +264,30 @@ func (tt validationTestCase) test(t *testing.T) { t.Parallel() _, err := pt.FromYAML(iv) if err == nil { - t.Errorf("Object should fail: %v\n%v", err, iv) + t.Fatalf("Object should fail:\n%v", iv) } if strings.Contains(err.Error(), "invalid atom") { t.Errorf("Error should be useful, but got: %v\n%v", err, iv) } }) } + for i, iv := range tt.duplicatesObjects { + iv := iv + t.Run(fmt.Sprintf("%v-duplicates-%v", tt.name, i), func(t *testing.T) { + t.Parallel() + _, err := pt.FromYAML(iv) + if err == nil { + t.Fatalf("Object should fail:\n%v", iv) + } + if strings.Contains(err.Error(), "invalid atom") { + t.Errorf("Error should be useful, but got: %v\n%v", err, iv) + } + _, err = pt.FromYAML(iv, typed.AllowDuplicates) + if err != nil { + t.Errorf("failed to parse/validate yaml: %v\n%v", err, iv) + } + }) + } } func TestSchemaValidation(t *testing.T) { diff --git a/value/equals_test.go b/value/equals_test.go new file mode 100644 index 00000000..ecfc7d03 --- /dev/null +++ b/value/equals_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value_test + +import ( + "io/ioutil" + "path/filepath" + "testing" + + yaml "go.yaml.in/yaml/v2" + "sigs.k8s.io/structured-merge-diff/v6/value" +) + +func testdata(file string) string { + return filepath.Join("..", "internal", "testdata", file) +} + +func read(file string) []byte { + s, err := ioutil.ReadFile(file) + if err != nil { + panic(err) + } + return s +} + +func BenchmarkEquals(b *testing.B) { + benches := []struct { + filename string + }{ + { + filename: "pod.yaml", + }, + { + filename: "endpoints.yaml", + }, + { + filename: "list.yaml", + }, + { + filename: "node.yaml", + }, + { + filename: "prometheus-crd.yaml", + }, + } + + for _, bench := range benches { + b.Run(bench.filename, func(b *testing.B) { + var obj interface{} + err := yaml.Unmarshal(read(testdata(bench.filename)), &obj) + if err != nil { + b.Fatalf("Failed to unmarshal object: %v", err) + } + v := value.NewValueInterface(obj) + b.Run("Equals", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if !value.Equals(v, v) { + b.Fatalf("Object should be equal") + } + } + }) + b.Run("EqualsUsingFreelist", func(b *testing.B) { + b.ReportAllocs() + a := value.NewFreelistAllocator() + for i := 0; i < b.N; i++ { + if !value.EqualsUsing(a, v, v) { + b.Fatalf("Object should be equal") + } + } + }) + }) + } +} diff --git a/value/fields.go b/value/fields.go index be3c6724..042b0487 100644 --- a/value/fields.go +++ b/value/fields.go @@ -31,6 +31,14 @@ type Field struct { // have a different name. type FieldList []Field +// Copy returns a copy of the FieldList. +// Values are not copied. +func (f FieldList) Copy() FieldList { + c := make(FieldList, len(f)) + copy(c, f) + return c +} + // Sort sorts the field list by Name. func (f FieldList) Sort() { if len(f) < 2 { diff --git a/value/jsontagutil.go b/value/jsontagutil.go index d4adb8fc..3aadceb2 100644 --- a/value/jsontagutil.go +++ b/value/jsontagutil.go @@ -22,22 +22,77 @@ import ( "strings" ) +type isZeroer interface { + IsZero() bool +} + +var isZeroerType = reflect.TypeOf((*isZeroer)(nil)).Elem() + +func reflectIsZero(dv reflect.Value) bool { + return dv.IsZero() +} + +// OmitZeroFunc returns a function for a type for a given struct field +// which determines if the value for that field is a zero value, matching +// how the stdlib JSON implementation. +func OmitZeroFunc(t reflect.Type) func(reflect.Value) bool { + // Provide a function that uses a type's IsZero method. + // This matches the go 1.24 custom IsZero() implementation matching + switch { + case t.Kind() == reflect.Interface && t.Implements(isZeroerType): + return func(v reflect.Value) bool { + // Avoid panics calling IsZero on a nil interface or + // non-nil interface with nil pointer. + return safeIsNil(v) || + (v.Elem().Kind() == reflect.Pointer && v.Elem().IsNil()) || + v.Interface().(isZeroer).IsZero() + } + case t.Kind() == reflect.Pointer && t.Implements(isZeroerType): + return func(v reflect.Value) bool { + // Avoid panics calling IsZero on nil pointer. + return safeIsNil(v) || v.Interface().(isZeroer).IsZero() + } + case t.Implements(isZeroerType): + return func(v reflect.Value) bool { + return v.Interface().(isZeroer).IsZero() + } + case reflect.PointerTo(t).Implements(isZeroerType): + return func(v reflect.Value) bool { + if !v.CanAddr() { + // Temporarily box v so we can take the address. + v2 := reflect.New(v.Type()).Elem() + v2.Set(v) + v = v2 + } + return v.Addr().Interface().(isZeroer).IsZero() + } + default: + // default to the reflect.IsZero implementation + return reflectIsZero + } +} + // TODO: This implements the same functionality as https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go#L236 // but is based on the highly efficient approach from https://golang.org/src/encoding/json/encode.go -func lookupJsonTags(f reflect.StructField) (name string, omit bool, inline bool, omitempty bool) { +func lookupJsonTags(f reflect.StructField) (name string, omit bool, inline bool, omitempty bool, omitzero func(reflect.Value) bool) { tag := f.Tag.Get("json") if tag == "-" { - return "", true, false, false + return "", true, false, false, nil } name, opts := parseTag(tag) if name == "" { name = f.Name } - return name, false, opts.Contains("inline"), opts.Contains("omitempty") + + if opts.Contains("omitzero") { + omitzero = OmitZeroFunc(f.Type) + } + + return name, false, opts.Contains("inline"), opts.Contains("omitempty"), omitzero } -func isZero(v reflect.Value) bool { +func isEmpty(v reflect.Value) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: return v.Len() == 0 diff --git a/value/mapreflect.go b/value/mapreflect.go index dc8b8c72..c38402b9 100644 --- a/value/mapreflect.go +++ b/value/mapreflect.go @@ -136,7 +136,7 @@ func (r mapReflect) EqualsUsing(a Allocator, m Map) bool { if !ok { return false } - return Equals(vr.mustReuse(lhsVal, entry, nil, nil), value) + return EqualsUsing(a, vr.mustReuse(lhsVal, entry, nil, nil), value) }) } diff --git a/value/mapunstructured.go b/value/mapunstructured.go index d8e20862..c3ae00b1 100644 --- a/value/mapunstructured.go +++ b/value/mapunstructured.go @@ -88,12 +88,12 @@ func (m mapUnstructuredInterface) EqualsUsing(a Allocator, other Map) bool { } vv := a.allocValueUnstructured() defer a.Free(vv) - return other.Iterate(func(key string, value Value) bool { + return other.IterateUsing(a, func(key string, value Value) bool { lhsVal, ok := m[key] if !ok { return false } - return Equals(vv.reuse(lhsVal), value) + return EqualsUsing(a, vv.reuse(lhsVal), value) }) } @@ -168,12 +168,12 @@ func (m mapUnstructuredString) EqualsUsing(a Allocator, other Map) bool { } vv := a.allocValueUnstructured() defer a.Free(vv) - return other.Iterate(func(key string, value Value) bool { + return other.IterateUsing(a, func(key string, value Value) bool { lhsVal, ok := m[key] if !ok { return false } - return Equals(vv.reuse(lhsVal), value) + return EqualsUsing(a, vv.reuse(lhsVal), value) }) } diff --git a/value/reflectcache.go b/value/reflectcache.go index a5a467c0..3b4a402e 100644 --- a/value/reflectcache.go +++ b/value/reflectcache.go @@ -19,7 +19,9 @@ package value import ( "bytes" "encoding/json" + "errors" "fmt" + "io" "reflect" "sort" "sync" @@ -57,6 +59,8 @@ type FieldCacheEntry struct { JsonName string // isOmitEmpty is true if the field has the json 'omitempty' tag. isOmitEmpty bool + // omitzero is set if the field has the json 'omitzero' tag. + omitzero func(reflect.Value) bool // fieldPath is a list of field indices (see FieldByIndex) to lookup the value of // a field in a reflect.Value struct. The field indices in the list form a path used // to traverse through intermediary 'inline' fields. @@ -67,7 +71,13 @@ type FieldCacheEntry struct { } func (f *FieldCacheEntry) CanOmit(fieldVal reflect.Value) bool { - return f.isOmitEmpty && (safeIsNil(fieldVal) || isZero(fieldVal)) + if f.isOmitEmpty && (safeIsNil(fieldVal) || isEmpty(fieldVal)) { + return true + } + if f.omitzero != nil && f.omitzero(fieldVal) { + return true + } + return false } // GetFrom returns the field identified by this FieldCacheEntry from the provided struct. @@ -145,7 +155,7 @@ func typeReflectEntryOf(cm reflectCacheMap, t reflect.Type, updates reflectCache func buildStructCacheEntry(t reflect.Type, infos map[string]*FieldCacheEntry, fieldPath [][]int) { for i := 0; i < t.NumField(); i++ { field := t.Field(i) - jsonName, omit, isInline, isOmitempty := lookupJsonTags(field) + jsonName, omit, isInline, isOmitempty, omitzero := lookupJsonTags(field) if omit { continue } @@ -154,10 +164,12 @@ func buildStructCacheEntry(t reflect.Type, infos map[string]*FieldCacheEntry, fi if field.Type.Kind() == reflect.Ptr { e = field.Type.Elem() } - buildStructCacheEntry(e, infos, append(fieldPath, field.Index)) + if e.Kind() == reflect.Struct { + buildStructCacheEntry(e, infos, append(fieldPath, field.Index)) + } continue } - info := &FieldCacheEntry{JsonName: jsonName, isOmitEmpty: isOmitempty, fieldPath: append(fieldPath, field.Index), fieldType: field.Type} + info := &FieldCacheEntry{JsonName: jsonName, isOmitEmpty: isOmitempty, omitzero: omitzero, fieldPath: append(fieldPath, field.Index), fieldType: field.Type} infos[jsonName] = info } } @@ -182,6 +194,11 @@ func (e TypeReflectCacheEntry) ToUnstructured(sv reflect.Value) (interface{}, er // This is based on https://github.com/kubernetes/kubernetes/blob/82c9e5c814eb7acc6cc0a090c057294d0667ad66/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go#L505 // and is intended to replace it. + // Check if the object is a nil pointer. + if sv.Kind() == reflect.Ptr && sv.IsNil() { + // We're done - we don't need to store anything. + return nil, nil + } // Check if the object has a custom string converter and use it if available, since it is much more efficient // than round tripping through json. if converter, ok := e.getUnstructuredConverter(sv); ok { @@ -189,11 +206,6 @@ func (e TypeReflectCacheEntry) ToUnstructured(sv reflect.Value) (interface{}, er } // Check if the object has a custom JSON marshaller/unmarshaller. if marshaler, ok := e.getJsonMarshaler(sv); ok { - if sv.Kind() == reflect.Ptr && sv.IsNil() { - // We're done - we don't need to store anything. - return nil, nil - } - data, err := marshaler.MarshalJSON() if err != nil { return nil, err @@ -377,34 +389,47 @@ const maxDepth = 10000 // unmarshal unmarshals the given data // If v is a *map[string]interface{}, numbers are converted to int64 or float64 func unmarshal(data []byte, v interface{}) error { + // Build a decoder from the given data + decoder := json.NewDecoder(bytes.NewBuffer(data)) + // Preserve numbers, rather than casting to float64 automatically + decoder.UseNumber() + // Run the decode + if err := decoder.Decode(v); err != nil { + return err + } + next := decoder.InputOffset() + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + tail := bytes.TrimLeft(data[next:], " \t\r\n") + return fmt.Errorf("unexpected trailing data at offset %d", len(data)-len(tail)) + } + + // If the decode succeeds, post-process the object to convert json.Number objects to int64 or float64 switch v := v.(type) { case *map[string]interface{}: - // Build a decoder from the given data - decoder := json.NewDecoder(bytes.NewBuffer(data)) - // Preserve numbers, rather than casting to float64 automatically - decoder.UseNumber() - // Run the decode - if err := decoder.Decode(v); err != nil { - return err - } - // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 return convertMapNumbers(*v, 0) case *[]interface{}: - // Build a decoder from the given data - decoder := json.NewDecoder(bytes.NewBuffer(data)) - // Preserve numbers, rather than casting to float64 automatically - decoder.UseNumber() - // Run the decode - if err := decoder.Decode(v); err != nil { - return err - } - // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 return convertSliceNumbers(*v, 0) + case *interface{}: + return convertInterfaceNumbers(v, 0) + default: - return json.Unmarshal(data, v) + return nil + } +} + +func convertInterfaceNumbers(v *interface{}, depth int) error { + var err error + switch v2 := (*v).(type) { + case json.Number: + *v, err = convertNumber(v2) + case map[string]interface{}: + err = convertMapNumbers(v2, depth+1) + case []interface{}: + err = convertSliceNumbers(v2, depth+1) } + return err } // convertMapNumbers traverses the map, converting any json.Number values to int64 or float64. diff --git a/value/reflectcache_test.go b/value/reflectcache_test.go index 9f54e0b0..c1a7c856 100644 --- a/value/reflectcache_test.go +++ b/value/reflectcache_test.go @@ -17,8 +17,11 @@ limitations under the License. package value import ( + "encoding/json" + "fmt" "reflect" "testing" + "time" ) type CustomValue struct { @@ -39,10 +42,26 @@ func (c *CustomPointer) MarshalJSON() ([]byte, error) { return c.data, nil } +// Mimics https://github.com/kubernetes/apimachinery/blob/master/pkg/apis/meta/v1/time.go. +type Time struct { + time.Time +} + +// ToUnstructured implements the value.UnstructuredConverter interface. +func (t Time) ToUnstructured() interface{} { + if t.IsZero() { + return nil + } + buf := make([]byte, 0, len(time.RFC3339)) + buf = t.UTC().AppendFormat(buf, time.RFC3339) + return string(buf) +} + func TestToUnstructured(t *testing.T) { testcases := []struct { - Data string - Expected interface{} + Data string + Expected interface{} + ExpectedErrorMessage string }{ {Data: `null`, Expected: nil}, {Data: `true`, Expected: true}, @@ -53,6 +72,12 @@ func TestToUnstructured(t *testing.T) { {Data: `{"a":1}`, Expected: map[string]interface{}{"a": int64(1)}}, {Data: `0`, Expected: int64(0)}, {Data: `0.0`, Expected: float64(0)}, + {Data: "{} \t\r\n", Expected: map[string]interface{}{}}, + {Data: "{} \t\r\n}", ExpectedErrorMessage: "error decoding object from json: unexpected trailing data at offset 6"}, + {Data: "{} \t\r\n{}", ExpectedErrorMessage: "error decoding object from json: unexpected trailing data at offset 6"}, + {Data: "[] \t\r\n", Expected: []interface{}{}}, + {Data: "[] \t\r\n]", ExpectedErrorMessage: "error decoding array from json: unexpected trailing data at offset 6"}, + {Data: "[] \t\r\n[]", ExpectedErrorMessage: "error decoding array from json: unexpected trailing data at offset 6"}, } for _, tc := range testcases { @@ -68,7 +93,13 @@ func TestToUnstructured(t *testing.T) { rv := reflect.ValueOf(custom) result, err := TypeReflectEntryOf(rv.Type()).ToUnstructured(rv) if err != nil { - t.Fatal(err) + if tc.ExpectedErrorMessage == "" { + t.Fatal(err) + } else if got := err.Error(); got != tc.ExpectedErrorMessage { + t.Fatalf("expected error message %q but got %q", tc.ExpectedErrorMessage, got) + } + } else if tc.ExpectedErrorMessage != "" { + t.Fatalf("expected error message %q but got nil error", tc.ExpectedErrorMessage) } if !reflect.DeepEqual(result, tc.Expected) { t.Errorf("expected %#v but got %#v", tc.Expected, result) @@ -77,3 +108,347 @@ func TestToUnstructured(t *testing.T) { }) } } + +func timePtr(t time.Time) *time.Time { return &t } + +func TestTimeToUnstructured(t *testing.T) { + testcases := []struct { + Name string + Time *time.Time + Expected interface{} + }{ + {Name: "nil", Time: nil, Expected: nil}, + {Name: "zero", Time: &time.Time{}, Expected: nil}, + {Name: "1", Time: timePtr(time.Time{}.Add(time.Second)), Expected: "0001-01-01T00:00:01Z"}, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + var time *Time + rv := reflect.ValueOf(time) + if tc.Time != nil { + rv = reflect.ValueOf(Time{Time: *tc.Time}) + } + result, err := TypeReflectEntryOf(rv.Type()).ToUnstructured(rv) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result, tc.Expected) { + t.Errorf("expected %#v but got %#v", tc.Expected, result) + } + }) + } +} + +func TestTypeReflectEntryOf(t *testing.T) { + testString := "" + testCustomType := customOmitZeroType{} + tests := map[string]struct { + arg interface{} + want *TypeReflectCacheEntry + }{ + "StructWithStringField": { + arg: struct { + F1 string `json:"f1"` + }{}, + want: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{ + "f1": { + JsonName: "f1", + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(testString), + TypeEntry: &TypeReflectCacheEntry{}, + }, + }, + orderedStructFields: []*FieldCacheEntry{ + { + JsonName: "f1", + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(testString), + TypeEntry: &TypeReflectCacheEntry{}, + }, + }, + }, + }, + "StructWith*StringFieldOmitempty": { + arg: struct { + F1 *string `json:"f1,omitempty"` + }{}, + want: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{ + "f1": { + JsonName: "f1", + isOmitEmpty: true, + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(&testString), + TypeEntry: &TypeReflectCacheEntry{}, + }, + }, + orderedStructFields: []*FieldCacheEntry{ + { + JsonName: "f1", + isOmitEmpty: true, + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(&testString), + TypeEntry: &TypeReflectCacheEntry{}, + }, + }, + }, + }, + "StructWith*StringFieldOmitzero": { + arg: struct { + F1 *string `json:"f1,omitzero"` + }{}, + want: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{ + "f1": { + JsonName: "f1", + omitzero: func(v reflect.Value) bool { return v.IsZero() }, + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(&testString), + TypeEntry: &TypeReflectCacheEntry{}, + }, + }, + orderedStructFields: []*FieldCacheEntry{ + { + JsonName: "f1", + omitzero: func(v reflect.Value) bool { return v.IsZero() }, + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(&testString), + TypeEntry: &TypeReflectCacheEntry{}, + }, + }, + }, + }, + "StructWith*CustomFieldOmitzero": { + arg: struct { + F1 customOmitZeroType `json:"f1,omitzero"` + }{}, + want: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{ + "f1": { + JsonName: "f1", + omitzero: func(v reflect.Value) bool { return false }, + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(testCustomType), + TypeEntry: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{}, + orderedStructFields: []*FieldCacheEntry{}, + }, + }, + }, + orderedStructFields: []*FieldCacheEntry{ + { + JsonName: "f1", + omitzero: func(v reflect.Value) bool { return false }, + fieldPath: [][]int{{0}}, + fieldType: reflect.TypeOf(testCustomType), + TypeEntry: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{}, + orderedStructFields: []*FieldCacheEntry{}, + }, + }, + }, + }, + }, + "StructWithInlinedField": { + arg: struct { + F1 string `json:",inline"` + }{}, + want: &TypeReflectCacheEntry{ + structFields: map[string]*FieldCacheEntry{}, + orderedStructFields: []*FieldCacheEntry{}, + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := TypeReflectEntryOf(reflect.TypeOf(tt.arg)) + + // evaluate non-comparable omitzero functions + for k, v := range got.structFields { + compareOmitZero(t, v.fieldType, v.omitzero, tt.want.structFields[k].omitzero) + } + for i, v := range got.orderedStructFields { + compareOmitZero(t, v.fieldType, v.omitzero, tt.want.orderedStructFields[i].omitzero) + } + + // clear non-comparable omitzero functions + for k, v := range got.structFields { + v.omitzero = nil + tt.want.structFields[k].omitzero = nil + } + for i, v := range got.orderedStructFields { + v.omitzero = nil + tt.want.orderedStructFields[i].omitzero = nil + } + + // compare remaining fields + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("TypeReflectEntryOf() got\n%#v\nwant\n%#v", got, tt.want) + } + }) + } +} + +type customOmitZeroType struct { +} + +func (c *customOmitZeroType) IsZero() bool { + return false +} + +func compareOmitZero(t *testing.T, fieldType reflect.Type, got, want func(reflect.Value) bool) { + t.Helper() + if (want == nil) != (got == nil) { + t.Fatalf("wanted omitzero=%v, got omitzero=%v", (want == nil), (got == nil)) + } + if want == nil { + return + } + v := reflect.New(fieldType).Elem() + if e, a := want(v), got(v); e != a { + t.Fatalf("wanted omitzero()=%v, got omitzero()=%v", e, a) + } +} + +func TestUnmarshal(t *testing.T) { + for _, tc := range []struct { + JSON string + IntoType reflect.Type + Want interface{} + WantError bool + }{ + { + JSON: "{}}", + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: map[string]interface{}{}, + WantError: true, + }, + { + JSON: `1.0`, + IntoType: reflect.TypeOf(json.Number("")), + Want: json.Number("1.0"), + }, + { + JSON: `1`, + IntoType: reflect.TypeOf(json.Number("")), + Want: json.Number("1"), + }, + { + JSON: `1.0`, + IntoType: reflect.TypeOf(float64(0)), + Want: float64(1), + }, + { + JSON: `1`, + IntoType: reflect.TypeOf(float64(0)), + Want: float64(1), + }, + { + JSON: `1.0`, + IntoType: reflect.TypeOf(int64(0)), + Want: int64(0), + WantError: true, + }, + { + JSON: `1`, + IntoType: reflect.TypeOf(int64(0)), + Want: int64(1), + }, + { + JSON: `1.0`, + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: float64(1), + }, + { + JSON: `1`, + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: int64(1), + }, + { + JSON: `[1.0,[1.0],{"":1.0}]`, + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: []interface{}{ + float64(1), + []interface{}{float64(1)}, + map[string]interface{}{"": float64(1)}, + }, + }, + { + JSON: `[1.0,[1.0],{"":1.0}]`, + IntoType: reflect.TypeOf([]interface{}{}), + Want: []interface{}{ + float64(1), + []interface{}{float64(1)}, + map[string]interface{}{"": float64(1)}, + }, + }, + { + JSON: `[1,[1],{"":1}]`, + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: []interface{}{ + int64(1), + []interface{}{int64(1)}, + map[string]interface{}{"": int64(1)}, + }, + }, + { + JSON: `[1,[1],{"":1}]`, + IntoType: reflect.TypeOf([]interface{}{}), + Want: []interface{}{ + int64(1), + []interface{}{int64(1)}, + map[string]interface{}{"": int64(1)}, + }, + }, + { + JSON: `{"x":1.0,"y":[1.0],"z":{"":1.0}}`, + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: map[string]interface{}{ + "x": float64(1), + "y": []interface{}{float64(1)}, + "z": map[string]interface{}{"": float64(1)}, + }, + }, + { + JSON: `{"x":1.0,"y":[1.0],"z":{"":1.0}}`, + IntoType: reflect.TypeOf(map[string]interface{}{}), + Want: map[string]interface{}{ + "x": float64(1), + "y": []interface{}{float64(1)}, + "z": map[string]interface{}{"": float64(1)}, + }, + }, + { + JSON: `{"x":1,"y":[1],"z":{"":1}}`, + IntoType: reflect.TypeOf([0]interface{}{}).Elem(), + Want: map[string]interface{}{ + "x": int64(1), + "y": []interface{}{int64(1)}, + "z": map[string]interface{}{"": int64(1)}, + }, + }, + { + JSON: `{"x":1,"y":[1],"z":{"":1}}`, + IntoType: reflect.TypeOf(map[string]interface{}{}), + Want: map[string]interface{}{ + "x": int64(1), + "y": []interface{}{int64(1)}, + "z": map[string]interface{}{"": int64(1)}, + }, + }, + } { + t.Run(fmt.Sprintf("%s into %v", tc.JSON, reflect.PointerTo(tc.IntoType)), func(t *testing.T) { + into := reflect.New(tc.IntoType) + if err := unmarshal([]byte(tc.JSON), into.Interface()); tc.WantError != (err != nil) { + t.Fatalf("unexpected error: %v", err) + } + if got := into.Elem().Interface(); !reflect.DeepEqual(tc.Want, got) { + t.Errorf("want %#v (%T), got %#v (%T)", tc.Want, tc.Want, got, got) + } + }) + } +} diff --git a/value/scalar.go b/value/scalar.go index c78a4c18..5824219e 100644 --- a/value/scalar.go +++ b/value/scalar.go @@ -43,7 +43,7 @@ func IntCompare(lhs, rhs int64) int { func BoolCompare(lhs, rhs bool) int { if lhs == rhs { return 0 - } else if lhs == false { + } else if !lhs { return -1 } return 1 diff --git a/value/value.go b/value/value.go index ea79e3a0..140b9903 100644 --- a/value/value.go +++ b/value/value.go @@ -23,7 +23,8 @@ import ( "strings" jsoniter "github.com/json-iterator/go" - "gopkg.in/yaml.v2" + + yaml "go.yaml.in/yaml/v2" ) var ( @@ -90,7 +91,7 @@ func FromJSON(input []byte) (Value, error) { func FromJSONFast(input []byte) (Value, error) { iter := readPool.BorrowIterator(input) defer readPool.ReturnIterator(iter) - return ReadJSONIter(iter) + return readJSONIter(iter) } // ToJSON is a helper function for producing a JSon document. @@ -98,7 +99,7 @@ func ToJSON(v Value) ([]byte, error) { buf := bytes.Buffer{} stream := writePool.BorrowStream(&buf) defer writePool.ReturnStream(stream) - WriteJSONStream(v, stream) + writeJSONStream(v, stream) b := stream.Buffer() err := stream.Flush() // Help jsoniter manage its buffers--without this, the next @@ -109,8 +110,10 @@ func ToJSON(v Value) ([]byte, error) { return buf.Bytes(), err } -// ReadJSONIter reads a Value from a JSON iterator. -func ReadJSONIter(iter *jsoniter.Iterator) (Value, error) { +// readJSONIter reads a Value from a JSON iterator. +// DO NOT EXPORT +// TODO: eliminate this https://github.com/kubernetes-sigs/structured-merge-diff/issues/202 +func readJSONIter(iter *jsoniter.Iterator) (Value, error) { v := iter.Read() if iter.Error != nil && iter.Error != io.EOF { return nil, iter.Error @@ -118,8 +121,10 @@ func ReadJSONIter(iter *jsoniter.Iterator) (Value, error) { return NewValueInterface(v), nil } -// WriteJSONStream writes a value into a JSON stream. -func WriteJSONStream(v Value, stream *jsoniter.Stream) { +// writeJSONStream writes a value into a JSON stream. +// DO NOT EXPORT +// TODO: eliminate this https://github.com/kubernetes-sigs/structured-merge-diff/issues/202 +func writeJSONStream(v Value, stream *jsoniter.Stream) { stream.WriteVal(v.Unstructured()) } diff --git a/vendor/github.com/google/gofuzz/.travis.yml b/vendor/github.com/google/gofuzz/.travis.yml deleted file mode 100644 index f8684d99..00000000 --- a/vendor/github.com/google/gofuzz/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: go - -go: - - 1.4 - - 1.3 - - 1.2 - - tip - -install: - - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi - -script: - - go test -cover diff --git a/vendor/github.com/google/gofuzz/CONTRIBUTING.md b/vendor/github.com/google/gofuzz/CONTRIBUTING.md deleted file mode 100644 index 51cf5cd1..00000000 --- a/vendor/github.com/google/gofuzz/CONTRIBUTING.md +++ /dev/null @@ -1,67 +0,0 @@ -# How to contribute # - -We'd love to accept your patches and contributions to this project. There are -a just a few small guidelines you need to follow. - - -## Contributor License Agreement ## - -Contributions to any Google project must be accompanied by a Contributor -License Agreement. This is not a copyright **assignment**, it simply gives -Google permission to use and redistribute your contributions as part of the -project. - - * If you are an individual writing original source code and you're sure you - own the intellectual property, then you'll need to sign an [individual - CLA][]. - - * If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a [corporate CLA][]. - -You generally only need to submit a CLA once, so if you've already submitted -one (even if it was for a different project), you probably don't need to do it -again. - -[individual CLA]: https://developers.google.com/open-source/cla/individual -[corporate CLA]: https://developers.google.com/open-source/cla/corporate - - -## Submitting a patch ## - - 1. It's generally best to start by opening a new issue describing the bug or - feature you're intending to fix. Even if you think it's relatively minor, - it's helpful to know what people are working on. Mention in the initial - issue that you are planning to work on that bug or feature so that it can - be assigned to you. - - 1. Follow the normal process of [forking][] the project, and setup a new - branch to work in. It's important that each group of changes be done in - separate branches in order to ensure that a pull request only includes the - commits related to that bug or feature. - - 1. Go makes it very simple to ensure properly formatted code, so always run - `go fmt` on your code before committing it. You should also run - [golint][] over your code. As noted in the [golint readme][], it's not - strictly necessary that your code be completely "lint-free", but this will - help you find common style issues. - - 1. Any significant changes should almost always be accompanied by tests. The - project already has good test coverage, so look at some of the existing - tests if you're unsure how to go about it. [gocov][] and [gocov-html][] - are invaluable tools for seeing which parts of your code aren't being - exercised by your tests. - - 1. Do your best to have [well-formed commit messages][] for each change. - This provides consistency throughout the project, and ensures that commit - messages are able to be formatted properly by various git tools. - - 1. Finally, push the commits to your fork and submit a [pull request][]. - -[forking]: https://help.github.com/articles/fork-a-repo -[golint]: https://github.com/golang/lint -[golint readme]: https://github.com/golang/lint/blob/master/README -[gocov]: https://github.com/axw/gocov -[gocov-html]: https://github.com/matm/gocov-html -[well-formed commit messages]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html -[squash]: http://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits -[pull request]: https://help.github.com/articles/creating-a-pull-request diff --git a/vendor/github.com/google/gofuzz/LICENSE b/vendor/github.com/google/gofuzz/LICENSE deleted file mode 100644 index d6456956..00000000 --- a/vendor/github.com/google/gofuzz/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/google/gofuzz/README.md b/vendor/github.com/google/gofuzz/README.md deleted file mode 100644 index 64869af3..00000000 --- a/vendor/github.com/google/gofuzz/README.md +++ /dev/null @@ -1,71 +0,0 @@ -gofuzz -====== - -gofuzz is a library for populating go objects with random values. - -[![GoDoc](https://godoc.org/github.com/google/gofuzz?status.png)](https://godoc.org/github.com/google/gofuzz) -[![Travis](https://travis-ci.org/google/gofuzz.svg?branch=master)](https://travis-ci.org/google/gofuzz) - -This is useful for testing: - -* Do your project's objects really serialize/unserialize correctly in all cases? -* Is there an incorrectly formatted object that will cause your project to panic? - -Import with ```import "github.com/google/gofuzz"``` - -You can use it on single variables: -```go -f := fuzz.New() -var myInt int -f.Fuzz(&myInt) // myInt gets a random value. -``` - -You can use it on maps: -```go -f := fuzz.New().NilChance(0).NumElements(1, 1) -var myMap map[ComplexKeyType]string -f.Fuzz(&myMap) // myMap will have exactly one element. -``` - -Customize the chance of getting a nil pointer: -```go -f := fuzz.New().NilChance(.5) -var fancyStruct struct { - A, B, C, D *string -} -f.Fuzz(&fancyStruct) // About half the pointers should be set. -``` - -You can even customize the randomization completely if needed: -```go -type MyEnum string -const ( - A MyEnum = "A" - B MyEnum = "B" -) -type MyInfo struct { - Type MyEnum - AInfo *string - BInfo *string -} - -f := fuzz.New().NilChance(0).Funcs( - func(e *MyInfo, c fuzz.Continue) { - switch c.Intn(2) { - case 0: - e.Type = A - c.Fuzz(&e.AInfo) - case 1: - e.Type = B - c.Fuzz(&e.BInfo) - } - }, -) - -var myObject MyInfo -f.Fuzz(&myObject) // Type will correspond to whether A or B info is set. -``` - -See more examples in ```example_test.go```. - -Happy testing! diff --git a/vendor/github.com/google/gofuzz/doc.go b/vendor/github.com/google/gofuzz/doc.go deleted file mode 100644 index 9f9956d4..00000000 --- a/vendor/github.com/google/gofuzz/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2014 Google Inc. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package fuzz is a library for populating go objects with random values. -package fuzz diff --git a/vendor/github.com/google/gofuzz/fuzz.go b/vendor/github.com/google/gofuzz/fuzz.go deleted file mode 100644 index 1dfa80a6..00000000 --- a/vendor/github.com/google/gofuzz/fuzz.go +++ /dev/null @@ -1,487 +0,0 @@ -/* -Copyright 2014 Google Inc. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fuzz - -import ( - "fmt" - "math/rand" - "reflect" - "time" -) - -// fuzzFuncMap is a map from a type to a fuzzFunc that handles that type. -type fuzzFuncMap map[reflect.Type]reflect.Value - -// Fuzzer knows how to fill any object with random fields. -type Fuzzer struct { - fuzzFuncs fuzzFuncMap - defaultFuzzFuncs fuzzFuncMap - r *rand.Rand - nilChance float64 - minElements int - maxElements int - maxDepth int -} - -// New returns a new Fuzzer. Customize your Fuzzer further by calling Funcs, -// RandSource, NilChance, or NumElements in any order. -func New() *Fuzzer { - return NewWithSeed(time.Now().UnixNano()) -} - -func NewWithSeed(seed int64) *Fuzzer { - f := &Fuzzer{ - defaultFuzzFuncs: fuzzFuncMap{ - reflect.TypeOf(&time.Time{}): reflect.ValueOf(fuzzTime), - }, - - fuzzFuncs: fuzzFuncMap{}, - r: rand.New(rand.NewSource(seed)), - nilChance: .2, - minElements: 1, - maxElements: 10, - maxDepth: 100, - } - return f -} - -// Funcs adds each entry in fuzzFuncs as a custom fuzzing function. -// -// Each entry in fuzzFuncs must be a function taking two parameters. -// The first parameter must be a pointer or map. It is the variable that -// function will fill with random data. The second parameter must be a -// fuzz.Continue, which will provide a source of randomness and a way -// to automatically continue fuzzing smaller pieces of the first parameter. -// -// These functions are called sensibly, e.g., if you wanted custom string -// fuzzing, the function `func(s *string, c fuzz.Continue)` would get -// called and passed the address of strings. Maps and pointers will always -// be made/new'd for you, ignoring the NilChange option. For slices, it -// doesn't make much sense to pre-create them--Fuzzer doesn't know how -// long you want your slice--so take a pointer to a slice, and make it -// yourself. (If you don't want your map/pointer type pre-made, take a -// pointer to it, and make it yourself.) See the examples for a range of -// custom functions. -func (f *Fuzzer) Funcs(fuzzFuncs ...interface{}) *Fuzzer { - for i := range fuzzFuncs { - v := reflect.ValueOf(fuzzFuncs[i]) - if v.Kind() != reflect.Func { - panic("Need only funcs!") - } - t := v.Type() - if t.NumIn() != 2 || t.NumOut() != 0 { - panic("Need 2 in and 0 out params!") - } - argT := t.In(0) - switch argT.Kind() { - case reflect.Ptr, reflect.Map: - default: - panic("fuzzFunc must take pointer or map type") - } - if t.In(1) != reflect.TypeOf(Continue{}) { - panic("fuzzFunc's second parameter must be type fuzz.Continue") - } - f.fuzzFuncs[argT] = v - } - return f -} - -// RandSource causes f to get values from the given source of randomness. -// Use if you want deterministic fuzzing. -func (f *Fuzzer) RandSource(s rand.Source) *Fuzzer { - f.r = rand.New(s) - return f -} - -// NilChance sets the probability of creating a nil pointer, map, or slice to -// 'p'. 'p' should be between 0 (no nils) and 1 (all nils), inclusive. -func (f *Fuzzer) NilChance(p float64) *Fuzzer { - if p < 0 || p > 1 { - panic("p should be between 0 and 1, inclusive.") - } - f.nilChance = p - return f -} - -// NumElements sets the minimum and maximum number of elements that will be -// added to a non-nil map or slice. -func (f *Fuzzer) NumElements(atLeast, atMost int) *Fuzzer { - if atLeast > atMost { - panic("atLeast must be <= atMost") - } - if atLeast < 0 { - panic("atLeast must be >= 0") - } - f.minElements = atLeast - f.maxElements = atMost - return f -} - -func (f *Fuzzer) genElementCount() int { - if f.minElements == f.maxElements { - return f.minElements - } - return f.minElements + f.r.Intn(f.maxElements-f.minElements+1) -} - -func (f *Fuzzer) genShouldFill() bool { - return f.r.Float64() > f.nilChance -} - -// MaxDepth sets the maximum number of recursive fuzz calls that will be made -// before stopping. This includes struct members, pointers, and map and slice -// elements. -func (f *Fuzzer) MaxDepth(d int) *Fuzzer { - f.maxDepth = d - return f -} - -// Fuzz recursively fills all of obj's fields with something random. First -// this tries to find a custom fuzz function (see Funcs). If there is no -// custom function this tests whether the object implements fuzz.Interface and, -// if so, calls Fuzz on it to fuzz itself. If that fails, this will see if -// there is a default fuzz function provided by this package. If all of that -// fails, this will generate random values for all primitive fields and then -// recurse for all non-primitives. -// -// This is safe for cyclic or tree-like structs, up to a limit. Use the -// MaxDepth method to adjust how deep you need it to recurse. -// -// obj must be a pointer. Only exported (public) fields can be set (thanks, -// golang :/ ) Intended for tests, so will panic on bad input or unimplemented -// fields. -func (f *Fuzzer) Fuzz(obj interface{}) { - v := reflect.ValueOf(obj) - if v.Kind() != reflect.Ptr { - panic("needed ptr!") - } - v = v.Elem() - f.fuzzWithContext(v, 0) -} - -// FuzzNoCustom is just like Fuzz, except that any custom fuzz function for -// obj's type will not be called and obj will not be tested for fuzz.Interface -// conformance. This applies only to obj and not other instances of obj's -// type. -// Not safe for cyclic or tree-like structs! -// obj must be a pointer. Only exported (public) fields can be set (thanks, golang :/ ) -// Intended for tests, so will panic on bad input or unimplemented fields. -func (f *Fuzzer) FuzzNoCustom(obj interface{}) { - v := reflect.ValueOf(obj) - if v.Kind() != reflect.Ptr { - panic("needed ptr!") - } - v = v.Elem() - f.fuzzWithContext(v, flagNoCustomFuzz) -} - -const ( - // Do not try to find a custom fuzz function. Does not apply recursively. - flagNoCustomFuzz uint64 = 1 << iota -) - -func (f *Fuzzer) fuzzWithContext(v reflect.Value, flags uint64) { - fc := &fuzzerContext{fuzzer: f} - fc.doFuzz(v, flags) -} - -// fuzzerContext carries context about a single fuzzing run, which lets Fuzzer -// be thread-safe. -type fuzzerContext struct { - fuzzer *Fuzzer - curDepth int -} - -func (fc *fuzzerContext) doFuzz(v reflect.Value, flags uint64) { - if fc.curDepth >= fc.fuzzer.maxDepth { - return - } - fc.curDepth++ - defer func() { fc.curDepth-- }() - - if !v.CanSet() { - return - } - - if flags&flagNoCustomFuzz == 0 { - // Check for both pointer and non-pointer custom functions. - if v.CanAddr() && fc.tryCustom(v.Addr()) { - return - } - if fc.tryCustom(v) { - return - } - } - - if fn, ok := fillFuncMap[v.Kind()]; ok { - fn(v, fc.fuzzer.r) - return - } - switch v.Kind() { - case reflect.Map: - if fc.fuzzer.genShouldFill() { - v.Set(reflect.MakeMap(v.Type())) - n := fc.fuzzer.genElementCount() - for i := 0; i < n; i++ { - key := reflect.New(v.Type().Key()).Elem() - fc.doFuzz(key, 0) - val := reflect.New(v.Type().Elem()).Elem() - fc.doFuzz(val, 0) - v.SetMapIndex(key, val) - } - return - } - v.Set(reflect.Zero(v.Type())) - case reflect.Ptr: - if fc.fuzzer.genShouldFill() { - v.Set(reflect.New(v.Type().Elem())) - fc.doFuzz(v.Elem(), 0) - return - } - v.Set(reflect.Zero(v.Type())) - case reflect.Slice: - if fc.fuzzer.genShouldFill() { - n := fc.fuzzer.genElementCount() - v.Set(reflect.MakeSlice(v.Type(), n, n)) - for i := 0; i < n; i++ { - fc.doFuzz(v.Index(i), 0) - } - return - } - v.Set(reflect.Zero(v.Type())) - case reflect.Array: - if fc.fuzzer.genShouldFill() { - n := v.Len() - for i := 0; i < n; i++ { - fc.doFuzz(v.Index(i), 0) - } - return - } - v.Set(reflect.Zero(v.Type())) - case reflect.Struct: - for i := 0; i < v.NumField(); i++ { - fc.doFuzz(v.Field(i), 0) - } - case reflect.Chan: - fallthrough - case reflect.Func: - fallthrough - case reflect.Interface: - fallthrough - default: - panic(fmt.Sprintf("Can't handle %#v", v.Interface())) - } -} - -// tryCustom searches for custom handlers, and returns true iff it finds a match -// and successfully randomizes v. -func (fc *fuzzerContext) tryCustom(v reflect.Value) bool { - // First: see if we have a fuzz function for it. - doCustom, ok := fc.fuzzer.fuzzFuncs[v.Type()] - if !ok { - // Second: see if it can fuzz itself. - if v.CanInterface() { - intf := v.Interface() - if fuzzable, ok := intf.(Interface); ok { - fuzzable.Fuzz(Continue{fc: fc, Rand: fc.fuzzer.r}) - return true - } - } - // Finally: see if there is a default fuzz function. - doCustom, ok = fc.fuzzer.defaultFuzzFuncs[v.Type()] - if !ok { - return false - } - } - - switch v.Kind() { - case reflect.Ptr: - if v.IsNil() { - if !v.CanSet() { - return false - } - v.Set(reflect.New(v.Type().Elem())) - } - case reflect.Map: - if v.IsNil() { - if !v.CanSet() { - return false - } - v.Set(reflect.MakeMap(v.Type())) - } - default: - return false - } - - doCustom.Call([]reflect.Value{v, reflect.ValueOf(Continue{ - fc: fc, - Rand: fc.fuzzer.r, - })}) - return true -} - -// Interface represents an object that knows how to fuzz itself. Any time we -// find a type that implements this interface we will delegate the act of -// fuzzing itself. -type Interface interface { - Fuzz(c Continue) -} - -// Continue can be passed to custom fuzzing functions to allow them to use -// the correct source of randomness and to continue fuzzing their members. -type Continue struct { - fc *fuzzerContext - - // For convenience, Continue implements rand.Rand via embedding. - // Use this for generating any randomness if you want your fuzzing - // to be repeatable for a given seed. - *rand.Rand -} - -// Fuzz continues fuzzing obj. obj must be a pointer. -func (c Continue) Fuzz(obj interface{}) { - v := reflect.ValueOf(obj) - if v.Kind() != reflect.Ptr { - panic("needed ptr!") - } - v = v.Elem() - c.fc.doFuzz(v, 0) -} - -// FuzzNoCustom continues fuzzing obj, except that any custom fuzz function for -// obj's type will not be called and obj will not be tested for fuzz.Interface -// conformance. This applies only to obj and not other instances of obj's -// type. -func (c Continue) FuzzNoCustom(obj interface{}) { - v := reflect.ValueOf(obj) - if v.Kind() != reflect.Ptr { - panic("needed ptr!") - } - v = v.Elem() - c.fc.doFuzz(v, flagNoCustomFuzz) -} - -// RandString makes a random string up to 20 characters long. The returned string -// may include a variety of (valid) UTF-8 encodings. -func (c Continue) RandString() string { - return randString(c.Rand) -} - -// RandUint64 makes random 64 bit numbers. -// Weirdly, rand doesn't have a function that gives you 64 random bits. -func (c Continue) RandUint64() uint64 { - return randUint64(c.Rand) -} - -// RandBool returns true or false randomly. -func (c Continue) RandBool() bool { - return randBool(c.Rand) -} - -func fuzzInt(v reflect.Value, r *rand.Rand) { - v.SetInt(int64(randUint64(r))) -} - -func fuzzUint(v reflect.Value, r *rand.Rand) { - v.SetUint(randUint64(r)) -} - -func fuzzTime(t *time.Time, c Continue) { - var sec, nsec int64 - // Allow for about 1000 years of random time values, which keeps things - // like JSON parsing reasonably happy. - sec = c.Rand.Int63n(1000 * 365 * 24 * 60 * 60) - c.Fuzz(&nsec) - *t = time.Unix(sec, nsec) -} - -var fillFuncMap = map[reflect.Kind]func(reflect.Value, *rand.Rand){ - reflect.Bool: func(v reflect.Value, r *rand.Rand) { - v.SetBool(randBool(r)) - }, - reflect.Int: fuzzInt, - reflect.Int8: fuzzInt, - reflect.Int16: fuzzInt, - reflect.Int32: fuzzInt, - reflect.Int64: fuzzInt, - reflect.Uint: fuzzUint, - reflect.Uint8: fuzzUint, - reflect.Uint16: fuzzUint, - reflect.Uint32: fuzzUint, - reflect.Uint64: fuzzUint, - reflect.Uintptr: fuzzUint, - reflect.Float32: func(v reflect.Value, r *rand.Rand) { - v.SetFloat(float64(r.Float32())) - }, - reflect.Float64: func(v reflect.Value, r *rand.Rand) { - v.SetFloat(r.Float64()) - }, - reflect.Complex64: func(v reflect.Value, r *rand.Rand) { - panic("unimplemented") - }, - reflect.Complex128: func(v reflect.Value, r *rand.Rand) { - panic("unimplemented") - }, - reflect.String: func(v reflect.Value, r *rand.Rand) { - v.SetString(randString(r)) - }, - reflect.UnsafePointer: func(v reflect.Value, r *rand.Rand) { - panic("unimplemented") - }, -} - -// randBool returns true or false randomly. -func randBool(r *rand.Rand) bool { - if r.Int()&1 == 1 { - return true - } - return false -} - -type charRange struct { - first, last rune -} - -// choose returns a random unicode character from the given range, using the -// given randomness source. -func (r *charRange) choose(rand *rand.Rand) rune { - count := int64(r.last - r.first) - return r.first + rune(rand.Int63n(count)) -} - -var unicodeRanges = []charRange{ - {' ', '~'}, // ASCII characters - {'\u00a0', '\u02af'}, // Multi-byte encoded characters - {'\u4e00', '\u9fff'}, // Common CJK (even longer encodings) -} - -// randString makes a random string up to 20 characters long. The returned string -// may include a variety of (valid) UTF-8 encodings. -func randString(r *rand.Rand) string { - n := r.Intn(20) - runes := make([]rune, n) - for i := range runes { - runes[i] = unicodeRanges[r.Intn(len(unicodeRanges))].choose(r) - } - return string(runes) -} - -// randUint64 makes random 64 bit numbers. -// Weirdly, rand doesn't have a function that gives you 64 random bits. -func randUint64(r *rand.Rand) uint64 { - return uint64(r.Uint32())<<32 | uint64(r.Uint32()) -} diff --git a/vendor/github.com/google/gofuzz/go.mod b/vendor/github.com/google/gofuzz/go.mod deleted file mode 100644 index 8ec4fe9e..00000000 --- a/vendor/github.com/google/gofuzz/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/google/gofuzz - -go 1.12 diff --git a/vendor/github.com/json-iterator/go/.codecov.yml b/vendor/github.com/json-iterator/go/.codecov.yml deleted file mode 100644 index 955dc0be..00000000 --- a/vendor/github.com/json-iterator/go/.codecov.yml +++ /dev/null @@ -1,3 +0,0 @@ -ignore: - - "output_tests/.*" - diff --git a/vendor/github.com/json-iterator/go/.gitignore b/vendor/github.com/json-iterator/go/.gitignore deleted file mode 100644 index 15556530..00000000 --- a/vendor/github.com/json-iterator/go/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/vendor -/bug_test.go -/coverage.txt -/.idea diff --git a/vendor/github.com/json-iterator/go/.travis.yml b/vendor/github.com/json-iterator/go/.travis.yml deleted file mode 100644 index 449e67cd..00000000 --- a/vendor/github.com/json-iterator/go/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: go - -go: - - 1.8.x - - 1.x - -before_install: - - go get -t -v ./... - -script: - - ./test.sh - -after_success: - - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/json-iterator/go/Gopkg.lock b/vendor/github.com/json-iterator/go/Gopkg.lock deleted file mode 100644 index c8a9fbb3..00000000 --- a/vendor/github.com/json-iterator/go/Gopkg.lock +++ /dev/null @@ -1,21 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/modern-go/concurrent" - packages = ["."] - revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a" - version = "1.0.0" - -[[projects]] - name = "github.com/modern-go/reflect2" - packages = ["."] - revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" - version = "1.0.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "ea54a775e5a354cb015502d2e7aa4b74230fc77e894f34a838b268c25ec8eeb8" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/vendor/github.com/json-iterator/go/Gopkg.toml b/vendor/github.com/json-iterator/go/Gopkg.toml deleted file mode 100644 index 313a0f88..00000000 --- a/vendor/github.com/json-iterator/go/Gopkg.toml +++ /dev/null @@ -1,26 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - -ignored = ["github.com/davecgh/go-spew*","github.com/google/gofuzz*","github.com/stretchr/testify*"] - -[[constraint]] - name = "github.com/modern-go/reflect2" - version = "1.0.1" diff --git a/vendor/github.com/json-iterator/go/LICENSE b/vendor/github.com/json-iterator/go/LICENSE deleted file mode 100644 index 2cf4f5ab..00000000 --- a/vendor/github.com/json-iterator/go/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 json-iterator - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/json-iterator/go/README.md b/vendor/github.com/json-iterator/go/README.md deleted file mode 100644 index 50d56ffb..00000000 --- a/vendor/github.com/json-iterator/go/README.md +++ /dev/null @@ -1,87 +0,0 @@ -[![Sourcegraph](https://sourcegraph.com/github.com/json-iterator/go/-/badge.svg)](https://sourcegraph.com/github.com/json-iterator/go?badge) -[![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/json-iterator/go) -[![Build Status](https://travis-ci.org/json-iterator/go.svg?branch=master)](https://travis-ci.org/json-iterator/go) -[![codecov](https://codecov.io/gh/json-iterator/go/branch/master/graph/badge.svg)](https://codecov.io/gh/json-iterator/go) -[![rcard](https://goreportcard.com/badge/github.com/json-iterator/go)](https://goreportcard.com/report/github.com/json-iterator/go) -[![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/json-iterator/go/master/LICENSE) -[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/json-iterator/Lobby) - -A high-performance 100% compatible drop-in replacement of "encoding/json" - -You can also use thrift like JSON using [thrift-iterator](https://github.com/thrift-iterator/go) - -# Benchmark - -![benchmark](http://jsoniter.com/benchmarks/go-benchmark.png) - -Source code: https://github.com/json-iterator/go-benchmark/blob/master/src/github.com/json-iterator/go-benchmark/benchmark_medium_payload_test.go - -Raw Result (easyjson requires static code generation) - -| | ns/op | allocation bytes | allocation times | -| --- | --- | --- | --- | -| std decode | 35510 ns/op | 1960 B/op | 99 allocs/op | -| easyjson decode | 8499 ns/op | 160 B/op | 4 allocs/op | -| jsoniter decode | 5623 ns/op | 160 B/op | 3 allocs/op | -| std encode | 2213 ns/op | 712 B/op | 5 allocs/op | -| easyjson encode | 883 ns/op | 576 B/op | 3 allocs/op | -| jsoniter encode | 837 ns/op | 384 B/op | 4 allocs/op | - -Always benchmark with your own workload. -The result depends heavily on the data input. - -# Usage - -100% compatibility with standard lib - -Replace - -```go -import "encoding/json" -json.Marshal(&data) -``` - -with - -```go -import "github.com/json-iterator/go" - -var json = jsoniter.ConfigCompatibleWithStandardLibrary -json.Marshal(&data) -``` - -Replace - -```go -import "encoding/json" -json.Unmarshal(input, &data) -``` - -with - -```go -import "github.com/json-iterator/go" - -var json = jsoniter.ConfigCompatibleWithStandardLibrary -json.Unmarshal(input, &data) -``` - -[More documentation](http://jsoniter.com/migrate-from-go-std.html) - -# How to get - -``` -go get github.com/json-iterator/go -``` - -# Contribution Welcomed ! - -Contributors - -* [thockin](https://github.com/thockin) -* [mattn](https://github.com/mattn) -* [cch123](https://github.com/cch123) -* [Oleg Shaldybin](https://github.com/olegshaldybin) -* [Jason Toffaletti](https://github.com/toffaletti) - -Report issue or pull request, or email taowen@gmail.com, or [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/json-iterator/Lobby) diff --git a/vendor/github.com/json-iterator/go/adapter.go b/vendor/github.com/json-iterator/go/adapter.go deleted file mode 100644 index e674d0f3..00000000 --- a/vendor/github.com/json-iterator/go/adapter.go +++ /dev/null @@ -1,150 +0,0 @@ -package jsoniter - -import ( - "bytes" - "io" -) - -// RawMessage to make replace json with jsoniter -type RawMessage []byte - -// Unmarshal adapts to json/encoding Unmarshal API -// -// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v. -// Refer to https://godoc.org/encoding/json#Unmarshal for more information -func Unmarshal(data []byte, v interface{}) error { - return ConfigDefault.Unmarshal(data, v) -} - -// UnmarshalFromString convenient method to read from string instead of []byte -func UnmarshalFromString(str string, v interface{}) error { - return ConfigDefault.UnmarshalFromString(str, v) -} - -// Get quick method to get value from deeply nested JSON structure -func Get(data []byte, path ...interface{}) Any { - return ConfigDefault.Get(data, path...) -} - -// Marshal adapts to json/encoding Marshal API -// -// Marshal returns the JSON encoding of v, adapts to json/encoding Marshal API -// Refer to https://godoc.org/encoding/json#Marshal for more information -func Marshal(v interface{}) ([]byte, error) { - return ConfigDefault.Marshal(v) -} - -// MarshalIndent same as json.MarshalIndent. Prefix is not supported. -func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { - return ConfigDefault.MarshalIndent(v, prefix, indent) -} - -// MarshalToString convenient method to write as string instead of []byte -func MarshalToString(v interface{}) (string, error) { - return ConfigDefault.MarshalToString(v) -} - -// NewDecoder adapts to json/stream NewDecoder API. -// -// NewDecoder returns a new decoder that reads from r. -// -// Instead of a json/encoding Decoder, an Decoder is returned -// Refer to https://godoc.org/encoding/json#NewDecoder for more information -func NewDecoder(reader io.Reader) *Decoder { - return ConfigDefault.NewDecoder(reader) -} - -// Decoder reads and decodes JSON values from an input stream. -// Decoder provides identical APIs with json/stream Decoder (Token() and UseNumber() are in progress) -type Decoder struct { - iter *Iterator -} - -// Decode decode JSON into interface{} -func (adapter *Decoder) Decode(obj interface{}) error { - if adapter.iter.head == adapter.iter.tail && adapter.iter.reader != nil { - if !adapter.iter.loadMore() { - return io.EOF - } - } - adapter.iter.ReadVal(obj) - err := adapter.iter.Error - if err == io.EOF { - return nil - } - return adapter.iter.Error -} - -// More is there more? -func (adapter *Decoder) More() bool { - iter := adapter.iter - if iter.Error != nil { - return false - } - c := iter.nextToken() - if c == 0 { - return false - } - iter.unreadByte() - return c != ']' && c != '}' -} - -// Buffered remaining buffer -func (adapter *Decoder) Buffered() io.Reader { - remaining := adapter.iter.buf[adapter.iter.head:adapter.iter.tail] - return bytes.NewReader(remaining) -} - -// UseNumber causes the Decoder to unmarshal a number into an interface{} as a -// Number instead of as a float64. -func (adapter *Decoder) UseNumber() { - cfg := adapter.iter.cfg.configBeforeFrozen - cfg.UseNumber = true - adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions) -} - -// DisallowUnknownFields causes the Decoder to return an error when the destination -// is a struct and the input contains object keys which do not match any -// non-ignored, exported fields in the destination. -func (adapter *Decoder) DisallowUnknownFields() { - cfg := adapter.iter.cfg.configBeforeFrozen - cfg.DisallowUnknownFields = true - adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions) -} - -// NewEncoder same as json.NewEncoder -func NewEncoder(writer io.Writer) *Encoder { - return ConfigDefault.NewEncoder(writer) -} - -// Encoder same as json.Encoder -type Encoder struct { - stream *Stream -} - -// Encode encode interface{} as JSON to io.Writer -func (adapter *Encoder) Encode(val interface{}) error { - adapter.stream.WriteVal(val) - adapter.stream.WriteRaw("\n") - adapter.stream.Flush() - return adapter.stream.Error -} - -// SetIndent set the indention. Prefix is not supported -func (adapter *Encoder) SetIndent(prefix, indent string) { - config := adapter.stream.cfg.configBeforeFrozen - config.IndentionStep = len(indent) - adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions) -} - -// SetEscapeHTML escape html by default, set to false to disable -func (adapter *Encoder) SetEscapeHTML(escapeHTML bool) { - config := adapter.stream.cfg.configBeforeFrozen - config.EscapeHTML = escapeHTML - adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions) -} - -// Valid reports whether data is a valid JSON encoding. -func Valid(data []byte) bool { - return ConfigDefault.Valid(data) -} diff --git a/vendor/github.com/json-iterator/go/any.go b/vendor/github.com/json-iterator/go/any.go deleted file mode 100644 index f6b8aeab..00000000 --- a/vendor/github.com/json-iterator/go/any.go +++ /dev/null @@ -1,325 +0,0 @@ -package jsoniter - -import ( - "errors" - "fmt" - "github.com/modern-go/reflect2" - "io" - "reflect" - "strconv" - "unsafe" -) - -// Any generic object representation. -// The lazy json implementation holds []byte and parse lazily. -type Any interface { - LastError() error - ValueType() ValueType - MustBeValid() Any - ToBool() bool - ToInt() int - ToInt32() int32 - ToInt64() int64 - ToUint() uint - ToUint32() uint32 - ToUint64() uint64 - ToFloat32() float32 - ToFloat64() float64 - ToString() string - ToVal(val interface{}) - Get(path ...interface{}) Any - Size() int - Keys() []string - GetInterface() interface{} - WriteTo(stream *Stream) -} - -type baseAny struct{} - -func (any *baseAny) Get(path ...interface{}) Any { - return &invalidAny{baseAny{}, fmt.Errorf("GetIndex %v from simple value", path)} -} - -func (any *baseAny) Size() int { - return 0 -} - -func (any *baseAny) Keys() []string { - return []string{} -} - -func (any *baseAny) ToVal(obj interface{}) { - panic("not implemented") -} - -// WrapInt32 turn int32 into Any interface -func WrapInt32(val int32) Any { - return &int32Any{baseAny{}, val} -} - -// WrapInt64 turn int64 into Any interface -func WrapInt64(val int64) Any { - return &int64Any{baseAny{}, val} -} - -// WrapUint32 turn uint32 into Any interface -func WrapUint32(val uint32) Any { - return &uint32Any{baseAny{}, val} -} - -// WrapUint64 turn uint64 into Any interface -func WrapUint64(val uint64) Any { - return &uint64Any{baseAny{}, val} -} - -// WrapFloat64 turn float64 into Any interface -func WrapFloat64(val float64) Any { - return &floatAny{baseAny{}, val} -} - -// WrapString turn string into Any interface -func WrapString(val string) Any { - return &stringAny{baseAny{}, val} -} - -// Wrap turn a go object into Any interface -func Wrap(val interface{}) Any { - if val == nil { - return &nilAny{} - } - asAny, isAny := val.(Any) - if isAny { - return asAny - } - typ := reflect2.TypeOf(val) - switch typ.Kind() { - case reflect.Slice: - return wrapArray(val) - case reflect.Struct: - return wrapStruct(val) - case reflect.Map: - return wrapMap(val) - case reflect.String: - return WrapString(val.(string)) - case reflect.Int: - if strconv.IntSize == 32 { - return WrapInt32(int32(val.(int))) - } - return WrapInt64(int64(val.(int))) - case reflect.Int8: - return WrapInt32(int32(val.(int8))) - case reflect.Int16: - return WrapInt32(int32(val.(int16))) - case reflect.Int32: - return WrapInt32(val.(int32)) - case reflect.Int64: - return WrapInt64(val.(int64)) - case reflect.Uint: - if strconv.IntSize == 32 { - return WrapUint32(uint32(val.(uint))) - } - return WrapUint64(uint64(val.(uint))) - case reflect.Uintptr: - if ptrSize == 32 { - return WrapUint32(uint32(val.(uintptr))) - } - return WrapUint64(uint64(val.(uintptr))) - case reflect.Uint8: - return WrapUint32(uint32(val.(uint8))) - case reflect.Uint16: - return WrapUint32(uint32(val.(uint16))) - case reflect.Uint32: - return WrapUint32(uint32(val.(uint32))) - case reflect.Uint64: - return WrapUint64(val.(uint64)) - case reflect.Float32: - return WrapFloat64(float64(val.(float32))) - case reflect.Float64: - return WrapFloat64(val.(float64)) - case reflect.Bool: - if val.(bool) == true { - return &trueAny{} - } - return &falseAny{} - } - return &invalidAny{baseAny{}, fmt.Errorf("unsupported type: %v", typ)} -} - -// ReadAny read next JSON element as an Any object. It is a better json.RawMessage. -func (iter *Iterator) ReadAny() Any { - return iter.readAny() -} - -func (iter *Iterator) readAny() Any { - c := iter.nextToken() - switch c { - case '"': - iter.unreadByte() - return &stringAny{baseAny{}, iter.ReadString()} - case 'n': - iter.skipThreeBytes('u', 'l', 'l') // null - return &nilAny{} - case 't': - iter.skipThreeBytes('r', 'u', 'e') // true - return &trueAny{} - case 'f': - iter.skipFourBytes('a', 'l', 's', 'e') // false - return &falseAny{} - case '{': - return iter.readObjectAny() - case '[': - return iter.readArrayAny() - case '-': - return iter.readNumberAny(false) - case 0: - return &invalidAny{baseAny{}, errors.New("input is empty")} - default: - return iter.readNumberAny(true) - } -} - -func (iter *Iterator) readNumberAny(positive bool) Any { - iter.startCapture(iter.head - 1) - iter.skipNumber() - lazyBuf := iter.stopCapture() - return &numberLazyAny{baseAny{}, iter.cfg, lazyBuf, nil} -} - -func (iter *Iterator) readObjectAny() Any { - iter.startCapture(iter.head - 1) - iter.skipObject() - lazyBuf := iter.stopCapture() - return &objectLazyAny{baseAny{}, iter.cfg, lazyBuf, nil} -} - -func (iter *Iterator) readArrayAny() Any { - iter.startCapture(iter.head - 1) - iter.skipArray() - lazyBuf := iter.stopCapture() - return &arrayLazyAny{baseAny{}, iter.cfg, lazyBuf, nil} -} - -func locateObjectField(iter *Iterator, target string) []byte { - var found []byte - iter.ReadObjectCB(func(iter *Iterator, field string) bool { - if field == target { - found = iter.SkipAndReturnBytes() - return false - } - iter.Skip() - return true - }) - return found -} - -func locateArrayElement(iter *Iterator, target int) []byte { - var found []byte - n := 0 - iter.ReadArrayCB(func(iter *Iterator) bool { - if n == target { - found = iter.SkipAndReturnBytes() - return false - } - iter.Skip() - n++ - return true - }) - return found -} - -func locatePath(iter *Iterator, path []interface{}) Any { - for i, pathKeyObj := range path { - switch pathKey := pathKeyObj.(type) { - case string: - valueBytes := locateObjectField(iter, pathKey) - if valueBytes == nil { - return newInvalidAny(path[i:]) - } - iter.ResetBytes(valueBytes) - case int: - valueBytes := locateArrayElement(iter, pathKey) - if valueBytes == nil { - return newInvalidAny(path[i:]) - } - iter.ResetBytes(valueBytes) - case int32: - if '*' == pathKey { - return iter.readAny().Get(path[i:]...) - } - return newInvalidAny(path[i:]) - default: - return newInvalidAny(path[i:]) - } - } - if iter.Error != nil && iter.Error != io.EOF { - return &invalidAny{baseAny{}, iter.Error} - } - return iter.readAny() -} - -var anyType = reflect2.TypeOfPtr((*Any)(nil)).Elem() - -func createDecoderOfAny(ctx *ctx, typ reflect2.Type) ValDecoder { - if typ == anyType { - return &directAnyCodec{} - } - if typ.Implements(anyType) { - return &anyCodec{ - valType: typ, - } - } - return nil -} - -func createEncoderOfAny(ctx *ctx, typ reflect2.Type) ValEncoder { - if typ == anyType { - return &directAnyCodec{} - } - if typ.Implements(anyType) { - return &anyCodec{ - valType: typ, - } - } - return nil -} - -type anyCodec struct { - valType reflect2.Type -} - -func (codec *anyCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - panic("not implemented") -} - -func (codec *anyCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - obj := codec.valType.UnsafeIndirect(ptr) - any := obj.(Any) - any.WriteTo(stream) -} - -func (codec *anyCodec) IsEmpty(ptr unsafe.Pointer) bool { - obj := codec.valType.UnsafeIndirect(ptr) - any := obj.(Any) - return any.Size() == 0 -} - -type directAnyCodec struct { -} - -func (codec *directAnyCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - *(*Any)(ptr) = iter.readAny() -} - -func (codec *directAnyCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - any := *(*Any)(ptr) - if any == nil { - stream.WriteNil() - return - } - any.WriteTo(stream) -} - -func (codec *directAnyCodec) IsEmpty(ptr unsafe.Pointer) bool { - any := *(*Any)(ptr) - return any.Size() == 0 -} diff --git a/vendor/github.com/json-iterator/go/any_array.go b/vendor/github.com/json-iterator/go/any_array.go deleted file mode 100644 index 0449e9aa..00000000 --- a/vendor/github.com/json-iterator/go/any_array.go +++ /dev/null @@ -1,278 +0,0 @@ -package jsoniter - -import ( - "reflect" - "unsafe" -) - -type arrayLazyAny struct { - baseAny - cfg *frozenConfig - buf []byte - err error -} - -func (any *arrayLazyAny) ValueType() ValueType { - return ArrayValue -} - -func (any *arrayLazyAny) MustBeValid() Any { - return any -} - -func (any *arrayLazyAny) LastError() error { - return any.err -} - -func (any *arrayLazyAny) ToBool() bool { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - return iter.ReadArray() -} - -func (any *arrayLazyAny) ToInt() int { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToInt32() int32 { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToInt64() int64 { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToUint() uint { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToUint32() uint32 { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToUint64() uint64 { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToFloat32() float32 { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToFloat64() float64 { - if any.ToBool() { - return 1 - } - return 0 -} - -func (any *arrayLazyAny) ToString() string { - return *(*string)(unsafe.Pointer(&any.buf)) -} - -func (any *arrayLazyAny) ToVal(val interface{}) { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - iter.ReadVal(val) -} - -func (any *arrayLazyAny) Get(path ...interface{}) Any { - if len(path) == 0 { - return any - } - switch firstPath := path[0].(type) { - case int: - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - valueBytes := locateArrayElement(iter, firstPath) - if valueBytes == nil { - return newInvalidAny(path) - } - iter.ResetBytes(valueBytes) - return locatePath(iter, path[1:]) - case int32: - if '*' == firstPath { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - arr := make([]Any, 0) - iter.ReadArrayCB(func(iter *Iterator) bool { - found := iter.readAny().Get(path[1:]...) - if found.ValueType() != InvalidValue { - arr = append(arr, found) - } - return true - }) - return wrapArray(arr) - } - return newInvalidAny(path) - default: - return newInvalidAny(path) - } -} - -func (any *arrayLazyAny) Size() int { - size := 0 - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - iter.ReadArrayCB(func(iter *Iterator) bool { - size++ - iter.Skip() - return true - }) - return size -} - -func (any *arrayLazyAny) WriteTo(stream *Stream) { - stream.Write(any.buf) -} - -func (any *arrayLazyAny) GetInterface() interface{} { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - return iter.Read() -} - -type arrayAny struct { - baseAny - val reflect.Value -} - -func wrapArray(val interface{}) *arrayAny { - return &arrayAny{baseAny{}, reflect.ValueOf(val)} -} - -func (any *arrayAny) ValueType() ValueType { - return ArrayValue -} - -func (any *arrayAny) MustBeValid() Any { - return any -} - -func (any *arrayAny) LastError() error { - return nil -} - -func (any *arrayAny) ToBool() bool { - return any.val.Len() != 0 -} - -func (any *arrayAny) ToInt() int { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToInt32() int32 { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToInt64() int64 { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToUint() uint { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToUint32() uint32 { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToUint64() uint64 { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToFloat32() float32 { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToFloat64() float64 { - if any.val.Len() == 0 { - return 0 - } - return 1 -} - -func (any *arrayAny) ToString() string { - str, _ := MarshalToString(any.val.Interface()) - return str -} - -func (any *arrayAny) Get(path ...interface{}) Any { - if len(path) == 0 { - return any - } - switch firstPath := path[0].(type) { - case int: - if firstPath < 0 || firstPath >= any.val.Len() { - return newInvalidAny(path) - } - return Wrap(any.val.Index(firstPath).Interface()) - case int32: - if '*' == firstPath { - mappedAll := make([]Any, 0) - for i := 0; i < any.val.Len(); i++ { - mapped := Wrap(any.val.Index(i).Interface()).Get(path[1:]...) - if mapped.ValueType() != InvalidValue { - mappedAll = append(mappedAll, mapped) - } - } - return wrapArray(mappedAll) - } - return newInvalidAny(path) - default: - return newInvalidAny(path) - } -} - -func (any *arrayAny) Size() int { - return any.val.Len() -} - -func (any *arrayAny) WriteTo(stream *Stream) { - stream.WriteVal(any.val) -} - -func (any *arrayAny) GetInterface() interface{} { - return any.val.Interface() -} diff --git a/vendor/github.com/json-iterator/go/any_bool.go b/vendor/github.com/json-iterator/go/any_bool.go deleted file mode 100644 index 9452324a..00000000 --- a/vendor/github.com/json-iterator/go/any_bool.go +++ /dev/null @@ -1,137 +0,0 @@ -package jsoniter - -type trueAny struct { - baseAny -} - -func (any *trueAny) LastError() error { - return nil -} - -func (any *trueAny) ToBool() bool { - return true -} - -func (any *trueAny) ToInt() int { - return 1 -} - -func (any *trueAny) ToInt32() int32 { - return 1 -} - -func (any *trueAny) ToInt64() int64 { - return 1 -} - -func (any *trueAny) ToUint() uint { - return 1 -} - -func (any *trueAny) ToUint32() uint32 { - return 1 -} - -func (any *trueAny) ToUint64() uint64 { - return 1 -} - -func (any *trueAny) ToFloat32() float32 { - return 1 -} - -func (any *trueAny) ToFloat64() float64 { - return 1 -} - -func (any *trueAny) ToString() string { - return "true" -} - -func (any *trueAny) WriteTo(stream *Stream) { - stream.WriteTrue() -} - -func (any *trueAny) Parse() *Iterator { - return nil -} - -func (any *trueAny) GetInterface() interface{} { - return true -} - -func (any *trueAny) ValueType() ValueType { - return BoolValue -} - -func (any *trueAny) MustBeValid() Any { - return any -} - -type falseAny struct { - baseAny -} - -func (any *falseAny) LastError() error { - return nil -} - -func (any *falseAny) ToBool() bool { - return false -} - -func (any *falseAny) ToInt() int { - return 0 -} - -func (any *falseAny) ToInt32() int32 { - return 0 -} - -func (any *falseAny) ToInt64() int64 { - return 0 -} - -func (any *falseAny) ToUint() uint { - return 0 -} - -func (any *falseAny) ToUint32() uint32 { - return 0 -} - -func (any *falseAny) ToUint64() uint64 { - return 0 -} - -func (any *falseAny) ToFloat32() float32 { - return 0 -} - -func (any *falseAny) ToFloat64() float64 { - return 0 -} - -func (any *falseAny) ToString() string { - return "false" -} - -func (any *falseAny) WriteTo(stream *Stream) { - stream.WriteFalse() -} - -func (any *falseAny) Parse() *Iterator { - return nil -} - -func (any *falseAny) GetInterface() interface{} { - return false -} - -func (any *falseAny) ValueType() ValueType { - return BoolValue -} - -func (any *falseAny) MustBeValid() Any { - return any -} diff --git a/vendor/github.com/json-iterator/go/any_float.go b/vendor/github.com/json-iterator/go/any_float.go deleted file mode 100644 index 35fdb094..00000000 --- a/vendor/github.com/json-iterator/go/any_float.go +++ /dev/null @@ -1,83 +0,0 @@ -package jsoniter - -import ( - "strconv" -) - -type floatAny struct { - baseAny - val float64 -} - -func (any *floatAny) Parse() *Iterator { - return nil -} - -func (any *floatAny) ValueType() ValueType { - return NumberValue -} - -func (any *floatAny) MustBeValid() Any { - return any -} - -func (any *floatAny) LastError() error { - return nil -} - -func (any *floatAny) ToBool() bool { - return any.ToFloat64() != 0 -} - -func (any *floatAny) ToInt() int { - return int(any.val) -} - -func (any *floatAny) ToInt32() int32 { - return int32(any.val) -} - -func (any *floatAny) ToInt64() int64 { - return int64(any.val) -} - -func (any *floatAny) ToUint() uint { - if any.val > 0 { - return uint(any.val) - } - return 0 -} - -func (any *floatAny) ToUint32() uint32 { - if any.val > 0 { - return uint32(any.val) - } - return 0 -} - -func (any *floatAny) ToUint64() uint64 { - if any.val > 0 { - return uint64(any.val) - } - return 0 -} - -func (any *floatAny) ToFloat32() float32 { - return float32(any.val) -} - -func (any *floatAny) ToFloat64() float64 { - return any.val -} - -func (any *floatAny) ToString() string { - return strconv.FormatFloat(any.val, 'E', -1, 64) -} - -func (any *floatAny) WriteTo(stream *Stream) { - stream.WriteFloat64(any.val) -} - -func (any *floatAny) GetInterface() interface{} { - return any.val -} diff --git a/vendor/github.com/json-iterator/go/any_int32.go b/vendor/github.com/json-iterator/go/any_int32.go deleted file mode 100644 index 1b56f399..00000000 --- a/vendor/github.com/json-iterator/go/any_int32.go +++ /dev/null @@ -1,74 +0,0 @@ -package jsoniter - -import ( - "strconv" -) - -type int32Any struct { - baseAny - val int32 -} - -func (any *int32Any) LastError() error { - return nil -} - -func (any *int32Any) ValueType() ValueType { - return NumberValue -} - -func (any *int32Any) MustBeValid() Any { - return any -} - -func (any *int32Any) ToBool() bool { - return any.val != 0 -} - -func (any *int32Any) ToInt() int { - return int(any.val) -} - -func (any *int32Any) ToInt32() int32 { - return any.val -} - -func (any *int32Any) ToInt64() int64 { - return int64(any.val) -} - -func (any *int32Any) ToUint() uint { - return uint(any.val) -} - -func (any *int32Any) ToUint32() uint32 { - return uint32(any.val) -} - -func (any *int32Any) ToUint64() uint64 { - return uint64(any.val) -} - -func (any *int32Any) ToFloat32() float32 { - return float32(any.val) -} - -func (any *int32Any) ToFloat64() float64 { - return float64(any.val) -} - -func (any *int32Any) ToString() string { - return strconv.FormatInt(int64(any.val), 10) -} - -func (any *int32Any) WriteTo(stream *Stream) { - stream.WriteInt32(any.val) -} - -func (any *int32Any) Parse() *Iterator { - return nil -} - -func (any *int32Any) GetInterface() interface{} { - return any.val -} diff --git a/vendor/github.com/json-iterator/go/any_int64.go b/vendor/github.com/json-iterator/go/any_int64.go deleted file mode 100644 index c440d72b..00000000 --- a/vendor/github.com/json-iterator/go/any_int64.go +++ /dev/null @@ -1,74 +0,0 @@ -package jsoniter - -import ( - "strconv" -) - -type int64Any struct { - baseAny - val int64 -} - -func (any *int64Any) LastError() error { - return nil -} - -func (any *int64Any) ValueType() ValueType { - return NumberValue -} - -func (any *int64Any) MustBeValid() Any { - return any -} - -func (any *int64Any) ToBool() bool { - return any.val != 0 -} - -func (any *int64Any) ToInt() int { - return int(any.val) -} - -func (any *int64Any) ToInt32() int32 { - return int32(any.val) -} - -func (any *int64Any) ToInt64() int64 { - return any.val -} - -func (any *int64Any) ToUint() uint { - return uint(any.val) -} - -func (any *int64Any) ToUint32() uint32 { - return uint32(any.val) -} - -func (any *int64Any) ToUint64() uint64 { - return uint64(any.val) -} - -func (any *int64Any) ToFloat32() float32 { - return float32(any.val) -} - -func (any *int64Any) ToFloat64() float64 { - return float64(any.val) -} - -func (any *int64Any) ToString() string { - return strconv.FormatInt(any.val, 10) -} - -func (any *int64Any) WriteTo(stream *Stream) { - stream.WriteInt64(any.val) -} - -func (any *int64Any) Parse() *Iterator { - return nil -} - -func (any *int64Any) GetInterface() interface{} { - return any.val -} diff --git a/vendor/github.com/json-iterator/go/any_invalid.go b/vendor/github.com/json-iterator/go/any_invalid.go deleted file mode 100644 index 1d859eac..00000000 --- a/vendor/github.com/json-iterator/go/any_invalid.go +++ /dev/null @@ -1,82 +0,0 @@ -package jsoniter - -import "fmt" - -type invalidAny struct { - baseAny - err error -} - -func newInvalidAny(path []interface{}) *invalidAny { - return &invalidAny{baseAny{}, fmt.Errorf("%v not found", path)} -} - -func (any *invalidAny) LastError() error { - return any.err -} - -func (any *invalidAny) ValueType() ValueType { - return InvalidValue -} - -func (any *invalidAny) MustBeValid() Any { - panic(any.err) -} - -func (any *invalidAny) ToBool() bool { - return false -} - -func (any *invalidAny) ToInt() int { - return 0 -} - -func (any *invalidAny) ToInt32() int32 { - return 0 -} - -func (any *invalidAny) ToInt64() int64 { - return 0 -} - -func (any *invalidAny) ToUint() uint { - return 0 -} - -func (any *invalidAny) ToUint32() uint32 { - return 0 -} - -func (any *invalidAny) ToUint64() uint64 { - return 0 -} - -func (any *invalidAny) ToFloat32() float32 { - return 0 -} - -func (any *invalidAny) ToFloat64() float64 { - return 0 -} - -func (any *invalidAny) ToString() string { - return "" -} - -func (any *invalidAny) WriteTo(stream *Stream) { -} - -func (any *invalidAny) Get(path ...interface{}) Any { - if any.err == nil { - return &invalidAny{baseAny{}, fmt.Errorf("get %v from invalid", path)} - } - return &invalidAny{baseAny{}, fmt.Errorf("%v, get %v from invalid", any.err, path)} -} - -func (any *invalidAny) Parse() *Iterator { - return nil -} - -func (any *invalidAny) GetInterface() interface{} { - return nil -} diff --git a/vendor/github.com/json-iterator/go/any_nil.go b/vendor/github.com/json-iterator/go/any_nil.go deleted file mode 100644 index d04cb54c..00000000 --- a/vendor/github.com/json-iterator/go/any_nil.go +++ /dev/null @@ -1,69 +0,0 @@ -package jsoniter - -type nilAny struct { - baseAny -} - -func (any *nilAny) LastError() error { - return nil -} - -func (any *nilAny) ValueType() ValueType { - return NilValue -} - -func (any *nilAny) MustBeValid() Any { - return any -} - -func (any *nilAny) ToBool() bool { - return false -} - -func (any *nilAny) ToInt() int { - return 0 -} - -func (any *nilAny) ToInt32() int32 { - return 0 -} - -func (any *nilAny) ToInt64() int64 { - return 0 -} - -func (any *nilAny) ToUint() uint { - return 0 -} - -func (any *nilAny) ToUint32() uint32 { - return 0 -} - -func (any *nilAny) ToUint64() uint64 { - return 0 -} - -func (any *nilAny) ToFloat32() float32 { - return 0 -} - -func (any *nilAny) ToFloat64() float64 { - return 0 -} - -func (any *nilAny) ToString() string { - return "" -} - -func (any *nilAny) WriteTo(stream *Stream) { - stream.WriteNil() -} - -func (any *nilAny) Parse() *Iterator { - return nil -} - -func (any *nilAny) GetInterface() interface{} { - return nil -} diff --git a/vendor/github.com/json-iterator/go/any_number.go b/vendor/github.com/json-iterator/go/any_number.go deleted file mode 100644 index 9d1e901a..00000000 --- a/vendor/github.com/json-iterator/go/any_number.go +++ /dev/null @@ -1,123 +0,0 @@ -package jsoniter - -import ( - "io" - "unsafe" -) - -type numberLazyAny struct { - baseAny - cfg *frozenConfig - buf []byte - err error -} - -func (any *numberLazyAny) ValueType() ValueType { - return NumberValue -} - -func (any *numberLazyAny) MustBeValid() Any { - return any -} - -func (any *numberLazyAny) LastError() error { - return any.err -} - -func (any *numberLazyAny) ToBool() bool { - return any.ToFloat64() != 0 -} - -func (any *numberLazyAny) ToInt() int { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadInt() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToInt32() int32 { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadInt32() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToInt64() int64 { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadInt64() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToUint() uint { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadUint() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToUint32() uint32 { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadUint32() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToUint64() uint64 { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadUint64() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToFloat32() float32 { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadFloat32() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToFloat64() float64 { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - val := iter.ReadFloat64() - if iter.Error != nil && iter.Error != io.EOF { - any.err = iter.Error - } - return val -} - -func (any *numberLazyAny) ToString() string { - return *(*string)(unsafe.Pointer(&any.buf)) -} - -func (any *numberLazyAny) WriteTo(stream *Stream) { - stream.Write(any.buf) -} - -func (any *numberLazyAny) GetInterface() interface{} { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - return iter.Read() -} diff --git a/vendor/github.com/json-iterator/go/any_object.go b/vendor/github.com/json-iterator/go/any_object.go deleted file mode 100644 index c44ef5c9..00000000 --- a/vendor/github.com/json-iterator/go/any_object.go +++ /dev/null @@ -1,374 +0,0 @@ -package jsoniter - -import ( - "reflect" - "unsafe" -) - -type objectLazyAny struct { - baseAny - cfg *frozenConfig - buf []byte - err error -} - -func (any *objectLazyAny) ValueType() ValueType { - return ObjectValue -} - -func (any *objectLazyAny) MustBeValid() Any { - return any -} - -func (any *objectLazyAny) LastError() error { - return any.err -} - -func (any *objectLazyAny) ToBool() bool { - return true -} - -func (any *objectLazyAny) ToInt() int { - return 0 -} - -func (any *objectLazyAny) ToInt32() int32 { - return 0 -} - -func (any *objectLazyAny) ToInt64() int64 { - return 0 -} - -func (any *objectLazyAny) ToUint() uint { - return 0 -} - -func (any *objectLazyAny) ToUint32() uint32 { - return 0 -} - -func (any *objectLazyAny) ToUint64() uint64 { - return 0 -} - -func (any *objectLazyAny) ToFloat32() float32 { - return 0 -} - -func (any *objectLazyAny) ToFloat64() float64 { - return 0 -} - -func (any *objectLazyAny) ToString() string { - return *(*string)(unsafe.Pointer(&any.buf)) -} - -func (any *objectLazyAny) ToVal(obj interface{}) { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - iter.ReadVal(obj) -} - -func (any *objectLazyAny) Get(path ...interface{}) Any { - if len(path) == 0 { - return any - } - switch firstPath := path[0].(type) { - case string: - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - valueBytes := locateObjectField(iter, firstPath) - if valueBytes == nil { - return newInvalidAny(path) - } - iter.ResetBytes(valueBytes) - return locatePath(iter, path[1:]) - case int32: - if '*' == firstPath { - mappedAll := map[string]Any{} - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - iter.ReadMapCB(func(iter *Iterator, field string) bool { - mapped := locatePath(iter, path[1:]) - if mapped.ValueType() != InvalidValue { - mappedAll[field] = mapped - } - return true - }) - return wrapMap(mappedAll) - } - return newInvalidAny(path) - default: - return newInvalidAny(path) - } -} - -func (any *objectLazyAny) Keys() []string { - keys := []string{} - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - iter.ReadMapCB(func(iter *Iterator, field string) bool { - iter.Skip() - keys = append(keys, field) - return true - }) - return keys -} - -func (any *objectLazyAny) Size() int { - size := 0 - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - iter.ReadObjectCB(func(iter *Iterator, field string) bool { - iter.Skip() - size++ - return true - }) - return size -} - -func (any *objectLazyAny) WriteTo(stream *Stream) { - stream.Write(any.buf) -} - -func (any *objectLazyAny) GetInterface() interface{} { - iter := any.cfg.BorrowIterator(any.buf) - defer any.cfg.ReturnIterator(iter) - return iter.Read() -} - -type objectAny struct { - baseAny - err error - val reflect.Value -} - -func wrapStruct(val interface{}) *objectAny { - return &objectAny{baseAny{}, nil, reflect.ValueOf(val)} -} - -func (any *objectAny) ValueType() ValueType { - return ObjectValue -} - -func (any *objectAny) MustBeValid() Any { - return any -} - -func (any *objectAny) Parse() *Iterator { - return nil -} - -func (any *objectAny) LastError() error { - return any.err -} - -func (any *objectAny) ToBool() bool { - return any.val.NumField() != 0 -} - -func (any *objectAny) ToInt() int { - return 0 -} - -func (any *objectAny) ToInt32() int32 { - return 0 -} - -func (any *objectAny) ToInt64() int64 { - return 0 -} - -func (any *objectAny) ToUint() uint { - return 0 -} - -func (any *objectAny) ToUint32() uint32 { - return 0 -} - -func (any *objectAny) ToUint64() uint64 { - return 0 -} - -func (any *objectAny) ToFloat32() float32 { - return 0 -} - -func (any *objectAny) ToFloat64() float64 { - return 0 -} - -func (any *objectAny) ToString() string { - str, err := MarshalToString(any.val.Interface()) - any.err = err - return str -} - -func (any *objectAny) Get(path ...interface{}) Any { - if len(path) == 0 { - return any - } - switch firstPath := path[0].(type) { - case string: - field := any.val.FieldByName(firstPath) - if !field.IsValid() { - return newInvalidAny(path) - } - return Wrap(field.Interface()) - case int32: - if '*' == firstPath { - mappedAll := map[string]Any{} - for i := 0; i < any.val.NumField(); i++ { - field := any.val.Field(i) - if field.CanInterface() { - mapped := Wrap(field.Interface()).Get(path[1:]...) - if mapped.ValueType() != InvalidValue { - mappedAll[any.val.Type().Field(i).Name] = mapped - } - } - } - return wrapMap(mappedAll) - } - return newInvalidAny(path) - default: - return newInvalidAny(path) - } -} - -func (any *objectAny) Keys() []string { - keys := make([]string, 0, any.val.NumField()) - for i := 0; i < any.val.NumField(); i++ { - keys = append(keys, any.val.Type().Field(i).Name) - } - return keys -} - -func (any *objectAny) Size() int { - return any.val.NumField() -} - -func (any *objectAny) WriteTo(stream *Stream) { - stream.WriteVal(any.val) -} - -func (any *objectAny) GetInterface() interface{} { - return any.val.Interface() -} - -type mapAny struct { - baseAny - err error - val reflect.Value -} - -func wrapMap(val interface{}) *mapAny { - return &mapAny{baseAny{}, nil, reflect.ValueOf(val)} -} - -func (any *mapAny) ValueType() ValueType { - return ObjectValue -} - -func (any *mapAny) MustBeValid() Any { - return any -} - -func (any *mapAny) Parse() *Iterator { - return nil -} - -func (any *mapAny) LastError() error { - return any.err -} - -func (any *mapAny) ToBool() bool { - return true -} - -func (any *mapAny) ToInt() int { - return 0 -} - -func (any *mapAny) ToInt32() int32 { - return 0 -} - -func (any *mapAny) ToInt64() int64 { - return 0 -} - -func (any *mapAny) ToUint() uint { - return 0 -} - -func (any *mapAny) ToUint32() uint32 { - return 0 -} - -func (any *mapAny) ToUint64() uint64 { - return 0 -} - -func (any *mapAny) ToFloat32() float32 { - return 0 -} - -func (any *mapAny) ToFloat64() float64 { - return 0 -} - -func (any *mapAny) ToString() string { - str, err := MarshalToString(any.val.Interface()) - any.err = err - return str -} - -func (any *mapAny) Get(path ...interface{}) Any { - if len(path) == 0 { - return any - } - switch firstPath := path[0].(type) { - case int32: - if '*' == firstPath { - mappedAll := map[string]Any{} - for _, key := range any.val.MapKeys() { - keyAsStr := key.String() - element := Wrap(any.val.MapIndex(key).Interface()) - mapped := element.Get(path[1:]...) - if mapped.ValueType() != InvalidValue { - mappedAll[keyAsStr] = mapped - } - } - return wrapMap(mappedAll) - } - return newInvalidAny(path) - default: - value := any.val.MapIndex(reflect.ValueOf(firstPath)) - if !value.IsValid() { - return newInvalidAny(path) - } - return Wrap(value.Interface()) - } -} - -func (any *mapAny) Keys() []string { - keys := make([]string, 0, any.val.Len()) - for _, key := range any.val.MapKeys() { - keys = append(keys, key.String()) - } - return keys -} - -func (any *mapAny) Size() int { - return any.val.Len() -} - -func (any *mapAny) WriteTo(stream *Stream) { - stream.WriteVal(any.val) -} - -func (any *mapAny) GetInterface() interface{} { - return any.val.Interface() -} diff --git a/vendor/github.com/json-iterator/go/any_str.go b/vendor/github.com/json-iterator/go/any_str.go deleted file mode 100644 index a4b93c78..00000000 --- a/vendor/github.com/json-iterator/go/any_str.go +++ /dev/null @@ -1,166 +0,0 @@ -package jsoniter - -import ( - "fmt" - "strconv" -) - -type stringAny struct { - baseAny - val string -} - -func (any *stringAny) Get(path ...interface{}) Any { - if len(path) == 0 { - return any - } - return &invalidAny{baseAny{}, fmt.Errorf("GetIndex %v from simple value", path)} -} - -func (any *stringAny) Parse() *Iterator { - return nil -} - -func (any *stringAny) ValueType() ValueType { - return StringValue -} - -func (any *stringAny) MustBeValid() Any { - return any -} - -func (any *stringAny) LastError() error { - return nil -} - -func (any *stringAny) ToBool() bool { - str := any.ToString() - if str == "0" { - return false - } - for _, c := range str { - switch c { - case ' ', '\n', '\r', '\t': - default: - return true - } - } - return false -} - -func (any *stringAny) ToInt() int { - return int(any.ToInt64()) - -} - -func (any *stringAny) ToInt32() int32 { - return int32(any.ToInt64()) -} - -func (any *stringAny) ToInt64() int64 { - if any.val == "" { - return 0 - } - - flag := 1 - startPos := 0 - endPos := 0 - if any.val[0] == '+' || any.val[0] == '-' { - startPos = 1 - } - - if any.val[0] == '-' { - flag = -1 - } - - for i := startPos; i < len(any.val); i++ { - if any.val[i] >= '0' && any.val[i] <= '9' { - endPos = i + 1 - } else { - break - } - } - parsed, _ := strconv.ParseInt(any.val[startPos:endPos], 10, 64) - return int64(flag) * parsed -} - -func (any *stringAny) ToUint() uint { - return uint(any.ToUint64()) -} - -func (any *stringAny) ToUint32() uint32 { - return uint32(any.ToUint64()) -} - -func (any *stringAny) ToUint64() uint64 { - if any.val == "" { - return 0 - } - - startPos := 0 - endPos := 0 - - if any.val[0] == '-' { - return 0 - } - if any.val[0] == '+' { - startPos = 1 - } - - for i := startPos; i < len(any.val); i++ { - if any.val[i] >= '0' && any.val[i] <= '9' { - endPos = i + 1 - } else { - break - } - } - parsed, _ := strconv.ParseUint(any.val[startPos:endPos], 10, 64) - return parsed -} - -func (any *stringAny) ToFloat32() float32 { - return float32(any.ToFloat64()) -} - -func (any *stringAny) ToFloat64() float64 { - if len(any.val) == 0 { - return 0 - } - - // first char invalid - if any.val[0] != '+' && any.val[0] != '-' && (any.val[0] > '9' || any.val[0] < '0') { - return 0 - } - - // extract valid num expression from string - // eg 123true => 123, -12.12xxa => -12.12 - endPos := 1 - for i := 1; i < len(any.val); i++ { - if any.val[i] == '.' || any.val[i] == 'e' || any.val[i] == 'E' || any.val[i] == '+' || any.val[i] == '-' { - endPos = i + 1 - continue - } - - // end position is the first char which is not digit - if any.val[i] >= '0' && any.val[i] <= '9' { - endPos = i + 1 - } else { - endPos = i - break - } - } - parsed, _ := strconv.ParseFloat(any.val[:endPos], 64) - return parsed -} - -func (any *stringAny) ToString() string { - return any.val -} - -func (any *stringAny) WriteTo(stream *Stream) { - stream.WriteString(any.val) -} - -func (any *stringAny) GetInterface() interface{} { - return any.val -} diff --git a/vendor/github.com/json-iterator/go/any_uint32.go b/vendor/github.com/json-iterator/go/any_uint32.go deleted file mode 100644 index 656bbd33..00000000 --- a/vendor/github.com/json-iterator/go/any_uint32.go +++ /dev/null @@ -1,74 +0,0 @@ -package jsoniter - -import ( - "strconv" -) - -type uint32Any struct { - baseAny - val uint32 -} - -func (any *uint32Any) LastError() error { - return nil -} - -func (any *uint32Any) ValueType() ValueType { - return NumberValue -} - -func (any *uint32Any) MustBeValid() Any { - return any -} - -func (any *uint32Any) ToBool() bool { - return any.val != 0 -} - -func (any *uint32Any) ToInt() int { - return int(any.val) -} - -func (any *uint32Any) ToInt32() int32 { - return int32(any.val) -} - -func (any *uint32Any) ToInt64() int64 { - return int64(any.val) -} - -func (any *uint32Any) ToUint() uint { - return uint(any.val) -} - -func (any *uint32Any) ToUint32() uint32 { - return any.val -} - -func (any *uint32Any) ToUint64() uint64 { - return uint64(any.val) -} - -func (any *uint32Any) ToFloat32() float32 { - return float32(any.val) -} - -func (any *uint32Any) ToFloat64() float64 { - return float64(any.val) -} - -func (any *uint32Any) ToString() string { - return strconv.FormatInt(int64(any.val), 10) -} - -func (any *uint32Any) WriteTo(stream *Stream) { - stream.WriteUint32(any.val) -} - -func (any *uint32Any) Parse() *Iterator { - return nil -} - -func (any *uint32Any) GetInterface() interface{} { - return any.val -} diff --git a/vendor/github.com/json-iterator/go/any_uint64.go b/vendor/github.com/json-iterator/go/any_uint64.go deleted file mode 100644 index 7df2fce3..00000000 --- a/vendor/github.com/json-iterator/go/any_uint64.go +++ /dev/null @@ -1,74 +0,0 @@ -package jsoniter - -import ( - "strconv" -) - -type uint64Any struct { - baseAny - val uint64 -} - -func (any *uint64Any) LastError() error { - return nil -} - -func (any *uint64Any) ValueType() ValueType { - return NumberValue -} - -func (any *uint64Any) MustBeValid() Any { - return any -} - -func (any *uint64Any) ToBool() bool { - return any.val != 0 -} - -func (any *uint64Any) ToInt() int { - return int(any.val) -} - -func (any *uint64Any) ToInt32() int32 { - return int32(any.val) -} - -func (any *uint64Any) ToInt64() int64 { - return int64(any.val) -} - -func (any *uint64Any) ToUint() uint { - return uint(any.val) -} - -func (any *uint64Any) ToUint32() uint32 { - return uint32(any.val) -} - -func (any *uint64Any) ToUint64() uint64 { - return any.val -} - -func (any *uint64Any) ToFloat32() float32 { - return float32(any.val) -} - -func (any *uint64Any) ToFloat64() float64 { - return float64(any.val) -} - -func (any *uint64Any) ToString() string { - return strconv.FormatUint(any.val, 10) -} - -func (any *uint64Any) WriteTo(stream *Stream) { - stream.WriteUint64(any.val) -} - -func (any *uint64Any) Parse() *Iterator { - return nil -} - -func (any *uint64Any) GetInterface() interface{} { - return any.val -} diff --git a/vendor/github.com/json-iterator/go/build.sh b/vendor/github.com/json-iterator/go/build.sh deleted file mode 100644 index b45ef688..00000000 --- a/vendor/github.com/json-iterator/go/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e -set -x - -if [ ! -d /tmp/build-golang/src/github.com/json-iterator ]; then - mkdir -p /tmp/build-golang/src/github.com/json-iterator - ln -s $PWD /tmp/build-golang/src/github.com/json-iterator/go -fi -export GOPATH=/tmp/build-golang -go get -u github.com/golang/dep/cmd/dep -cd /tmp/build-golang/src/github.com/json-iterator/go -exec $GOPATH/bin/dep ensure -update diff --git a/vendor/github.com/json-iterator/go/config.go b/vendor/github.com/json-iterator/go/config.go deleted file mode 100644 index 8c58fcba..00000000 --- a/vendor/github.com/json-iterator/go/config.go +++ /dev/null @@ -1,375 +0,0 @@ -package jsoniter - -import ( - "encoding/json" - "io" - "reflect" - "sync" - "unsafe" - - "github.com/modern-go/concurrent" - "github.com/modern-go/reflect2" -) - -// Config customize how the API should behave. -// The API is created from Config by Froze. -type Config struct { - IndentionStep int - MarshalFloatWith6Digits bool - EscapeHTML bool - SortMapKeys bool - UseNumber bool - DisallowUnknownFields bool - TagKey string - OnlyTaggedField bool - ValidateJsonRawMessage bool - ObjectFieldMustBeSimpleString bool - CaseSensitive bool -} - -// API the public interface of this package. -// Primary Marshal and Unmarshal. -type API interface { - IteratorPool - StreamPool - MarshalToString(v interface{}) (string, error) - Marshal(v interface{}) ([]byte, error) - MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) - UnmarshalFromString(str string, v interface{}) error - Unmarshal(data []byte, v interface{}) error - Get(data []byte, path ...interface{}) Any - NewEncoder(writer io.Writer) *Encoder - NewDecoder(reader io.Reader) *Decoder - Valid(data []byte) bool - RegisterExtension(extension Extension) - DecoderOf(typ reflect2.Type) ValDecoder - EncoderOf(typ reflect2.Type) ValEncoder -} - -// ConfigDefault the default API -var ConfigDefault = Config{ - EscapeHTML: true, -}.Froze() - -// ConfigCompatibleWithStandardLibrary tries to be 100% compatible with standard library behavior -var ConfigCompatibleWithStandardLibrary = Config{ - EscapeHTML: true, - SortMapKeys: true, - ValidateJsonRawMessage: true, -}.Froze() - -// ConfigFastest marshals float with only 6 digits precision -var ConfigFastest = Config{ - EscapeHTML: false, - MarshalFloatWith6Digits: true, // will lose precession - ObjectFieldMustBeSimpleString: true, // do not unescape object field -}.Froze() - -type frozenConfig struct { - configBeforeFrozen Config - sortMapKeys bool - indentionStep int - objectFieldMustBeSimpleString bool - onlyTaggedField bool - disallowUnknownFields bool - decoderCache *concurrent.Map - encoderCache *concurrent.Map - encoderExtension Extension - decoderExtension Extension - extraExtensions []Extension - streamPool *sync.Pool - iteratorPool *sync.Pool - caseSensitive bool -} - -func (cfg *frozenConfig) initCache() { - cfg.decoderCache = concurrent.NewMap() - cfg.encoderCache = concurrent.NewMap() -} - -func (cfg *frozenConfig) addDecoderToCache(cacheKey uintptr, decoder ValDecoder) { - cfg.decoderCache.Store(cacheKey, decoder) -} - -func (cfg *frozenConfig) addEncoderToCache(cacheKey uintptr, encoder ValEncoder) { - cfg.encoderCache.Store(cacheKey, encoder) -} - -func (cfg *frozenConfig) getDecoderFromCache(cacheKey uintptr) ValDecoder { - decoder, found := cfg.decoderCache.Load(cacheKey) - if found { - return decoder.(ValDecoder) - } - return nil -} - -func (cfg *frozenConfig) getEncoderFromCache(cacheKey uintptr) ValEncoder { - encoder, found := cfg.encoderCache.Load(cacheKey) - if found { - return encoder.(ValEncoder) - } - return nil -} - -var cfgCache = concurrent.NewMap() - -func getFrozenConfigFromCache(cfg Config) *frozenConfig { - obj, found := cfgCache.Load(cfg) - if found { - return obj.(*frozenConfig) - } - return nil -} - -func addFrozenConfigToCache(cfg Config, frozenConfig *frozenConfig) { - cfgCache.Store(cfg, frozenConfig) -} - -// Froze forge API from config -func (cfg Config) Froze() API { - api := &frozenConfig{ - sortMapKeys: cfg.SortMapKeys, - indentionStep: cfg.IndentionStep, - objectFieldMustBeSimpleString: cfg.ObjectFieldMustBeSimpleString, - onlyTaggedField: cfg.OnlyTaggedField, - disallowUnknownFields: cfg.DisallowUnknownFields, - caseSensitive: cfg.CaseSensitive, - } - api.streamPool = &sync.Pool{ - New: func() interface{} { - return NewStream(api, nil, 512) - }, - } - api.iteratorPool = &sync.Pool{ - New: func() interface{} { - return NewIterator(api) - }, - } - api.initCache() - encoderExtension := EncoderExtension{} - decoderExtension := DecoderExtension{} - if cfg.MarshalFloatWith6Digits { - api.marshalFloatWith6Digits(encoderExtension) - } - if cfg.EscapeHTML { - api.escapeHTML(encoderExtension) - } - if cfg.UseNumber { - api.useNumber(decoderExtension) - } - if cfg.ValidateJsonRawMessage { - api.validateJsonRawMessage(encoderExtension) - } - api.encoderExtension = encoderExtension - api.decoderExtension = decoderExtension - api.configBeforeFrozen = cfg - return api -} - -func (cfg Config) frozeWithCacheReuse(extraExtensions []Extension) *frozenConfig { - api := getFrozenConfigFromCache(cfg) - if api != nil { - return api - } - api = cfg.Froze().(*frozenConfig) - for _, extension := range extraExtensions { - api.RegisterExtension(extension) - } - addFrozenConfigToCache(cfg, api) - return api -} - -func (cfg *frozenConfig) validateJsonRawMessage(extension EncoderExtension) { - encoder := &funcEncoder{func(ptr unsafe.Pointer, stream *Stream) { - rawMessage := *(*json.RawMessage)(ptr) - iter := cfg.BorrowIterator([]byte(rawMessage)) - iter.Read() - if iter.Error != nil { - stream.WriteRaw("null") - } else { - cfg.ReturnIterator(iter) - stream.WriteRaw(string(rawMessage)) - } - }, func(ptr unsafe.Pointer) bool { - return len(*((*json.RawMessage)(ptr))) == 0 - }} - extension[reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem()] = encoder - extension[reflect2.TypeOfPtr((*RawMessage)(nil)).Elem()] = encoder -} - -func (cfg *frozenConfig) useNumber(extension DecoderExtension) { - extension[reflect2.TypeOfPtr((*interface{})(nil)).Elem()] = &funcDecoder{func(ptr unsafe.Pointer, iter *Iterator) { - exitingValue := *((*interface{})(ptr)) - if exitingValue != nil && reflect.TypeOf(exitingValue).Kind() == reflect.Ptr { - iter.ReadVal(exitingValue) - return - } - if iter.WhatIsNext() == NumberValue { - *((*interface{})(ptr)) = json.Number(iter.readNumberAsString()) - } else { - *((*interface{})(ptr)) = iter.Read() - } - }} -} -func (cfg *frozenConfig) getTagKey() string { - tagKey := cfg.configBeforeFrozen.TagKey - if tagKey == "" { - return "json" - } - return tagKey -} - -func (cfg *frozenConfig) RegisterExtension(extension Extension) { - cfg.extraExtensions = append(cfg.extraExtensions, extension) - copied := cfg.configBeforeFrozen - cfg.configBeforeFrozen = copied -} - -type lossyFloat32Encoder struct { -} - -func (encoder *lossyFloat32Encoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteFloat32Lossy(*((*float32)(ptr))) -} - -func (encoder *lossyFloat32Encoder) IsEmpty(ptr unsafe.Pointer) bool { - return *((*float32)(ptr)) == 0 -} - -type lossyFloat64Encoder struct { -} - -func (encoder *lossyFloat64Encoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteFloat64Lossy(*((*float64)(ptr))) -} - -func (encoder *lossyFloat64Encoder) IsEmpty(ptr unsafe.Pointer) bool { - return *((*float64)(ptr)) == 0 -} - -// EnableLossyFloatMarshalling keeps 10**(-6) precision -// for float variables for better performance. -func (cfg *frozenConfig) marshalFloatWith6Digits(extension EncoderExtension) { - // for better performance - extension[reflect2.TypeOfPtr((*float32)(nil)).Elem()] = &lossyFloat32Encoder{} - extension[reflect2.TypeOfPtr((*float64)(nil)).Elem()] = &lossyFloat64Encoder{} -} - -type htmlEscapedStringEncoder struct { -} - -func (encoder *htmlEscapedStringEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - str := *((*string)(ptr)) - stream.WriteStringWithHTMLEscaped(str) -} - -func (encoder *htmlEscapedStringEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return *((*string)(ptr)) == "" -} - -func (cfg *frozenConfig) escapeHTML(encoderExtension EncoderExtension) { - encoderExtension[reflect2.TypeOfPtr((*string)(nil)).Elem()] = &htmlEscapedStringEncoder{} -} - -func (cfg *frozenConfig) cleanDecoders() { - typeDecoders = map[string]ValDecoder{} - fieldDecoders = map[string]ValDecoder{} - *cfg = *(cfg.configBeforeFrozen.Froze().(*frozenConfig)) -} - -func (cfg *frozenConfig) cleanEncoders() { - typeEncoders = map[string]ValEncoder{} - fieldEncoders = map[string]ValEncoder{} - *cfg = *(cfg.configBeforeFrozen.Froze().(*frozenConfig)) -} - -func (cfg *frozenConfig) MarshalToString(v interface{}) (string, error) { - stream := cfg.BorrowStream(nil) - defer cfg.ReturnStream(stream) - stream.WriteVal(v) - if stream.Error != nil { - return "", stream.Error - } - return string(stream.Buffer()), nil -} - -func (cfg *frozenConfig) Marshal(v interface{}) ([]byte, error) { - stream := cfg.BorrowStream(nil) - defer cfg.ReturnStream(stream) - stream.WriteVal(v) - if stream.Error != nil { - return nil, stream.Error - } - result := stream.Buffer() - copied := make([]byte, len(result)) - copy(copied, result) - return copied, nil -} - -func (cfg *frozenConfig) MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { - if prefix != "" { - panic("prefix is not supported") - } - for _, r := range indent { - if r != ' ' { - panic("indent can only be space") - } - } - newCfg := cfg.configBeforeFrozen - newCfg.IndentionStep = len(indent) - return newCfg.frozeWithCacheReuse(cfg.extraExtensions).Marshal(v) -} - -func (cfg *frozenConfig) UnmarshalFromString(str string, v interface{}) error { - data := []byte(str) - iter := cfg.BorrowIterator(data) - defer cfg.ReturnIterator(iter) - iter.ReadVal(v) - c := iter.nextToken() - if c == 0 { - if iter.Error == io.EOF { - return nil - } - return iter.Error - } - iter.ReportError("Unmarshal", "there are bytes left after unmarshal") - return iter.Error -} - -func (cfg *frozenConfig) Get(data []byte, path ...interface{}) Any { - iter := cfg.BorrowIterator(data) - defer cfg.ReturnIterator(iter) - return locatePath(iter, path) -} - -func (cfg *frozenConfig) Unmarshal(data []byte, v interface{}) error { - iter := cfg.BorrowIterator(data) - defer cfg.ReturnIterator(iter) - iter.ReadVal(v) - c := iter.nextToken() - if c == 0 { - if iter.Error == io.EOF { - return nil - } - return iter.Error - } - iter.ReportError("Unmarshal", "there are bytes left after unmarshal") - return iter.Error -} - -func (cfg *frozenConfig) NewEncoder(writer io.Writer) *Encoder { - stream := NewStream(cfg, writer, 512) - return &Encoder{stream} -} - -func (cfg *frozenConfig) NewDecoder(reader io.Reader) *Decoder { - iter := Parse(cfg, reader, 512) - return &Decoder{iter} -} - -func (cfg *frozenConfig) Valid(data []byte) bool { - iter := cfg.BorrowIterator(data) - defer cfg.ReturnIterator(iter) - iter.Skip() - return iter.Error == nil -} diff --git a/vendor/github.com/json-iterator/go/fuzzy_mode_convert_table.md b/vendor/github.com/json-iterator/go/fuzzy_mode_convert_table.md deleted file mode 100644 index 3095662b..00000000 --- a/vendor/github.com/json-iterator/go/fuzzy_mode_convert_table.md +++ /dev/null @@ -1,7 +0,0 @@ -| json type \ dest type | bool | int | uint | float |string| -| --- | --- | --- | --- |--|--| -| number | positive => true
negative => true
zero => false| 23.2 => 23
-32.1 => -32| 12.1 => 12
-12.1 => 0|as normal|same as origin| -| string | empty string => false
string "0" => false
other strings => true | "123.32" => 123
"-123.4" => -123
"123.23xxxw" => 123
"abcde12" => 0
"-32.1" => -32| 13.2 => 13
-1.1 => 0 |12.1 => 12.1
-12.3 => -12.3
12.4xxa => 12.4
+1.1e2 =>110 |same as origin| -| bool | true => true
false => false| true => 1
false => 0 | true => 1
false => 0 |true => 1
false => 0|true => "true"
false => "false"| -| object | true | 0 | 0 |0|originnal json| -| array | empty array => false
nonempty array => true| [] => 0
[1,2] => 1 | [] => 0
[1,2] => 1 |[] => 0
[1,2] => 1|original json| \ No newline at end of file diff --git a/vendor/github.com/json-iterator/go/iter.go b/vendor/github.com/json-iterator/go/iter.go deleted file mode 100644 index 95ae54fb..00000000 --- a/vendor/github.com/json-iterator/go/iter.go +++ /dev/null @@ -1,322 +0,0 @@ -package jsoniter - -import ( - "encoding/json" - "fmt" - "io" -) - -// ValueType the type for JSON element -type ValueType int - -const ( - // InvalidValue invalid JSON element - InvalidValue ValueType = iota - // StringValue JSON element "string" - StringValue - // NumberValue JSON element 100 or 0.10 - NumberValue - // NilValue JSON element null - NilValue - // BoolValue JSON element true or false - BoolValue - // ArrayValue JSON element [] - ArrayValue - // ObjectValue JSON element {} - ObjectValue -) - -var hexDigits []byte -var valueTypes []ValueType - -func init() { - hexDigits = make([]byte, 256) - for i := 0; i < len(hexDigits); i++ { - hexDigits[i] = 255 - } - for i := '0'; i <= '9'; i++ { - hexDigits[i] = byte(i - '0') - } - for i := 'a'; i <= 'f'; i++ { - hexDigits[i] = byte((i - 'a') + 10) - } - for i := 'A'; i <= 'F'; i++ { - hexDigits[i] = byte((i - 'A') + 10) - } - valueTypes = make([]ValueType, 256) - for i := 0; i < len(valueTypes); i++ { - valueTypes[i] = InvalidValue - } - valueTypes['"'] = StringValue - valueTypes['-'] = NumberValue - valueTypes['0'] = NumberValue - valueTypes['1'] = NumberValue - valueTypes['2'] = NumberValue - valueTypes['3'] = NumberValue - valueTypes['4'] = NumberValue - valueTypes['5'] = NumberValue - valueTypes['6'] = NumberValue - valueTypes['7'] = NumberValue - valueTypes['8'] = NumberValue - valueTypes['9'] = NumberValue - valueTypes['t'] = BoolValue - valueTypes['f'] = BoolValue - valueTypes['n'] = NilValue - valueTypes['['] = ArrayValue - valueTypes['{'] = ObjectValue -} - -// Iterator is a io.Reader like object, with JSON specific read functions. -// Error is not returned as return value, but stored as Error member on this iterator instance. -type Iterator struct { - cfg *frozenConfig - reader io.Reader - buf []byte - head int - tail int - captureStartedAt int - captured []byte - Error error - Attachment interface{} // open for customized decoder -} - -// NewIterator creates an empty Iterator instance -func NewIterator(cfg API) *Iterator { - return &Iterator{ - cfg: cfg.(*frozenConfig), - reader: nil, - buf: nil, - head: 0, - tail: 0, - } -} - -// Parse creates an Iterator instance from io.Reader -func Parse(cfg API, reader io.Reader, bufSize int) *Iterator { - return &Iterator{ - cfg: cfg.(*frozenConfig), - reader: reader, - buf: make([]byte, bufSize), - head: 0, - tail: 0, - } -} - -// ParseBytes creates an Iterator instance from byte array -func ParseBytes(cfg API, input []byte) *Iterator { - return &Iterator{ - cfg: cfg.(*frozenConfig), - reader: nil, - buf: input, - head: 0, - tail: len(input), - } -} - -// ParseString creates an Iterator instance from string -func ParseString(cfg API, input string) *Iterator { - return ParseBytes(cfg, []byte(input)) -} - -// Pool returns a pool can provide more iterator with same configuration -func (iter *Iterator) Pool() IteratorPool { - return iter.cfg -} - -// Reset reuse iterator instance by specifying another reader -func (iter *Iterator) Reset(reader io.Reader) *Iterator { - iter.reader = reader - iter.head = 0 - iter.tail = 0 - return iter -} - -// ResetBytes reuse iterator instance by specifying another byte array as input -func (iter *Iterator) ResetBytes(input []byte) *Iterator { - iter.reader = nil - iter.buf = input - iter.head = 0 - iter.tail = len(input) - return iter -} - -// WhatIsNext gets ValueType of relatively next json element -func (iter *Iterator) WhatIsNext() ValueType { - valueType := valueTypes[iter.nextToken()] - iter.unreadByte() - return valueType -} - -func (iter *Iterator) skipWhitespacesWithoutLoadMore() bool { - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - switch c { - case ' ', '\n', '\t', '\r': - continue - } - iter.head = i - return false - } - return true -} - -func (iter *Iterator) isObjectEnd() bool { - c := iter.nextToken() - if c == ',' { - return false - } - if c == '}' { - return true - } - iter.ReportError("isObjectEnd", "object ended prematurely, unexpected char "+string([]byte{c})) - return true -} - -func (iter *Iterator) nextToken() byte { - // a variation of skip whitespaces, returning the next non-whitespace token - for { - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - switch c { - case ' ', '\n', '\t', '\r': - continue - } - iter.head = i + 1 - return c - } - if !iter.loadMore() { - return 0 - } - } -} - -// ReportError record a error in iterator instance with current position. -func (iter *Iterator) ReportError(operation string, msg string) { - if iter.Error != nil { - if iter.Error != io.EOF { - return - } - } - peekStart := iter.head - 10 - if peekStart < 0 { - peekStart = 0 - } - peekEnd := iter.head + 10 - if peekEnd > iter.tail { - peekEnd = iter.tail - } - parsing := string(iter.buf[peekStart:peekEnd]) - contextStart := iter.head - 50 - if contextStart < 0 { - contextStart = 0 - } - contextEnd := iter.head + 50 - if contextEnd > iter.tail { - contextEnd = iter.tail - } - context := string(iter.buf[contextStart:contextEnd]) - iter.Error = fmt.Errorf("%s: %s, error found in #%v byte of ...|%s|..., bigger context ...|%s|...", - operation, msg, iter.head-peekStart, parsing, context) -} - -// CurrentBuffer gets current buffer as string for debugging purpose -func (iter *Iterator) CurrentBuffer() string { - peekStart := iter.head - 10 - if peekStart < 0 { - peekStart = 0 - } - return fmt.Sprintf("parsing #%v byte, around ...|%s|..., whole buffer ...|%s|...", iter.head, - string(iter.buf[peekStart:iter.head]), string(iter.buf[0:iter.tail])) -} - -func (iter *Iterator) readByte() (ret byte) { - if iter.head == iter.tail { - if iter.loadMore() { - ret = iter.buf[iter.head] - iter.head++ - return ret - } - return 0 - } - ret = iter.buf[iter.head] - iter.head++ - return ret -} - -func (iter *Iterator) loadMore() bool { - if iter.reader == nil { - if iter.Error == nil { - iter.head = iter.tail - iter.Error = io.EOF - } - return false - } - if iter.captured != nil { - iter.captured = append(iter.captured, - iter.buf[iter.captureStartedAt:iter.tail]...) - iter.captureStartedAt = 0 - } - for { - n, err := iter.reader.Read(iter.buf) - if n == 0 { - if err != nil { - if iter.Error == nil { - iter.Error = err - } - return false - } - } else { - iter.head = 0 - iter.tail = n - return true - } - } -} - -func (iter *Iterator) unreadByte() { - if iter.Error != nil { - return - } - iter.head-- - return -} - -// Read read the next JSON element as generic interface{}. -func (iter *Iterator) Read() interface{} { - valueType := iter.WhatIsNext() - switch valueType { - case StringValue: - return iter.ReadString() - case NumberValue: - if iter.cfg.configBeforeFrozen.UseNumber { - return json.Number(iter.readNumberAsString()) - } - return iter.ReadFloat64() - case NilValue: - iter.skipFourBytes('n', 'u', 'l', 'l') - return nil - case BoolValue: - return iter.ReadBool() - case ArrayValue: - arr := []interface{}{} - iter.ReadArrayCB(func(iter *Iterator) bool { - var elem interface{} - iter.ReadVal(&elem) - arr = append(arr, elem) - return true - }) - return arr - case ObjectValue: - obj := map[string]interface{}{} - iter.ReadMapCB(func(Iter *Iterator, field string) bool { - var elem interface{} - iter.ReadVal(&elem) - obj[field] = elem - return true - }) - return obj - default: - iter.ReportError("Read", fmt.Sprintf("unexpected value type: %v", valueType)) - return nil - } -} diff --git a/vendor/github.com/json-iterator/go/iter_array.go b/vendor/github.com/json-iterator/go/iter_array.go deleted file mode 100644 index 6188cb45..00000000 --- a/vendor/github.com/json-iterator/go/iter_array.go +++ /dev/null @@ -1,58 +0,0 @@ -package jsoniter - -// ReadArray read array element, tells if the array has more element to read. -func (iter *Iterator) ReadArray() (ret bool) { - c := iter.nextToken() - switch c { - case 'n': - iter.skipThreeBytes('u', 'l', 'l') - return false // null - case '[': - c = iter.nextToken() - if c != ']' { - iter.unreadByte() - return true - } - return false - case ']': - return false - case ',': - return true - default: - iter.ReportError("ReadArray", "expect [ or , or ] or n, but found "+string([]byte{c})) - return - } -} - -// ReadArrayCB read array with callback -func (iter *Iterator) ReadArrayCB(callback func(*Iterator) bool) (ret bool) { - c := iter.nextToken() - if c == '[' { - c = iter.nextToken() - if c != ']' { - iter.unreadByte() - if !callback(iter) { - return false - } - c = iter.nextToken() - for c == ',' { - if !callback(iter) { - return false - } - c = iter.nextToken() - } - if c != ']' { - iter.ReportError("ReadArrayCB", "expect ] in the end, but found "+string([]byte{c})) - return false - } - return true - } - return true - } - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - return true // null - } - iter.ReportError("ReadArrayCB", "expect [ or n, but found "+string([]byte{c})) - return false -} diff --git a/vendor/github.com/json-iterator/go/iter_float.go b/vendor/github.com/json-iterator/go/iter_float.go deleted file mode 100644 index b9754638..00000000 --- a/vendor/github.com/json-iterator/go/iter_float.go +++ /dev/null @@ -1,339 +0,0 @@ -package jsoniter - -import ( - "encoding/json" - "io" - "math/big" - "strconv" - "strings" - "unsafe" -) - -var floatDigits []int8 - -const invalidCharForNumber = int8(-1) -const endOfNumber = int8(-2) -const dotInNumber = int8(-3) - -func init() { - floatDigits = make([]int8, 256) - for i := 0; i < len(floatDigits); i++ { - floatDigits[i] = invalidCharForNumber - } - for i := int8('0'); i <= int8('9'); i++ { - floatDigits[i] = i - int8('0') - } - floatDigits[','] = endOfNumber - floatDigits[']'] = endOfNumber - floatDigits['}'] = endOfNumber - floatDigits[' '] = endOfNumber - floatDigits['\t'] = endOfNumber - floatDigits['\n'] = endOfNumber - floatDigits['.'] = dotInNumber -} - -// ReadBigFloat read big.Float -func (iter *Iterator) ReadBigFloat() (ret *big.Float) { - str := iter.readNumberAsString() - if iter.Error != nil && iter.Error != io.EOF { - return nil - } - prec := 64 - if len(str) > prec { - prec = len(str) - } - val, _, err := big.ParseFloat(str, 10, uint(prec), big.ToZero) - if err != nil { - iter.Error = err - return nil - } - return val -} - -// ReadBigInt read big.Int -func (iter *Iterator) ReadBigInt() (ret *big.Int) { - str := iter.readNumberAsString() - if iter.Error != nil && iter.Error != io.EOF { - return nil - } - ret = big.NewInt(0) - var success bool - ret, success = ret.SetString(str, 10) - if !success { - iter.ReportError("ReadBigInt", "invalid big int") - return nil - } - return ret -} - -//ReadFloat32 read float32 -func (iter *Iterator) ReadFloat32() (ret float32) { - c := iter.nextToken() - if c == '-' { - return -iter.readPositiveFloat32() - } - iter.unreadByte() - return iter.readPositiveFloat32() -} - -func (iter *Iterator) readPositiveFloat32() (ret float32) { - i := iter.head - // first char - if i == iter.tail { - return iter.readFloat32SlowPath() - } - c := iter.buf[i] - i++ - ind := floatDigits[c] - switch ind { - case invalidCharForNumber: - return iter.readFloat32SlowPath() - case endOfNumber: - iter.ReportError("readFloat32", "empty number") - return - case dotInNumber: - iter.ReportError("readFloat32", "leading dot is invalid") - return - case 0: - if i == iter.tail { - return iter.readFloat32SlowPath() - } - c = iter.buf[i] - switch c { - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - iter.ReportError("readFloat32", "leading zero is invalid") - return - } - } - value := uint64(ind) - // chars before dot -non_decimal_loop: - for ; i < iter.tail; i++ { - c = iter.buf[i] - ind := floatDigits[c] - switch ind { - case invalidCharForNumber: - return iter.readFloat32SlowPath() - case endOfNumber: - iter.head = i - return float32(value) - case dotInNumber: - break non_decimal_loop - } - if value > uint64SafeToMultiple10 { - return iter.readFloat32SlowPath() - } - value = (value << 3) + (value << 1) + uint64(ind) // value = value * 10 + ind; - } - // chars after dot - if c == '.' { - i++ - decimalPlaces := 0 - if i == iter.tail { - return iter.readFloat32SlowPath() - } - for ; i < iter.tail; i++ { - c = iter.buf[i] - ind := floatDigits[c] - switch ind { - case endOfNumber: - if decimalPlaces > 0 && decimalPlaces < len(pow10) { - iter.head = i - return float32(float64(value) / float64(pow10[decimalPlaces])) - } - // too many decimal places - return iter.readFloat32SlowPath() - case invalidCharForNumber, dotInNumber: - return iter.readFloat32SlowPath() - } - decimalPlaces++ - if value > uint64SafeToMultiple10 { - return iter.readFloat32SlowPath() - } - value = (value << 3) + (value << 1) + uint64(ind) - } - } - return iter.readFloat32SlowPath() -} - -func (iter *Iterator) readNumberAsString() (ret string) { - strBuf := [16]byte{} - str := strBuf[0:0] -load_loop: - for { - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - switch c { - case '+', '-', '.', 'e', 'E', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - str = append(str, c) - continue - default: - iter.head = i - break load_loop - } - } - if !iter.loadMore() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - return - } - if len(str) == 0 { - iter.ReportError("readNumberAsString", "invalid number") - } - return *(*string)(unsafe.Pointer(&str)) -} - -func (iter *Iterator) readFloat32SlowPath() (ret float32) { - str := iter.readNumberAsString() - if iter.Error != nil && iter.Error != io.EOF { - return - } - errMsg := validateFloat(str) - if errMsg != "" { - iter.ReportError("readFloat32SlowPath", errMsg) - return - } - val, err := strconv.ParseFloat(str, 32) - if err != nil { - iter.Error = err - return - } - return float32(val) -} - -// ReadFloat64 read float64 -func (iter *Iterator) ReadFloat64() (ret float64) { - c := iter.nextToken() - if c == '-' { - return -iter.readPositiveFloat64() - } - iter.unreadByte() - return iter.readPositiveFloat64() -} - -func (iter *Iterator) readPositiveFloat64() (ret float64) { - i := iter.head - // first char - if i == iter.tail { - return iter.readFloat64SlowPath() - } - c := iter.buf[i] - i++ - ind := floatDigits[c] - switch ind { - case invalidCharForNumber: - return iter.readFloat64SlowPath() - case endOfNumber: - iter.ReportError("readFloat64", "empty number") - return - case dotInNumber: - iter.ReportError("readFloat64", "leading dot is invalid") - return - case 0: - if i == iter.tail { - return iter.readFloat64SlowPath() - } - c = iter.buf[i] - switch c { - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - iter.ReportError("readFloat64", "leading zero is invalid") - return - } - } - value := uint64(ind) - // chars before dot -non_decimal_loop: - for ; i < iter.tail; i++ { - c = iter.buf[i] - ind := floatDigits[c] - switch ind { - case invalidCharForNumber: - return iter.readFloat64SlowPath() - case endOfNumber: - iter.head = i - return float64(value) - case dotInNumber: - break non_decimal_loop - } - if value > uint64SafeToMultiple10 { - return iter.readFloat64SlowPath() - } - value = (value << 3) + (value << 1) + uint64(ind) // value = value * 10 + ind; - } - // chars after dot - if c == '.' { - i++ - decimalPlaces := 0 - if i == iter.tail { - return iter.readFloat64SlowPath() - } - for ; i < iter.tail; i++ { - c = iter.buf[i] - ind := floatDigits[c] - switch ind { - case endOfNumber: - if decimalPlaces > 0 && decimalPlaces < len(pow10) { - iter.head = i - return float64(value) / float64(pow10[decimalPlaces]) - } - // too many decimal places - return iter.readFloat64SlowPath() - case invalidCharForNumber, dotInNumber: - return iter.readFloat64SlowPath() - } - decimalPlaces++ - if value > uint64SafeToMultiple10 { - return iter.readFloat64SlowPath() - } - value = (value << 3) + (value << 1) + uint64(ind) - } - } - return iter.readFloat64SlowPath() -} - -func (iter *Iterator) readFloat64SlowPath() (ret float64) { - str := iter.readNumberAsString() - if iter.Error != nil && iter.Error != io.EOF { - return - } - errMsg := validateFloat(str) - if errMsg != "" { - iter.ReportError("readFloat64SlowPath", errMsg) - return - } - val, err := strconv.ParseFloat(str, 64) - if err != nil { - iter.Error = err - return - } - return val -} - -func validateFloat(str string) string { - // strconv.ParseFloat is not validating `1.` or `1.e1` - if len(str) == 0 { - return "empty number" - } - if str[0] == '-' { - return "-- is not valid" - } - dotPos := strings.IndexByte(str, '.') - if dotPos != -1 { - if dotPos == len(str)-1 { - return "dot can not be last character" - } - switch str[dotPos+1] { - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - default: - return "missing digit after dot" - } - } - return "" -} - -// ReadNumber read json.Number -func (iter *Iterator) ReadNumber() (ret json.Number) { - return json.Number(iter.readNumberAsString()) -} diff --git a/vendor/github.com/json-iterator/go/iter_int.go b/vendor/github.com/json-iterator/go/iter_int.go deleted file mode 100644 index 21423203..00000000 --- a/vendor/github.com/json-iterator/go/iter_int.go +++ /dev/null @@ -1,345 +0,0 @@ -package jsoniter - -import ( - "math" - "strconv" -) - -var intDigits []int8 - -const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1 -const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1 - -func init() { - intDigits = make([]int8, 256) - for i := 0; i < len(intDigits); i++ { - intDigits[i] = invalidCharForNumber - } - for i := int8('0'); i <= int8('9'); i++ { - intDigits[i] = i - int8('0') - } -} - -// ReadUint read uint -func (iter *Iterator) ReadUint() uint { - if strconv.IntSize == 32 { - return uint(iter.ReadUint32()) - } - return uint(iter.ReadUint64()) -} - -// ReadInt read int -func (iter *Iterator) ReadInt() int { - if strconv.IntSize == 32 { - return int(iter.ReadInt32()) - } - return int(iter.ReadInt64()) -} - -// ReadInt8 read int8 -func (iter *Iterator) ReadInt8() (ret int8) { - c := iter.nextToken() - if c == '-' { - val := iter.readUint32(iter.readByte()) - if val > math.MaxInt8+1 { - iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return -int8(val) - } - val := iter.readUint32(c) - if val > math.MaxInt8 { - iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return int8(val) -} - -// ReadUint8 read uint8 -func (iter *Iterator) ReadUint8() (ret uint8) { - val := iter.readUint32(iter.nextToken()) - if val > math.MaxUint8 { - iter.ReportError("ReadUint8", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return uint8(val) -} - -// ReadInt16 read int16 -func (iter *Iterator) ReadInt16() (ret int16) { - c := iter.nextToken() - if c == '-' { - val := iter.readUint32(iter.readByte()) - if val > math.MaxInt16+1 { - iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return -int16(val) - } - val := iter.readUint32(c) - if val > math.MaxInt16 { - iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return int16(val) -} - -// ReadUint16 read uint16 -func (iter *Iterator) ReadUint16() (ret uint16) { - val := iter.readUint32(iter.nextToken()) - if val > math.MaxUint16 { - iter.ReportError("ReadUint16", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return uint16(val) -} - -// ReadInt32 read int32 -func (iter *Iterator) ReadInt32() (ret int32) { - c := iter.nextToken() - if c == '-' { - val := iter.readUint32(iter.readByte()) - if val > math.MaxInt32+1 { - iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return -int32(val) - } - val := iter.readUint32(c) - if val > math.MaxInt32 { - iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10)) - return - } - return int32(val) -} - -// ReadUint32 read uint32 -func (iter *Iterator) ReadUint32() (ret uint32) { - return iter.readUint32(iter.nextToken()) -} - -func (iter *Iterator) readUint32(c byte) (ret uint32) { - ind := intDigits[c] - if ind == 0 { - iter.assertInteger() - return 0 // single zero - } - if ind == invalidCharForNumber { - iter.ReportError("readUint32", "unexpected character: "+string([]byte{byte(ind)})) - return - } - value := uint32(ind) - if iter.tail-iter.head > 10 { - i := iter.head - ind2 := intDigits[iter.buf[i]] - if ind2 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value - } - i++ - ind3 := intDigits[iter.buf[i]] - if ind3 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*10 + uint32(ind2) - } - //iter.head = i + 1 - //value = value * 100 + uint32(ind2) * 10 + uint32(ind3) - i++ - ind4 := intDigits[iter.buf[i]] - if ind4 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*100 + uint32(ind2)*10 + uint32(ind3) - } - i++ - ind5 := intDigits[iter.buf[i]] - if ind5 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*1000 + uint32(ind2)*100 + uint32(ind3)*10 + uint32(ind4) - } - i++ - ind6 := intDigits[iter.buf[i]] - if ind6 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*10000 + uint32(ind2)*1000 + uint32(ind3)*100 + uint32(ind4)*10 + uint32(ind5) - } - i++ - ind7 := intDigits[iter.buf[i]] - if ind7 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*100000 + uint32(ind2)*10000 + uint32(ind3)*1000 + uint32(ind4)*100 + uint32(ind5)*10 + uint32(ind6) - } - i++ - ind8 := intDigits[iter.buf[i]] - if ind8 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*1000000 + uint32(ind2)*100000 + uint32(ind3)*10000 + uint32(ind4)*1000 + uint32(ind5)*100 + uint32(ind6)*10 + uint32(ind7) - } - i++ - ind9 := intDigits[iter.buf[i]] - value = value*10000000 + uint32(ind2)*1000000 + uint32(ind3)*100000 + uint32(ind4)*10000 + uint32(ind5)*1000 + uint32(ind6)*100 + uint32(ind7)*10 + uint32(ind8) - iter.head = i - if ind9 == invalidCharForNumber { - iter.assertInteger() - return value - } - } - for { - for i := iter.head; i < iter.tail; i++ { - ind = intDigits[iter.buf[i]] - if ind == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value - } - if value > uint32SafeToMultiply10 { - value2 := (value << 3) + (value << 1) + uint32(ind) - if value2 < value { - iter.ReportError("readUint32", "overflow") - return - } - value = value2 - continue - } - value = (value << 3) + (value << 1) + uint32(ind) - } - if !iter.loadMore() { - iter.assertInteger() - return value - } - } -} - -// ReadInt64 read int64 -func (iter *Iterator) ReadInt64() (ret int64) { - c := iter.nextToken() - if c == '-' { - val := iter.readUint64(iter.readByte()) - if val > math.MaxInt64+1 { - iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10)) - return - } - return -int64(val) - } - val := iter.readUint64(c) - if val > math.MaxInt64 { - iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10)) - return - } - return int64(val) -} - -// ReadUint64 read uint64 -func (iter *Iterator) ReadUint64() uint64 { - return iter.readUint64(iter.nextToken()) -} - -func (iter *Iterator) readUint64(c byte) (ret uint64) { - ind := intDigits[c] - if ind == 0 { - iter.assertInteger() - return 0 // single zero - } - if ind == invalidCharForNumber { - iter.ReportError("readUint64", "unexpected character: "+string([]byte{byte(ind)})) - return - } - value := uint64(ind) - if iter.tail-iter.head > 10 { - i := iter.head - ind2 := intDigits[iter.buf[i]] - if ind2 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value - } - i++ - ind3 := intDigits[iter.buf[i]] - if ind3 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*10 + uint64(ind2) - } - //iter.head = i + 1 - //value = value * 100 + uint32(ind2) * 10 + uint32(ind3) - i++ - ind4 := intDigits[iter.buf[i]] - if ind4 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*100 + uint64(ind2)*10 + uint64(ind3) - } - i++ - ind5 := intDigits[iter.buf[i]] - if ind5 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*1000 + uint64(ind2)*100 + uint64(ind3)*10 + uint64(ind4) - } - i++ - ind6 := intDigits[iter.buf[i]] - if ind6 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*10000 + uint64(ind2)*1000 + uint64(ind3)*100 + uint64(ind4)*10 + uint64(ind5) - } - i++ - ind7 := intDigits[iter.buf[i]] - if ind7 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*100000 + uint64(ind2)*10000 + uint64(ind3)*1000 + uint64(ind4)*100 + uint64(ind5)*10 + uint64(ind6) - } - i++ - ind8 := intDigits[iter.buf[i]] - if ind8 == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value*1000000 + uint64(ind2)*100000 + uint64(ind3)*10000 + uint64(ind4)*1000 + uint64(ind5)*100 + uint64(ind6)*10 + uint64(ind7) - } - i++ - ind9 := intDigits[iter.buf[i]] - value = value*10000000 + uint64(ind2)*1000000 + uint64(ind3)*100000 + uint64(ind4)*10000 + uint64(ind5)*1000 + uint64(ind6)*100 + uint64(ind7)*10 + uint64(ind8) - iter.head = i - if ind9 == invalidCharForNumber { - iter.assertInteger() - return value - } - } - for { - for i := iter.head; i < iter.tail; i++ { - ind = intDigits[iter.buf[i]] - if ind == invalidCharForNumber { - iter.head = i - iter.assertInteger() - return value - } - if value > uint64SafeToMultiple10 { - value2 := (value << 3) + (value << 1) + uint64(ind) - if value2 < value { - iter.ReportError("readUint64", "overflow") - return - } - value = value2 - continue - } - value = (value << 3) + (value << 1) + uint64(ind) - } - if !iter.loadMore() { - iter.assertInteger() - return value - } - } -} - -func (iter *Iterator) assertInteger() { - if iter.head < len(iter.buf) && iter.buf[iter.head] == '.' { - iter.ReportError("assertInteger", "can not decode float as int") - } -} diff --git a/vendor/github.com/json-iterator/go/iter_object.go b/vendor/github.com/json-iterator/go/iter_object.go deleted file mode 100644 index 1c575767..00000000 --- a/vendor/github.com/json-iterator/go/iter_object.go +++ /dev/null @@ -1,251 +0,0 @@ -package jsoniter - -import ( - "fmt" - "strings" -) - -// ReadObject read one field from object. -// If object ended, returns empty string. -// Otherwise, returns the field name. -func (iter *Iterator) ReadObject() (ret string) { - c := iter.nextToken() - switch c { - case 'n': - iter.skipThreeBytes('u', 'l', 'l') - return "" // null - case '{': - c = iter.nextToken() - if c == '"' { - iter.unreadByte() - field := iter.ReadString() - c = iter.nextToken() - if c != ':' { - iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) - } - return field - } - if c == '}' { - return "" // end of object - } - iter.ReportError("ReadObject", `expect " after {, but found `+string([]byte{c})) - return - case ',': - field := iter.ReadString() - c = iter.nextToken() - if c != ':' { - iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) - } - return field - case '}': - return "" // end of object - default: - iter.ReportError("ReadObject", fmt.Sprintf(`expect { or , or } or n, but found %s`, string([]byte{c}))) - return - } -} - -// CaseInsensitive -func (iter *Iterator) readFieldHash() int64 { - hash := int64(0x811c9dc5) - c := iter.nextToken() - if c != '"' { - iter.ReportError("readFieldHash", `expect ", but found `+string([]byte{c})) - return 0 - } - for { - for i := iter.head; i < iter.tail; i++ { - // require ascii string and no escape - b := iter.buf[i] - if b == '\\' { - iter.head = i - for _, b := range iter.readStringSlowPath() { - if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive { - b += 'a' - 'A' - } - hash ^= int64(b) - hash *= 0x1000193 - } - c = iter.nextToken() - if c != ':' { - iter.ReportError("readFieldHash", `expect :, but found `+string([]byte{c})) - return 0 - } - return hash - } - if b == '"' { - iter.head = i + 1 - c = iter.nextToken() - if c != ':' { - iter.ReportError("readFieldHash", `expect :, but found `+string([]byte{c})) - return 0 - } - return hash - } - if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive { - b += 'a' - 'A' - } - hash ^= int64(b) - hash *= 0x1000193 - } - if !iter.loadMore() { - iter.ReportError("readFieldHash", `incomplete field name`) - return 0 - } - } -} - -func calcHash(str string, caseSensitive bool) int64 { - if !caseSensitive { - str = strings.ToLower(str) - } - hash := int64(0x811c9dc5) - for _, b := range []byte(str) { - hash ^= int64(b) - hash *= 0x1000193 - } - return int64(hash) -} - -// ReadObjectCB read object with callback, the key is ascii only and field name not copied -func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool { - c := iter.nextToken() - var field string - if c == '{' { - c = iter.nextToken() - if c == '"' { - iter.unreadByte() - field = iter.ReadString() - c = iter.nextToken() - if c != ':' { - iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) - } - if !callback(iter, field) { - return false - } - c = iter.nextToken() - for c == ',' { - field = iter.ReadString() - c = iter.nextToken() - if c != ':' { - iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) - } - if !callback(iter, field) { - return false - } - c = iter.nextToken() - } - if c != '}' { - iter.ReportError("ReadObjectCB", `object not ended with }`) - return false - } - return true - } - if c == '}' { - return true - } - iter.ReportError("ReadObjectCB", `expect " after }, but found `+string([]byte{c})) - return false - } - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - return true // null - } - iter.ReportError("ReadObjectCB", `expect { or n, but found `+string([]byte{c})) - return false -} - -// ReadMapCB read map with callback, the key can be any string -func (iter *Iterator) ReadMapCB(callback func(*Iterator, string) bool) bool { - c := iter.nextToken() - if c == '{' { - c = iter.nextToken() - if c == '"' { - iter.unreadByte() - field := iter.ReadString() - if iter.nextToken() != ':' { - iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) - return false - } - if !callback(iter, field) { - return false - } - c = iter.nextToken() - for c == ',' { - field = iter.ReadString() - if iter.nextToken() != ':' { - iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) - return false - } - if !callback(iter, field) { - return false - } - c = iter.nextToken() - } - if c != '}' { - iter.ReportError("ReadMapCB", `object not ended with }`) - return false - } - return true - } - if c == '}' { - return true - } - iter.ReportError("ReadMapCB", `expect " after }, but found `+string([]byte{c})) - return false - } - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - return true // null - } - iter.ReportError("ReadMapCB", `expect { or n, but found `+string([]byte{c})) - return false -} - -func (iter *Iterator) readObjectStart() bool { - c := iter.nextToken() - if c == '{' { - c = iter.nextToken() - if c == '}' { - return false - } - iter.unreadByte() - return true - } else if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - return false - } - iter.ReportError("readObjectStart", "expect { or n, but found "+string([]byte{c})) - return false -} - -func (iter *Iterator) readObjectFieldAsBytes() (ret []byte) { - str := iter.ReadStringAsSlice() - if iter.skipWhitespacesWithoutLoadMore() { - if ret == nil { - ret = make([]byte, len(str)) - copy(ret, str) - } - if !iter.loadMore() { - return - } - } - if iter.buf[iter.head] != ':' { - iter.ReportError("readObjectFieldAsBytes", "expect : after object field, but found "+string([]byte{iter.buf[iter.head]})) - return - } - iter.head++ - if iter.skipWhitespacesWithoutLoadMore() { - if ret == nil { - ret = make([]byte, len(str)) - copy(ret, str) - } - if !iter.loadMore() { - return - } - } - if ret == nil { - return str - } - return ret -} diff --git a/vendor/github.com/json-iterator/go/iter_skip.go b/vendor/github.com/json-iterator/go/iter_skip.go deleted file mode 100644 index f58beb91..00000000 --- a/vendor/github.com/json-iterator/go/iter_skip.go +++ /dev/null @@ -1,129 +0,0 @@ -package jsoniter - -import "fmt" - -// ReadNil reads a json object as nil and -// returns whether it's a nil or not -func (iter *Iterator) ReadNil() (ret bool) { - c := iter.nextToken() - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') // null - return true - } - iter.unreadByte() - return false -} - -// ReadBool reads a json object as BoolValue -func (iter *Iterator) ReadBool() (ret bool) { - c := iter.nextToken() - if c == 't' { - iter.skipThreeBytes('r', 'u', 'e') - return true - } - if c == 'f' { - iter.skipFourBytes('a', 'l', 's', 'e') - return false - } - iter.ReportError("ReadBool", "expect t or f, but found "+string([]byte{c})) - return -} - -// SkipAndReturnBytes skip next JSON element, and return its content as []byte. -// The []byte can be kept, it is a copy of data. -func (iter *Iterator) SkipAndReturnBytes() []byte { - iter.startCapture(iter.head) - iter.Skip() - return iter.stopCapture() -} - -type captureBuffer struct { - startedAt int - captured []byte -} - -func (iter *Iterator) startCapture(captureStartedAt int) { - if iter.captured != nil { - panic("already in capture mode") - } - iter.captureStartedAt = captureStartedAt - iter.captured = make([]byte, 0, 32) -} - -func (iter *Iterator) stopCapture() []byte { - if iter.captured == nil { - panic("not in capture mode") - } - captured := iter.captured - remaining := iter.buf[iter.captureStartedAt:iter.head] - iter.captureStartedAt = -1 - iter.captured = nil - if len(captured) == 0 { - copied := make([]byte, len(remaining)) - copy(copied, remaining) - return copied - } - captured = append(captured, remaining...) - return captured -} - -// Skip skips a json object and positions to relatively the next json object -func (iter *Iterator) Skip() { - c := iter.nextToken() - switch c { - case '"': - iter.skipString() - case 'n': - iter.skipThreeBytes('u', 'l', 'l') // null - case 't': - iter.skipThreeBytes('r', 'u', 'e') // true - case 'f': - iter.skipFourBytes('a', 'l', 's', 'e') // false - case '0': - iter.unreadByte() - iter.ReadFloat32() - case '-', '1', '2', '3', '4', '5', '6', '7', '8', '9': - iter.skipNumber() - case '[': - iter.skipArray() - case '{': - iter.skipObject() - default: - iter.ReportError("Skip", fmt.Sprintf("do not know how to skip: %v", c)) - return - } -} - -func (iter *Iterator) skipFourBytes(b1, b2, b3, b4 byte) { - if iter.readByte() != b1 { - iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) - return - } - if iter.readByte() != b2 { - iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) - return - } - if iter.readByte() != b3 { - iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) - return - } - if iter.readByte() != b4 { - iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) - return - } -} - -func (iter *Iterator) skipThreeBytes(b1, b2, b3 byte) { - if iter.readByte() != b1 { - iter.ReportError("skipThreeBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3}))) - return - } - if iter.readByte() != b2 { - iter.ReportError("skipThreeBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3}))) - return - } - if iter.readByte() != b3 { - iter.ReportError("skipThreeBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3}))) - return - } -} diff --git a/vendor/github.com/json-iterator/go/iter_skip_sloppy.go b/vendor/github.com/json-iterator/go/iter_skip_sloppy.go deleted file mode 100644 index 8fcdc3b6..00000000 --- a/vendor/github.com/json-iterator/go/iter_skip_sloppy.go +++ /dev/null @@ -1,144 +0,0 @@ -//+build jsoniter_sloppy - -package jsoniter - -// sloppy but faster implementation, do not validate the input json - -func (iter *Iterator) skipNumber() { - for { - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - switch c { - case ' ', '\n', '\r', '\t', ',', '}', ']': - iter.head = i - return - } - } - if !iter.loadMore() { - return - } - } -} - -func (iter *Iterator) skipArray() { - level := 1 - for { - for i := iter.head; i < iter.tail; i++ { - switch iter.buf[i] { - case '"': // If inside string, skip it - iter.head = i + 1 - iter.skipString() - i = iter.head - 1 // it will be i++ soon - case '[': // If open symbol, increase level - level++ - case ']': // If close symbol, increase level - level-- - - // If we have returned to the original level, we're done - if level == 0 { - iter.head = i + 1 - return - } - } - } - if !iter.loadMore() { - iter.ReportError("skipObject", "incomplete array") - return - } - } -} - -func (iter *Iterator) skipObject() { - level := 1 - for { - for i := iter.head; i < iter.tail; i++ { - switch iter.buf[i] { - case '"': // If inside string, skip it - iter.head = i + 1 - iter.skipString() - i = iter.head - 1 // it will be i++ soon - case '{': // If open symbol, increase level - level++ - case '}': // If close symbol, increase level - level-- - - // If we have returned to the original level, we're done - if level == 0 { - iter.head = i + 1 - return - } - } - } - if !iter.loadMore() { - iter.ReportError("skipObject", "incomplete object") - return - } - } -} - -func (iter *Iterator) skipString() { - for { - end, escaped := iter.findStringEnd() - if end == -1 { - if !iter.loadMore() { - iter.ReportError("skipString", "incomplete string") - return - } - if escaped { - iter.head = 1 // skip the first char as last char read is \ - } - } else { - iter.head = end - return - } - } -} - -// adapted from: https://github.com/buger/jsonparser/blob/master/parser.go -// Tries to find the end of string -// Support if string contains escaped quote symbols. -func (iter *Iterator) findStringEnd() (int, bool) { - escaped := false - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - if c == '"' { - if !escaped { - return i + 1, false - } - j := i - 1 - for { - if j < iter.head || iter.buf[j] != '\\' { - // even number of backslashes - // either end of buffer, or " found - return i + 1, true - } - j-- - if j < iter.head || iter.buf[j] != '\\' { - // odd number of backslashes - // it is \" or \\\" - break - } - j-- - } - } else if c == '\\' { - escaped = true - } - } - j := iter.tail - 1 - for { - if j < iter.head || iter.buf[j] != '\\' { - // even number of backslashes - // either end of buffer, or " found - return -1, false // do not end with \ - } - j-- - if j < iter.head || iter.buf[j] != '\\' { - // odd number of backslashes - // it is \" or \\\" - break - } - j-- - - } - return -1, true // end with \ -} diff --git a/vendor/github.com/json-iterator/go/iter_skip_strict.go b/vendor/github.com/json-iterator/go/iter_skip_strict.go deleted file mode 100644 index 6cf66d04..00000000 --- a/vendor/github.com/json-iterator/go/iter_skip_strict.go +++ /dev/null @@ -1,99 +0,0 @@ -//+build !jsoniter_sloppy - -package jsoniter - -import ( - "fmt" - "io" -) - -func (iter *Iterator) skipNumber() { - if !iter.trySkipNumber() { - iter.unreadByte() - if iter.Error != nil && iter.Error != io.EOF { - return - } - iter.ReadFloat64() - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = nil - iter.ReadBigFloat() - } - } -} - -func (iter *Iterator) trySkipNumber() bool { - dotFound := false - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - switch c { - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - case '.': - if dotFound { - iter.ReportError("validateNumber", `more than one dot found in number`) - return true // already failed - } - if i+1 == iter.tail { - return false - } - c = iter.buf[i+1] - switch c { - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - default: - iter.ReportError("validateNumber", `missing digit after dot`) - return true // already failed - } - dotFound = true - default: - switch c { - case ',', ']', '}', ' ', '\t', '\n', '\r': - if iter.head == i { - return false // if - without following digits - } - iter.head = i - return true // must be valid - } - return false // may be invalid - } - } - return false -} - -func (iter *Iterator) skipString() { - if !iter.trySkipString() { - iter.unreadByte() - iter.ReadString() - } -} - -func (iter *Iterator) trySkipString() bool { - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - if c == '"' { - iter.head = i + 1 - return true // valid - } else if c == '\\' { - return false - } else if c < ' ' { - iter.ReportError("trySkipString", - fmt.Sprintf(`invalid control character found: %d`, c)) - return true // already failed - } - } - return false -} - -func (iter *Iterator) skipObject() { - iter.unreadByte() - iter.ReadObjectCB(func(iter *Iterator, field string) bool { - iter.Skip() - return true - }) -} - -func (iter *Iterator) skipArray() { - iter.unreadByte() - iter.ReadArrayCB(func(iter *Iterator) bool { - iter.Skip() - return true - }) -} diff --git a/vendor/github.com/json-iterator/go/iter_str.go b/vendor/github.com/json-iterator/go/iter_str.go deleted file mode 100644 index adc487ea..00000000 --- a/vendor/github.com/json-iterator/go/iter_str.go +++ /dev/null @@ -1,215 +0,0 @@ -package jsoniter - -import ( - "fmt" - "unicode/utf16" -) - -// ReadString read string from iterator -func (iter *Iterator) ReadString() (ret string) { - c := iter.nextToken() - if c == '"' { - for i := iter.head; i < iter.tail; i++ { - c := iter.buf[i] - if c == '"' { - ret = string(iter.buf[iter.head:i]) - iter.head = i + 1 - return ret - } else if c == '\\' { - break - } else if c < ' ' { - iter.ReportError("ReadString", - fmt.Sprintf(`invalid control character found: %d`, c)) - return - } - } - return iter.readStringSlowPath() - } else if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - return "" - } - iter.ReportError("ReadString", `expects " or n, but found `+string([]byte{c})) - return -} - -func (iter *Iterator) readStringSlowPath() (ret string) { - var str []byte - var c byte - for iter.Error == nil { - c = iter.readByte() - if c == '"' { - return string(str) - } - if c == '\\' { - c = iter.readByte() - str = iter.readEscapedChar(c, str) - } else { - str = append(str, c) - } - } - iter.ReportError("readStringSlowPath", "unexpected end of input") - return -} - -func (iter *Iterator) readEscapedChar(c byte, str []byte) []byte { - switch c { - case 'u': - r := iter.readU4() - if utf16.IsSurrogate(r) { - c = iter.readByte() - if iter.Error != nil { - return nil - } - if c != '\\' { - iter.unreadByte() - str = appendRune(str, r) - return str - } - c = iter.readByte() - if iter.Error != nil { - return nil - } - if c != 'u' { - str = appendRune(str, r) - return iter.readEscapedChar(c, str) - } - r2 := iter.readU4() - if iter.Error != nil { - return nil - } - combined := utf16.DecodeRune(r, r2) - if combined == '\uFFFD' { - str = appendRune(str, r) - str = appendRune(str, r2) - } else { - str = appendRune(str, combined) - } - } else { - str = appendRune(str, r) - } - case '"': - str = append(str, '"') - case '\\': - str = append(str, '\\') - case '/': - str = append(str, '/') - case 'b': - str = append(str, '\b') - case 'f': - str = append(str, '\f') - case 'n': - str = append(str, '\n') - case 'r': - str = append(str, '\r') - case 't': - str = append(str, '\t') - default: - iter.ReportError("readEscapedChar", - `invalid escape char after \`) - return nil - } - return str -} - -// ReadStringAsSlice read string from iterator without copying into string form. -// The []byte can not be kept, as it will change after next iterator call. -func (iter *Iterator) ReadStringAsSlice() (ret []byte) { - c := iter.nextToken() - if c == '"' { - for i := iter.head; i < iter.tail; i++ { - // require ascii string and no escape - // for: field name, base64, number - if iter.buf[i] == '"' { - // fast path: reuse the underlying buffer - ret = iter.buf[iter.head:i] - iter.head = i + 1 - return ret - } - } - readLen := iter.tail - iter.head - copied := make([]byte, readLen, readLen*2) - copy(copied, iter.buf[iter.head:iter.tail]) - iter.head = iter.tail - for iter.Error == nil { - c := iter.readByte() - if c == '"' { - return copied - } - copied = append(copied, c) - } - return copied - } - iter.ReportError("ReadStringAsSlice", `expects " or n, but found `+string([]byte{c})) - return -} - -func (iter *Iterator) readU4() (ret rune) { - for i := 0; i < 4; i++ { - c := iter.readByte() - if iter.Error != nil { - return - } - if c >= '0' && c <= '9' { - ret = ret*16 + rune(c-'0') - } else if c >= 'a' && c <= 'f' { - ret = ret*16 + rune(c-'a'+10) - } else if c >= 'A' && c <= 'F' { - ret = ret*16 + rune(c-'A'+10) - } else { - iter.ReportError("readU4", "expects 0~9 or a~f, but found "+string([]byte{c})) - return - } - } - return ret -} - -const ( - t1 = 0x00 // 0000 0000 - tx = 0x80 // 1000 0000 - t2 = 0xC0 // 1100 0000 - t3 = 0xE0 // 1110 0000 - t4 = 0xF0 // 1111 0000 - t5 = 0xF8 // 1111 1000 - - maskx = 0x3F // 0011 1111 - mask2 = 0x1F // 0001 1111 - mask3 = 0x0F // 0000 1111 - mask4 = 0x07 // 0000 0111 - - rune1Max = 1<<7 - 1 - rune2Max = 1<<11 - 1 - rune3Max = 1<<16 - 1 - - surrogateMin = 0xD800 - surrogateMax = 0xDFFF - - maxRune = '\U0010FFFF' // Maximum valid Unicode code point. - runeError = '\uFFFD' // the "error" Rune or "Unicode replacement character" -) - -func appendRune(p []byte, r rune) []byte { - // Negative values are erroneous. Making it unsigned addresses the problem. - switch i := uint32(r); { - case i <= rune1Max: - p = append(p, byte(r)) - return p - case i <= rune2Max: - p = append(p, t2|byte(r>>6)) - p = append(p, tx|byte(r)&maskx) - return p - case i > maxRune, surrogateMin <= i && i <= surrogateMax: - r = runeError - fallthrough - case i <= rune3Max: - p = append(p, t3|byte(r>>12)) - p = append(p, tx|byte(r>>6)&maskx) - p = append(p, tx|byte(r)&maskx) - return p - default: - p = append(p, t4|byte(r>>18)) - p = append(p, tx|byte(r>>12)&maskx) - p = append(p, tx|byte(r>>6)&maskx) - p = append(p, tx|byte(r)&maskx) - return p - } -} diff --git a/vendor/github.com/json-iterator/go/jsoniter.go b/vendor/github.com/json-iterator/go/jsoniter.go deleted file mode 100644 index c2934f91..00000000 --- a/vendor/github.com/json-iterator/go/jsoniter.go +++ /dev/null @@ -1,18 +0,0 @@ -// Package jsoniter implements encoding and decoding of JSON as defined in -// RFC 4627 and provides interfaces with identical syntax of standard lib encoding/json. -// Converting from encoding/json to jsoniter is no more than replacing the package with jsoniter -// and variable type declarations (if any). -// jsoniter interfaces gives 100% compatibility with code using standard lib. -// -// "JSON and Go" -// (https://golang.org/doc/articles/json_and_go.html) -// gives a description of how Marshal/Unmarshal operate -// between arbitrary or predefined json objects and bytes, -// and it applies to jsoniter.Marshal/Unmarshal as well. -// -// Besides, jsoniter.Iterator provides a different set of interfaces -// iterating given bytes/string/reader -// and yielding parsed elements one by one. -// This set of interfaces reads input as required and gives -// better performance. -package jsoniter diff --git a/vendor/github.com/json-iterator/go/pool.go b/vendor/github.com/json-iterator/go/pool.go deleted file mode 100644 index e2389b56..00000000 --- a/vendor/github.com/json-iterator/go/pool.go +++ /dev/null @@ -1,42 +0,0 @@ -package jsoniter - -import ( - "io" -) - -// IteratorPool a thread safe pool of iterators with same configuration -type IteratorPool interface { - BorrowIterator(data []byte) *Iterator - ReturnIterator(iter *Iterator) -} - -// StreamPool a thread safe pool of streams with same configuration -type StreamPool interface { - BorrowStream(writer io.Writer) *Stream - ReturnStream(stream *Stream) -} - -func (cfg *frozenConfig) BorrowStream(writer io.Writer) *Stream { - stream := cfg.streamPool.Get().(*Stream) - stream.Reset(writer) - return stream -} - -func (cfg *frozenConfig) ReturnStream(stream *Stream) { - stream.out = nil - stream.Error = nil - stream.Attachment = nil - cfg.streamPool.Put(stream) -} - -func (cfg *frozenConfig) BorrowIterator(data []byte) *Iterator { - iter := cfg.iteratorPool.Get().(*Iterator) - iter.ResetBytes(data) - return iter -} - -func (cfg *frozenConfig) ReturnIterator(iter *Iterator) { - iter.Error = nil - iter.Attachment = nil - cfg.iteratorPool.Put(iter) -} diff --git a/vendor/github.com/json-iterator/go/reflect.go b/vendor/github.com/json-iterator/go/reflect.go deleted file mode 100644 index 4459e203..00000000 --- a/vendor/github.com/json-iterator/go/reflect.go +++ /dev/null @@ -1,332 +0,0 @@ -package jsoniter - -import ( - "fmt" - "reflect" - "unsafe" - - "github.com/modern-go/reflect2" -) - -// ValDecoder is an internal type registered to cache as needed. -// Don't confuse jsoniter.ValDecoder with json.Decoder. -// For json.Decoder's adapter, refer to jsoniter.AdapterDecoder(todo link). -// -// Reflection on type to create decoders, which is then cached -// Reflection on value is avoided as we can, as the reflect.Value itself will allocate, with following exceptions -// 1. create instance of new value, for example *int will need a int to be allocated -// 2. append to slice, if the existing cap is not enough, allocate will be done using Reflect.New -// 3. assignment to map, both key and value will be reflect.Value -// For a simple struct binding, it will be reflect.Value free and allocation free -type ValDecoder interface { - Decode(ptr unsafe.Pointer, iter *Iterator) -} - -// ValEncoder is an internal type registered to cache as needed. -// Don't confuse jsoniter.ValEncoder with json.Encoder. -// For json.Encoder's adapter, refer to jsoniter.AdapterEncoder(todo godoc link). -type ValEncoder interface { - IsEmpty(ptr unsafe.Pointer) bool - Encode(ptr unsafe.Pointer, stream *Stream) -} - -type checkIsEmpty interface { - IsEmpty(ptr unsafe.Pointer) bool -} - -type ctx struct { - *frozenConfig - prefix string - encoders map[reflect2.Type]ValEncoder - decoders map[reflect2.Type]ValDecoder -} - -func (b *ctx) caseSensitive() bool { - if b.frozenConfig == nil { - // default is case-insensitive - return false - } - return b.frozenConfig.caseSensitive -} - -func (b *ctx) append(prefix string) *ctx { - return &ctx{ - frozenConfig: b.frozenConfig, - prefix: b.prefix + " " + prefix, - encoders: b.encoders, - decoders: b.decoders, - } -} - -// ReadVal copy the underlying JSON into go interface, same as json.Unmarshal -func (iter *Iterator) ReadVal(obj interface{}) { - cacheKey := reflect2.RTypeOf(obj) - decoder := iter.cfg.getDecoderFromCache(cacheKey) - if decoder == nil { - typ := reflect2.TypeOf(obj) - if typ.Kind() != reflect.Ptr { - iter.ReportError("ReadVal", "can only unmarshal into pointer") - return - } - decoder = iter.cfg.DecoderOf(typ) - } - ptr := reflect2.PtrOf(obj) - if ptr == nil { - iter.ReportError("ReadVal", "can not read into nil pointer") - return - } - decoder.Decode(ptr, iter) -} - -// WriteVal copy the go interface into underlying JSON, same as json.Marshal -func (stream *Stream) WriteVal(val interface{}) { - if nil == val { - stream.WriteNil() - return - } - cacheKey := reflect2.RTypeOf(val) - encoder := stream.cfg.getEncoderFromCache(cacheKey) - if encoder == nil { - typ := reflect2.TypeOf(val) - encoder = stream.cfg.EncoderOf(typ) - } - encoder.Encode(reflect2.PtrOf(val), stream) -} - -func (cfg *frozenConfig) DecoderOf(typ reflect2.Type) ValDecoder { - cacheKey := typ.RType() - decoder := cfg.getDecoderFromCache(cacheKey) - if decoder != nil { - return decoder - } - ctx := &ctx{ - frozenConfig: cfg, - prefix: "", - decoders: map[reflect2.Type]ValDecoder{}, - encoders: map[reflect2.Type]ValEncoder{}, - } - ptrType := typ.(*reflect2.UnsafePtrType) - decoder = decoderOfType(ctx, ptrType.Elem()) - cfg.addDecoderToCache(cacheKey, decoder) - return decoder -} - -func decoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder { - decoder := getTypeDecoderFromExtension(ctx, typ) - if decoder != nil { - return decoder - } - decoder = createDecoderOfType(ctx, typ) - for _, extension := range extensions { - decoder = extension.DecorateDecoder(typ, decoder) - } - decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder) - for _, extension := range ctx.extraExtensions { - decoder = extension.DecorateDecoder(typ, decoder) - } - return decoder -} - -func createDecoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder { - decoder := ctx.decoders[typ] - if decoder != nil { - return decoder - } - placeholder := &placeholderDecoder{} - ctx.decoders[typ] = placeholder - decoder = _createDecoderOfType(ctx, typ) - placeholder.decoder = decoder - return decoder -} - -func _createDecoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder { - decoder := createDecoderOfJsonRawMessage(ctx, typ) - if decoder != nil { - return decoder - } - decoder = createDecoderOfJsonNumber(ctx, typ) - if decoder != nil { - return decoder - } - decoder = createDecoderOfMarshaler(ctx, typ) - if decoder != nil { - return decoder - } - decoder = createDecoderOfAny(ctx, typ) - if decoder != nil { - return decoder - } - decoder = createDecoderOfNative(ctx, typ) - if decoder != nil { - return decoder - } - switch typ.Kind() { - case reflect.Interface: - ifaceType, isIFace := typ.(*reflect2.UnsafeIFaceType) - if isIFace { - return &ifaceDecoder{valType: ifaceType} - } - return &efaceDecoder{} - case reflect.Struct: - return decoderOfStruct(ctx, typ) - case reflect.Array: - return decoderOfArray(ctx, typ) - case reflect.Slice: - return decoderOfSlice(ctx, typ) - case reflect.Map: - return decoderOfMap(ctx, typ) - case reflect.Ptr: - return decoderOfOptional(ctx, typ) - default: - return &lazyErrorDecoder{err: fmt.Errorf("%s%s is unsupported type", ctx.prefix, typ.String())} - } -} - -func (cfg *frozenConfig) EncoderOf(typ reflect2.Type) ValEncoder { - cacheKey := typ.RType() - encoder := cfg.getEncoderFromCache(cacheKey) - if encoder != nil { - return encoder - } - ctx := &ctx{ - frozenConfig: cfg, - prefix: "", - decoders: map[reflect2.Type]ValDecoder{}, - encoders: map[reflect2.Type]ValEncoder{}, - } - encoder = encoderOfType(ctx, typ) - if typ.LikePtr() { - encoder = &onePtrEncoder{encoder} - } - cfg.addEncoderToCache(cacheKey, encoder) - return encoder -} - -type onePtrEncoder struct { - encoder ValEncoder -} - -func (encoder *onePtrEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.encoder.IsEmpty(unsafe.Pointer(&ptr)) -} - -func (encoder *onePtrEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - encoder.encoder.Encode(unsafe.Pointer(&ptr), stream) -} - -func encoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder { - encoder := getTypeEncoderFromExtension(ctx, typ) - if encoder != nil { - return encoder - } - encoder = createEncoderOfType(ctx, typ) - for _, extension := range extensions { - encoder = extension.DecorateEncoder(typ, encoder) - } - encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder) - for _, extension := range ctx.extraExtensions { - encoder = extension.DecorateEncoder(typ, encoder) - } - return encoder -} - -func createEncoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder { - encoder := ctx.encoders[typ] - if encoder != nil { - return encoder - } - placeholder := &placeholderEncoder{} - ctx.encoders[typ] = placeholder - encoder = _createEncoderOfType(ctx, typ) - placeholder.encoder = encoder - return encoder -} -func _createEncoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder { - encoder := createEncoderOfJsonRawMessage(ctx, typ) - if encoder != nil { - return encoder - } - encoder = createEncoderOfJsonNumber(ctx, typ) - if encoder != nil { - return encoder - } - encoder = createEncoderOfMarshaler(ctx, typ) - if encoder != nil { - return encoder - } - encoder = createEncoderOfAny(ctx, typ) - if encoder != nil { - return encoder - } - encoder = createEncoderOfNative(ctx, typ) - if encoder != nil { - return encoder - } - kind := typ.Kind() - switch kind { - case reflect.Interface: - return &dynamicEncoder{typ} - case reflect.Struct: - return encoderOfStruct(ctx, typ) - case reflect.Array: - return encoderOfArray(ctx, typ) - case reflect.Slice: - return encoderOfSlice(ctx, typ) - case reflect.Map: - return encoderOfMap(ctx, typ) - case reflect.Ptr: - return encoderOfOptional(ctx, typ) - default: - return &lazyErrorEncoder{err: fmt.Errorf("%s%s is unsupported type", ctx.prefix, typ.String())} - } -} - -type lazyErrorDecoder struct { - err error -} - -func (decoder *lazyErrorDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if iter.WhatIsNext() != NilValue { - if iter.Error == nil { - iter.Error = decoder.err - } - } else { - iter.Skip() - } -} - -type lazyErrorEncoder struct { - err error -} - -func (encoder *lazyErrorEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - if ptr == nil { - stream.WriteNil() - } else if stream.Error == nil { - stream.Error = encoder.err - } -} - -func (encoder *lazyErrorEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return false -} - -type placeholderDecoder struct { - decoder ValDecoder -} - -func (decoder *placeholderDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - decoder.decoder.Decode(ptr, iter) -} - -type placeholderEncoder struct { - encoder ValEncoder -} - -func (encoder *placeholderEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - encoder.encoder.Encode(ptr, stream) -} - -func (encoder *placeholderEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.encoder.IsEmpty(ptr) -} diff --git a/vendor/github.com/json-iterator/go/reflect_array.go b/vendor/github.com/json-iterator/go/reflect_array.go deleted file mode 100644 index 13a0b7b0..00000000 --- a/vendor/github.com/json-iterator/go/reflect_array.go +++ /dev/null @@ -1,104 +0,0 @@ -package jsoniter - -import ( - "fmt" - "github.com/modern-go/reflect2" - "io" - "unsafe" -) - -func decoderOfArray(ctx *ctx, typ reflect2.Type) ValDecoder { - arrayType := typ.(*reflect2.UnsafeArrayType) - decoder := decoderOfType(ctx.append("[arrayElem]"), arrayType.Elem()) - return &arrayDecoder{arrayType, decoder} -} - -func encoderOfArray(ctx *ctx, typ reflect2.Type) ValEncoder { - arrayType := typ.(*reflect2.UnsafeArrayType) - if arrayType.Len() == 0 { - return emptyArrayEncoder{} - } - encoder := encoderOfType(ctx.append("[arrayElem]"), arrayType.Elem()) - return &arrayEncoder{arrayType, encoder} -} - -type emptyArrayEncoder struct{} - -func (encoder emptyArrayEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteEmptyArray() -} - -func (encoder emptyArrayEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return true -} - -type arrayEncoder struct { - arrayType *reflect2.UnsafeArrayType - elemEncoder ValEncoder -} - -func (encoder *arrayEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteArrayStart() - elemPtr := unsafe.Pointer(ptr) - encoder.elemEncoder.Encode(elemPtr, stream) - for i := 1; i < encoder.arrayType.Len(); i++ { - stream.WriteMore() - elemPtr = encoder.arrayType.UnsafeGetIndex(ptr, i) - encoder.elemEncoder.Encode(elemPtr, stream) - } - stream.WriteArrayEnd() - if stream.Error != nil && stream.Error != io.EOF { - stream.Error = fmt.Errorf("%v: %s", encoder.arrayType, stream.Error.Error()) - } -} - -func (encoder *arrayEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return false -} - -type arrayDecoder struct { - arrayType *reflect2.UnsafeArrayType - elemDecoder ValDecoder -} - -func (decoder *arrayDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - decoder.doDecode(ptr, iter) - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v: %s", decoder.arrayType, iter.Error.Error()) - } -} - -func (decoder *arrayDecoder) doDecode(ptr unsafe.Pointer, iter *Iterator) { - c := iter.nextToken() - arrayType := decoder.arrayType - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - return - } - if c != '[' { - iter.ReportError("decode array", "expect [ or n, but found "+string([]byte{c})) - return - } - c = iter.nextToken() - if c == ']' { - return - } - iter.unreadByte() - elemPtr := arrayType.UnsafeGetIndex(ptr, 0) - decoder.elemDecoder.Decode(elemPtr, iter) - length := 1 - for c = iter.nextToken(); c == ','; c = iter.nextToken() { - if length >= arrayType.Len() { - iter.Skip() - continue - } - idx := length - length += 1 - elemPtr = arrayType.UnsafeGetIndex(ptr, idx) - decoder.elemDecoder.Decode(elemPtr, iter) - } - if c != ']' { - iter.ReportError("decode array", "expect ], but found "+string([]byte{c})) - return - } -} diff --git a/vendor/github.com/json-iterator/go/reflect_dynamic.go b/vendor/github.com/json-iterator/go/reflect_dynamic.go deleted file mode 100644 index 8b6bc8b4..00000000 --- a/vendor/github.com/json-iterator/go/reflect_dynamic.go +++ /dev/null @@ -1,70 +0,0 @@ -package jsoniter - -import ( - "github.com/modern-go/reflect2" - "reflect" - "unsafe" -) - -type dynamicEncoder struct { - valType reflect2.Type -} - -func (encoder *dynamicEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - obj := encoder.valType.UnsafeIndirect(ptr) - stream.WriteVal(obj) -} - -func (encoder *dynamicEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.valType.UnsafeIndirect(ptr) == nil -} - -type efaceDecoder struct { -} - -func (decoder *efaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - pObj := (*interface{})(ptr) - obj := *pObj - if obj == nil { - *pObj = iter.Read() - return - } - typ := reflect2.TypeOf(obj) - if typ.Kind() != reflect.Ptr { - *pObj = iter.Read() - return - } - ptrType := typ.(*reflect2.UnsafePtrType) - ptrElemType := ptrType.Elem() - if iter.WhatIsNext() == NilValue { - if ptrElemType.Kind() != reflect.Ptr { - iter.skipFourBytes('n', 'u', 'l', 'l') - *pObj = nil - return - } - } - if reflect2.IsNil(obj) { - obj := ptrElemType.New() - iter.ReadVal(obj) - *pObj = obj - return - } - iter.ReadVal(obj) -} - -type ifaceDecoder struct { - valType *reflect2.UnsafeIFaceType -} - -func (decoder *ifaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if iter.ReadNil() { - decoder.valType.UnsafeSet(ptr, decoder.valType.UnsafeNew()) - return - } - obj := decoder.valType.UnsafeIndirect(ptr) - if reflect2.IsNil(obj) { - iter.ReportError("decode non empty interface", "can not unmarshal into nil") - return - } - iter.ReadVal(obj) -} diff --git a/vendor/github.com/json-iterator/go/reflect_extension.go b/vendor/github.com/json-iterator/go/reflect_extension.go deleted file mode 100644 index 05e8fbf1..00000000 --- a/vendor/github.com/json-iterator/go/reflect_extension.go +++ /dev/null @@ -1,483 +0,0 @@ -package jsoniter - -import ( - "fmt" - "github.com/modern-go/reflect2" - "reflect" - "sort" - "strings" - "unicode" - "unsafe" -) - -var typeDecoders = map[string]ValDecoder{} -var fieldDecoders = map[string]ValDecoder{} -var typeEncoders = map[string]ValEncoder{} -var fieldEncoders = map[string]ValEncoder{} -var extensions = []Extension{} - -// StructDescriptor describe how should we encode/decode the struct -type StructDescriptor struct { - Type reflect2.Type - Fields []*Binding -} - -// GetField get one field from the descriptor by its name. -// Can not use map here to keep field orders. -func (structDescriptor *StructDescriptor) GetField(fieldName string) *Binding { - for _, binding := range structDescriptor.Fields { - if binding.Field.Name() == fieldName { - return binding - } - } - return nil -} - -// Binding describe how should we encode/decode the struct field -type Binding struct { - levels []int - Field reflect2.StructField - FromNames []string - ToNames []string - Encoder ValEncoder - Decoder ValDecoder -} - -// Extension the one for all SPI. Customize encoding/decoding by specifying alternate encoder/decoder. -// Can also rename fields by UpdateStructDescriptor. -type Extension interface { - UpdateStructDescriptor(structDescriptor *StructDescriptor) - CreateMapKeyDecoder(typ reflect2.Type) ValDecoder - CreateMapKeyEncoder(typ reflect2.Type) ValEncoder - CreateDecoder(typ reflect2.Type) ValDecoder - CreateEncoder(typ reflect2.Type) ValEncoder - DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder - DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder -} - -// DummyExtension embed this type get dummy implementation for all methods of Extension -type DummyExtension struct { -} - -// UpdateStructDescriptor No-op -func (extension *DummyExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) { -} - -// CreateMapKeyDecoder No-op -func (extension *DummyExtension) CreateMapKeyDecoder(typ reflect2.Type) ValDecoder { - return nil -} - -// CreateMapKeyEncoder No-op -func (extension *DummyExtension) CreateMapKeyEncoder(typ reflect2.Type) ValEncoder { - return nil -} - -// CreateDecoder No-op -func (extension *DummyExtension) CreateDecoder(typ reflect2.Type) ValDecoder { - return nil -} - -// CreateEncoder No-op -func (extension *DummyExtension) CreateEncoder(typ reflect2.Type) ValEncoder { - return nil -} - -// DecorateDecoder No-op -func (extension *DummyExtension) DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder { - return decoder -} - -// DecorateEncoder No-op -func (extension *DummyExtension) DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder { - return encoder -} - -type EncoderExtension map[reflect2.Type]ValEncoder - -// UpdateStructDescriptor No-op -func (extension EncoderExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) { -} - -// CreateDecoder No-op -func (extension EncoderExtension) CreateDecoder(typ reflect2.Type) ValDecoder { - return nil -} - -// CreateEncoder get encoder from map -func (extension EncoderExtension) CreateEncoder(typ reflect2.Type) ValEncoder { - return extension[typ] -} - -// CreateMapKeyDecoder No-op -func (extension EncoderExtension) CreateMapKeyDecoder(typ reflect2.Type) ValDecoder { - return nil -} - -// CreateMapKeyEncoder No-op -func (extension EncoderExtension) CreateMapKeyEncoder(typ reflect2.Type) ValEncoder { - return nil -} - -// DecorateDecoder No-op -func (extension EncoderExtension) DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder { - return decoder -} - -// DecorateEncoder No-op -func (extension EncoderExtension) DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder { - return encoder -} - -type DecoderExtension map[reflect2.Type]ValDecoder - -// UpdateStructDescriptor No-op -func (extension DecoderExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) { -} - -// CreateMapKeyDecoder No-op -func (extension DecoderExtension) CreateMapKeyDecoder(typ reflect2.Type) ValDecoder { - return nil -} - -// CreateMapKeyEncoder No-op -func (extension DecoderExtension) CreateMapKeyEncoder(typ reflect2.Type) ValEncoder { - return nil -} - -// CreateDecoder get decoder from map -func (extension DecoderExtension) CreateDecoder(typ reflect2.Type) ValDecoder { - return extension[typ] -} - -// CreateEncoder No-op -func (extension DecoderExtension) CreateEncoder(typ reflect2.Type) ValEncoder { - return nil -} - -// DecorateDecoder No-op -func (extension DecoderExtension) DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder { - return decoder -} - -// DecorateEncoder No-op -func (extension DecoderExtension) DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder { - return encoder -} - -type funcDecoder struct { - fun DecoderFunc -} - -func (decoder *funcDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - decoder.fun(ptr, iter) -} - -type funcEncoder struct { - fun EncoderFunc - isEmptyFunc func(ptr unsafe.Pointer) bool -} - -func (encoder *funcEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - encoder.fun(ptr, stream) -} - -func (encoder *funcEncoder) IsEmpty(ptr unsafe.Pointer) bool { - if encoder.isEmptyFunc == nil { - return false - } - return encoder.isEmptyFunc(ptr) -} - -// DecoderFunc the function form of TypeDecoder -type DecoderFunc func(ptr unsafe.Pointer, iter *Iterator) - -// EncoderFunc the function form of TypeEncoder -type EncoderFunc func(ptr unsafe.Pointer, stream *Stream) - -// RegisterTypeDecoderFunc register TypeDecoder for a type with function -func RegisterTypeDecoderFunc(typ string, fun DecoderFunc) { - typeDecoders[typ] = &funcDecoder{fun} -} - -// RegisterTypeDecoder register TypeDecoder for a typ -func RegisterTypeDecoder(typ string, decoder ValDecoder) { - typeDecoders[typ] = decoder -} - -// RegisterFieldDecoderFunc register TypeDecoder for a struct field with function -func RegisterFieldDecoderFunc(typ string, field string, fun DecoderFunc) { - RegisterFieldDecoder(typ, field, &funcDecoder{fun}) -} - -// RegisterFieldDecoder register TypeDecoder for a struct field -func RegisterFieldDecoder(typ string, field string, decoder ValDecoder) { - fieldDecoders[fmt.Sprintf("%s/%s", typ, field)] = decoder -} - -// RegisterTypeEncoderFunc register TypeEncoder for a type with encode/isEmpty function -func RegisterTypeEncoderFunc(typ string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) { - typeEncoders[typ] = &funcEncoder{fun, isEmptyFunc} -} - -// RegisterTypeEncoder register TypeEncoder for a type -func RegisterTypeEncoder(typ string, encoder ValEncoder) { - typeEncoders[typ] = encoder -} - -// RegisterFieldEncoderFunc register TypeEncoder for a struct field with encode/isEmpty function -func RegisterFieldEncoderFunc(typ string, field string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) { - RegisterFieldEncoder(typ, field, &funcEncoder{fun, isEmptyFunc}) -} - -// RegisterFieldEncoder register TypeEncoder for a struct field -func RegisterFieldEncoder(typ string, field string, encoder ValEncoder) { - fieldEncoders[fmt.Sprintf("%s/%s", typ, field)] = encoder -} - -// RegisterExtension register extension -func RegisterExtension(extension Extension) { - extensions = append(extensions, extension) -} - -func getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder { - decoder := _getTypeDecoderFromExtension(ctx, typ) - if decoder != nil { - for _, extension := range extensions { - decoder = extension.DecorateDecoder(typ, decoder) - } - decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder) - for _, extension := range ctx.extraExtensions { - decoder = extension.DecorateDecoder(typ, decoder) - } - } - return decoder -} -func _getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder { - for _, extension := range extensions { - decoder := extension.CreateDecoder(typ) - if decoder != nil { - return decoder - } - } - decoder := ctx.decoderExtension.CreateDecoder(typ) - if decoder != nil { - return decoder - } - for _, extension := range ctx.extraExtensions { - decoder := extension.CreateDecoder(typ) - if decoder != nil { - return decoder - } - } - typeName := typ.String() - decoder = typeDecoders[typeName] - if decoder != nil { - return decoder - } - if typ.Kind() == reflect.Ptr { - ptrType := typ.(*reflect2.UnsafePtrType) - decoder := typeDecoders[ptrType.Elem().String()] - if decoder != nil { - return &OptionalDecoder{ptrType.Elem(), decoder} - } - } - return nil -} - -func getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder { - encoder := _getTypeEncoderFromExtension(ctx, typ) - if encoder != nil { - for _, extension := range extensions { - encoder = extension.DecorateEncoder(typ, encoder) - } - encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder) - for _, extension := range ctx.extraExtensions { - encoder = extension.DecorateEncoder(typ, encoder) - } - } - return encoder -} - -func _getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder { - for _, extension := range extensions { - encoder := extension.CreateEncoder(typ) - if encoder != nil { - return encoder - } - } - encoder := ctx.encoderExtension.CreateEncoder(typ) - if encoder != nil { - return encoder - } - for _, extension := range ctx.extraExtensions { - encoder := extension.CreateEncoder(typ) - if encoder != nil { - return encoder - } - } - typeName := typ.String() - encoder = typeEncoders[typeName] - if encoder != nil { - return encoder - } - if typ.Kind() == reflect.Ptr { - typePtr := typ.(*reflect2.UnsafePtrType) - encoder := typeEncoders[typePtr.Elem().String()] - if encoder != nil { - return &OptionalEncoder{encoder} - } - } - return nil -} - -func describeStruct(ctx *ctx, typ reflect2.Type) *StructDescriptor { - structType := typ.(*reflect2.UnsafeStructType) - embeddedBindings := []*Binding{} - bindings := []*Binding{} - for i := 0; i < structType.NumField(); i++ { - field := structType.Field(i) - tag, hastag := field.Tag().Lookup(ctx.getTagKey()) - if ctx.onlyTaggedField && !hastag && !field.Anonymous() { - continue - } - tagParts := strings.Split(tag, ",") - if tag == "-" { - continue - } - if field.Anonymous() && (tag == "" || tagParts[0] == "") { - if field.Type().Kind() == reflect.Struct { - structDescriptor := describeStruct(ctx, field.Type()) - for _, binding := range structDescriptor.Fields { - binding.levels = append([]int{i}, binding.levels...) - omitempty := binding.Encoder.(*structFieldEncoder).omitempty - binding.Encoder = &structFieldEncoder{field, binding.Encoder, omitempty} - binding.Decoder = &structFieldDecoder{field, binding.Decoder} - embeddedBindings = append(embeddedBindings, binding) - } - continue - } else if field.Type().Kind() == reflect.Ptr { - ptrType := field.Type().(*reflect2.UnsafePtrType) - if ptrType.Elem().Kind() == reflect.Struct { - structDescriptor := describeStruct(ctx, ptrType.Elem()) - for _, binding := range structDescriptor.Fields { - binding.levels = append([]int{i}, binding.levels...) - omitempty := binding.Encoder.(*structFieldEncoder).omitempty - binding.Encoder = &dereferenceEncoder{binding.Encoder} - binding.Encoder = &structFieldEncoder{field, binding.Encoder, omitempty} - binding.Decoder = &dereferenceDecoder{ptrType.Elem(), binding.Decoder} - binding.Decoder = &structFieldDecoder{field, binding.Decoder} - embeddedBindings = append(embeddedBindings, binding) - } - continue - } - } - } - fieldNames := calcFieldNames(field.Name(), tagParts[0], tag) - fieldCacheKey := fmt.Sprintf("%s/%s", typ.String(), field.Name()) - decoder := fieldDecoders[fieldCacheKey] - if decoder == nil { - decoder = decoderOfType(ctx.append(field.Name()), field.Type()) - } - encoder := fieldEncoders[fieldCacheKey] - if encoder == nil { - encoder = encoderOfType(ctx.append(field.Name()), field.Type()) - } - binding := &Binding{ - Field: field, - FromNames: fieldNames, - ToNames: fieldNames, - Decoder: decoder, - Encoder: encoder, - } - binding.levels = []int{i} - bindings = append(bindings, binding) - } - return createStructDescriptor(ctx, typ, bindings, embeddedBindings) -} -func createStructDescriptor(ctx *ctx, typ reflect2.Type, bindings []*Binding, embeddedBindings []*Binding) *StructDescriptor { - structDescriptor := &StructDescriptor{ - Type: typ, - Fields: bindings, - } - for _, extension := range extensions { - extension.UpdateStructDescriptor(structDescriptor) - } - ctx.encoderExtension.UpdateStructDescriptor(structDescriptor) - ctx.decoderExtension.UpdateStructDescriptor(structDescriptor) - for _, extension := range ctx.extraExtensions { - extension.UpdateStructDescriptor(structDescriptor) - } - processTags(structDescriptor, ctx.frozenConfig) - // merge normal & embedded bindings & sort with original order - allBindings := sortableBindings(append(embeddedBindings, structDescriptor.Fields...)) - sort.Sort(allBindings) - structDescriptor.Fields = allBindings - return structDescriptor -} - -type sortableBindings []*Binding - -func (bindings sortableBindings) Len() int { - return len(bindings) -} - -func (bindings sortableBindings) Less(i, j int) bool { - left := bindings[i].levels - right := bindings[j].levels - k := 0 - for { - if left[k] < right[k] { - return true - } else if left[k] > right[k] { - return false - } - k++ - } -} - -func (bindings sortableBindings) Swap(i, j int) { - bindings[i], bindings[j] = bindings[j], bindings[i] -} - -func processTags(structDescriptor *StructDescriptor, cfg *frozenConfig) { - for _, binding := range structDescriptor.Fields { - shouldOmitEmpty := false - tagParts := strings.Split(binding.Field.Tag().Get(cfg.getTagKey()), ",") - for _, tagPart := range tagParts[1:] { - if tagPart == "omitempty" { - shouldOmitEmpty = true - } else if tagPart == "string" { - if binding.Field.Type().Kind() == reflect.String { - binding.Decoder = &stringModeStringDecoder{binding.Decoder, cfg} - binding.Encoder = &stringModeStringEncoder{binding.Encoder, cfg} - } else { - binding.Decoder = &stringModeNumberDecoder{binding.Decoder} - binding.Encoder = &stringModeNumberEncoder{binding.Encoder} - } - } - } - binding.Decoder = &structFieldDecoder{binding.Field, binding.Decoder} - binding.Encoder = &structFieldEncoder{binding.Field, binding.Encoder, shouldOmitEmpty} - } -} - -func calcFieldNames(originalFieldName string, tagProvidedFieldName string, wholeTag string) []string { - // ignore? - if wholeTag == "-" { - return []string{} - } - // rename? - var fieldNames []string - if tagProvidedFieldName == "" { - fieldNames = []string{originalFieldName} - } else { - fieldNames = []string{tagProvidedFieldName} - } - // private? - isNotExported := unicode.IsLower(rune(originalFieldName[0])) - if isNotExported { - fieldNames = []string{} - } - return fieldNames -} diff --git a/vendor/github.com/json-iterator/go/reflect_json_number.go b/vendor/github.com/json-iterator/go/reflect_json_number.go deleted file mode 100644 index 98d45c1e..00000000 --- a/vendor/github.com/json-iterator/go/reflect_json_number.go +++ /dev/null @@ -1,112 +0,0 @@ -package jsoniter - -import ( - "encoding/json" - "github.com/modern-go/reflect2" - "strconv" - "unsafe" -) - -type Number string - -// String returns the literal text of the number. -func (n Number) String() string { return string(n) } - -// Float64 returns the number as a float64. -func (n Number) Float64() (float64, error) { - return strconv.ParseFloat(string(n), 64) -} - -// Int64 returns the number as an int64. -func (n Number) Int64() (int64, error) { - return strconv.ParseInt(string(n), 10, 64) -} - -func CastJsonNumber(val interface{}) (string, bool) { - switch typedVal := val.(type) { - case json.Number: - return string(typedVal), true - case Number: - return string(typedVal), true - } - return "", false -} - -var jsonNumberType = reflect2.TypeOfPtr((*json.Number)(nil)).Elem() -var jsoniterNumberType = reflect2.TypeOfPtr((*Number)(nil)).Elem() - -func createDecoderOfJsonNumber(ctx *ctx, typ reflect2.Type) ValDecoder { - if typ.AssignableTo(jsonNumberType) { - return &jsonNumberCodec{} - } - if typ.AssignableTo(jsoniterNumberType) { - return &jsoniterNumberCodec{} - } - return nil -} - -func createEncoderOfJsonNumber(ctx *ctx, typ reflect2.Type) ValEncoder { - if typ.AssignableTo(jsonNumberType) { - return &jsonNumberCodec{} - } - if typ.AssignableTo(jsoniterNumberType) { - return &jsoniterNumberCodec{} - } - return nil -} - -type jsonNumberCodec struct { -} - -func (codec *jsonNumberCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - switch iter.WhatIsNext() { - case StringValue: - *((*json.Number)(ptr)) = json.Number(iter.ReadString()) - case NilValue: - iter.skipFourBytes('n', 'u', 'l', 'l') - *((*json.Number)(ptr)) = "" - default: - *((*json.Number)(ptr)) = json.Number([]byte(iter.readNumberAsString())) - } -} - -func (codec *jsonNumberCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - number := *((*json.Number)(ptr)) - if len(number) == 0 { - stream.writeByte('0') - } else { - stream.WriteRaw(string(number)) - } -} - -func (codec *jsonNumberCodec) IsEmpty(ptr unsafe.Pointer) bool { - return len(*((*json.Number)(ptr))) == 0 -} - -type jsoniterNumberCodec struct { -} - -func (codec *jsoniterNumberCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - switch iter.WhatIsNext() { - case StringValue: - *((*Number)(ptr)) = Number(iter.ReadString()) - case NilValue: - iter.skipFourBytes('n', 'u', 'l', 'l') - *((*Number)(ptr)) = "" - default: - *((*Number)(ptr)) = Number([]byte(iter.readNumberAsString())) - } -} - -func (codec *jsoniterNumberCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - number := *((*Number)(ptr)) - if len(number) == 0 { - stream.writeByte('0') - } else { - stream.WriteRaw(string(number)) - } -} - -func (codec *jsoniterNumberCodec) IsEmpty(ptr unsafe.Pointer) bool { - return len(*((*Number)(ptr))) == 0 -} diff --git a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go deleted file mode 100644 index f2619936..00000000 --- a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go +++ /dev/null @@ -1,60 +0,0 @@ -package jsoniter - -import ( - "encoding/json" - "github.com/modern-go/reflect2" - "unsafe" -) - -var jsonRawMessageType = reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem() -var jsoniterRawMessageType = reflect2.TypeOfPtr((*RawMessage)(nil)).Elem() - -func createEncoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValEncoder { - if typ == jsonRawMessageType { - return &jsonRawMessageCodec{} - } - if typ == jsoniterRawMessageType { - return &jsoniterRawMessageCodec{} - } - return nil -} - -func createDecoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValDecoder { - if typ == jsonRawMessageType { - return &jsonRawMessageCodec{} - } - if typ == jsoniterRawMessageType { - return &jsoniterRawMessageCodec{} - } - return nil -} - -type jsonRawMessageCodec struct { -} - -func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - *((*json.RawMessage)(ptr)) = json.RawMessage(iter.SkipAndReturnBytes()) -} - -func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) -} - -func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { - return len(*((*json.RawMessage)(ptr))) == 0 -} - -type jsoniterRawMessageCodec struct { -} - -func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - *((*RawMessage)(ptr)) = RawMessage(iter.SkipAndReturnBytes()) -} - -func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteRaw(string(*((*RawMessage)(ptr)))) -} - -func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { - return len(*((*RawMessage)(ptr))) == 0 -} diff --git a/vendor/github.com/json-iterator/go/reflect_map.go b/vendor/github.com/json-iterator/go/reflect_map.go deleted file mode 100644 index 547b4421..00000000 --- a/vendor/github.com/json-iterator/go/reflect_map.go +++ /dev/null @@ -1,338 +0,0 @@ -package jsoniter - -import ( - "fmt" - "github.com/modern-go/reflect2" - "io" - "reflect" - "sort" - "unsafe" -) - -func decoderOfMap(ctx *ctx, typ reflect2.Type) ValDecoder { - mapType := typ.(*reflect2.UnsafeMapType) - keyDecoder := decoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()) - elemDecoder := decoderOfType(ctx.append("[mapElem]"), mapType.Elem()) - return &mapDecoder{ - mapType: mapType, - keyType: mapType.Key(), - elemType: mapType.Elem(), - keyDecoder: keyDecoder, - elemDecoder: elemDecoder, - } -} - -func encoderOfMap(ctx *ctx, typ reflect2.Type) ValEncoder { - mapType := typ.(*reflect2.UnsafeMapType) - if ctx.sortMapKeys { - return &sortKeysMapEncoder{ - mapType: mapType, - keyEncoder: encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()), - elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()), - } - } - return &mapEncoder{ - mapType: mapType, - keyEncoder: encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()), - elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()), - } -} - -func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder { - decoder := ctx.decoderExtension.CreateMapKeyDecoder(typ) - if decoder != nil { - return decoder - } - for _, extension := range ctx.extraExtensions { - decoder := extension.CreateMapKeyDecoder(typ) - if decoder != nil { - return decoder - } - } - switch typ.Kind() { - case reflect.String: - return decoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String)) - case reflect.Bool, - reflect.Uint8, reflect.Int8, - reflect.Uint16, reflect.Int16, - reflect.Uint32, reflect.Int32, - reflect.Uint64, reflect.Int64, - reflect.Uint, reflect.Int, - reflect.Float32, reflect.Float64, - reflect.Uintptr: - typ = reflect2.DefaultTypeOfKind(typ.Kind()) - return &numericMapKeyDecoder{decoderOfType(ctx, typ)} - default: - ptrType := reflect2.PtrTo(typ) - if ptrType.Implements(unmarshalerType) { - return &referenceDecoder{ - &unmarshalerDecoder{ - valType: ptrType, - }, - } - } - if typ.Implements(unmarshalerType) { - return &unmarshalerDecoder{ - valType: typ, - } - } - if ptrType.Implements(textUnmarshalerType) { - return &referenceDecoder{ - &textUnmarshalerDecoder{ - valType: ptrType, - }, - } - } - if typ.Implements(textUnmarshalerType) { - return &textUnmarshalerDecoder{ - valType: typ, - } - } - return &lazyErrorDecoder{err: fmt.Errorf("unsupported map key type: %v", typ)} - } -} - -func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder { - encoder := ctx.encoderExtension.CreateMapKeyEncoder(typ) - if encoder != nil { - return encoder - } - for _, extension := range ctx.extraExtensions { - encoder := extension.CreateMapKeyEncoder(typ) - if encoder != nil { - return encoder - } - } - switch typ.Kind() { - case reflect.String: - return encoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String)) - case reflect.Bool, - reflect.Uint8, reflect.Int8, - reflect.Uint16, reflect.Int16, - reflect.Uint32, reflect.Int32, - reflect.Uint64, reflect.Int64, - reflect.Uint, reflect.Int, - reflect.Float32, reflect.Float64, - reflect.Uintptr: - typ = reflect2.DefaultTypeOfKind(typ.Kind()) - return &numericMapKeyEncoder{encoderOfType(ctx, typ)} - default: - if typ == textMarshalerType { - return &directTextMarshalerEncoder{ - stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), - } - } - if typ.Implements(textMarshalerType) { - return &textMarshalerEncoder{ - valType: typ, - stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), - } - } - if typ.Kind() == reflect.Interface { - return &dynamicMapKeyEncoder{ctx, typ} - } - return &lazyErrorEncoder{err: fmt.Errorf("unsupported map key type: %v", typ)} - } -} - -type mapDecoder struct { - mapType *reflect2.UnsafeMapType - keyType reflect2.Type - elemType reflect2.Type - keyDecoder ValDecoder - elemDecoder ValDecoder -} - -func (decoder *mapDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - mapType := decoder.mapType - c := iter.nextToken() - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - *(*unsafe.Pointer)(ptr) = nil - mapType.UnsafeSet(ptr, mapType.UnsafeNew()) - return - } - if mapType.UnsafeIsNil(ptr) { - mapType.UnsafeSet(ptr, mapType.UnsafeMakeMap(0)) - } - if c != '{' { - iter.ReportError("ReadMapCB", `expect { or n, but found `+string([]byte{c})) - return - } - c = iter.nextToken() - if c == '}' { - return - } - if c != '"' { - iter.ReportError("ReadMapCB", `expect " after }, but found `+string([]byte{c})) - return - } - iter.unreadByte() - key := decoder.keyType.UnsafeNew() - decoder.keyDecoder.Decode(key, iter) - c = iter.nextToken() - if c != ':' { - iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) - return - } - elem := decoder.elemType.UnsafeNew() - decoder.elemDecoder.Decode(elem, iter) - decoder.mapType.UnsafeSetIndex(ptr, key, elem) - for c = iter.nextToken(); c == ','; c = iter.nextToken() { - key := decoder.keyType.UnsafeNew() - decoder.keyDecoder.Decode(key, iter) - c = iter.nextToken() - if c != ':' { - iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) - return - } - elem := decoder.elemType.UnsafeNew() - decoder.elemDecoder.Decode(elem, iter) - decoder.mapType.UnsafeSetIndex(ptr, key, elem) - } - if c != '}' { - iter.ReportError("ReadMapCB", `expect }, but found `+string([]byte{c})) - } -} - -type numericMapKeyDecoder struct { - decoder ValDecoder -} - -func (decoder *numericMapKeyDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - c := iter.nextToken() - if c != '"' { - iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c})) - return - } - decoder.decoder.Decode(ptr, iter) - c = iter.nextToken() - if c != '"' { - iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c})) - return - } -} - -type numericMapKeyEncoder struct { - encoder ValEncoder -} - -func (encoder *numericMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.writeByte('"') - encoder.encoder.Encode(ptr, stream) - stream.writeByte('"') -} - -func (encoder *numericMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return false -} - -type dynamicMapKeyEncoder struct { - ctx *ctx - valType reflect2.Type -} - -func (encoder *dynamicMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - obj := encoder.valType.UnsafeIndirect(ptr) - encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).Encode(reflect2.PtrOf(obj), stream) -} - -func (encoder *dynamicMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool { - obj := encoder.valType.UnsafeIndirect(ptr) - return encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).IsEmpty(reflect2.PtrOf(obj)) -} - -type mapEncoder struct { - mapType *reflect2.UnsafeMapType - keyEncoder ValEncoder - elemEncoder ValEncoder -} - -func (encoder *mapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteObjectStart() - iter := encoder.mapType.UnsafeIterate(ptr) - for i := 0; iter.HasNext(); i++ { - if i != 0 { - stream.WriteMore() - } - key, elem := iter.UnsafeNext() - encoder.keyEncoder.Encode(key, stream) - if stream.indention > 0 { - stream.writeTwoBytes(byte(':'), byte(' ')) - } else { - stream.writeByte(':') - } - encoder.elemEncoder.Encode(elem, stream) - } - stream.WriteObjectEnd() -} - -func (encoder *mapEncoder) IsEmpty(ptr unsafe.Pointer) bool { - iter := encoder.mapType.UnsafeIterate(ptr) - return !iter.HasNext() -} - -type sortKeysMapEncoder struct { - mapType *reflect2.UnsafeMapType - keyEncoder ValEncoder - elemEncoder ValEncoder -} - -func (encoder *sortKeysMapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - if *(*unsafe.Pointer)(ptr) == nil { - stream.WriteNil() - return - } - stream.WriteObjectStart() - mapIter := encoder.mapType.UnsafeIterate(ptr) - subStream := stream.cfg.BorrowStream(nil) - subIter := stream.cfg.BorrowIterator(nil) - keyValues := encodedKeyValues{} - for mapIter.HasNext() { - subStream.buf = make([]byte, 0, 64) - key, elem := mapIter.UnsafeNext() - encoder.keyEncoder.Encode(key, subStream) - if subStream.Error != nil && subStream.Error != io.EOF && stream.Error == nil { - stream.Error = subStream.Error - } - encodedKey := subStream.Buffer() - subIter.ResetBytes(encodedKey) - decodedKey := subIter.ReadString() - if stream.indention > 0 { - subStream.writeTwoBytes(byte(':'), byte(' ')) - } else { - subStream.writeByte(':') - } - encoder.elemEncoder.Encode(elem, subStream) - keyValues = append(keyValues, encodedKV{ - key: decodedKey, - keyValue: subStream.Buffer(), - }) - } - sort.Sort(keyValues) - for i, keyValue := range keyValues { - if i != 0 { - stream.WriteMore() - } - stream.Write(keyValue.keyValue) - } - stream.WriteObjectEnd() - stream.cfg.ReturnStream(subStream) - stream.cfg.ReturnIterator(subIter) -} - -func (encoder *sortKeysMapEncoder) IsEmpty(ptr unsafe.Pointer) bool { - iter := encoder.mapType.UnsafeIterate(ptr) - return !iter.HasNext() -} - -type encodedKeyValues []encodedKV - -type encodedKV struct { - key string - keyValue []byte -} - -func (sv encodedKeyValues) Len() int { return len(sv) } -func (sv encodedKeyValues) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } -func (sv encodedKeyValues) Less(i, j int) bool { return sv[i].key < sv[j].key } diff --git a/vendor/github.com/json-iterator/go/reflect_marshaler.go b/vendor/github.com/json-iterator/go/reflect_marshaler.go deleted file mode 100644 index fea50719..00000000 --- a/vendor/github.com/json-iterator/go/reflect_marshaler.go +++ /dev/null @@ -1,217 +0,0 @@ -package jsoniter - -import ( - "encoding" - "encoding/json" - "github.com/modern-go/reflect2" - "unsafe" -) - -var marshalerType = reflect2.TypeOfPtr((*json.Marshaler)(nil)).Elem() -var unmarshalerType = reflect2.TypeOfPtr((*json.Unmarshaler)(nil)).Elem() -var textMarshalerType = reflect2.TypeOfPtr((*encoding.TextMarshaler)(nil)).Elem() -var textUnmarshalerType = reflect2.TypeOfPtr((*encoding.TextUnmarshaler)(nil)).Elem() - -func createDecoderOfMarshaler(ctx *ctx, typ reflect2.Type) ValDecoder { - ptrType := reflect2.PtrTo(typ) - if ptrType.Implements(unmarshalerType) { - return &referenceDecoder{ - &unmarshalerDecoder{ptrType}, - } - } - if ptrType.Implements(textUnmarshalerType) { - return &referenceDecoder{ - &textUnmarshalerDecoder{ptrType}, - } - } - return nil -} - -func createEncoderOfMarshaler(ctx *ctx, typ reflect2.Type) ValEncoder { - if typ == marshalerType { - checkIsEmpty := createCheckIsEmpty(ctx, typ) - var encoder ValEncoder = &directMarshalerEncoder{ - checkIsEmpty: checkIsEmpty, - } - return encoder - } - if typ.Implements(marshalerType) { - checkIsEmpty := createCheckIsEmpty(ctx, typ) - var encoder ValEncoder = &marshalerEncoder{ - valType: typ, - checkIsEmpty: checkIsEmpty, - } - return encoder - } - ptrType := reflect2.PtrTo(typ) - if ctx.prefix != "" && ptrType.Implements(marshalerType) { - checkIsEmpty := createCheckIsEmpty(ctx, ptrType) - var encoder ValEncoder = &marshalerEncoder{ - valType: ptrType, - checkIsEmpty: checkIsEmpty, - } - return &referenceEncoder{encoder} - } - if typ == textMarshalerType { - checkIsEmpty := createCheckIsEmpty(ctx, typ) - var encoder ValEncoder = &directTextMarshalerEncoder{ - checkIsEmpty: checkIsEmpty, - stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), - } - return encoder - } - if typ.Implements(textMarshalerType) { - checkIsEmpty := createCheckIsEmpty(ctx, typ) - var encoder ValEncoder = &textMarshalerEncoder{ - valType: typ, - stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), - checkIsEmpty: checkIsEmpty, - } - return encoder - } - // if prefix is empty, the type is the root type - if ctx.prefix != "" && ptrType.Implements(textMarshalerType) { - checkIsEmpty := createCheckIsEmpty(ctx, ptrType) - var encoder ValEncoder = &textMarshalerEncoder{ - valType: ptrType, - stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), - checkIsEmpty: checkIsEmpty, - } - return &referenceEncoder{encoder} - } - return nil -} - -type marshalerEncoder struct { - checkIsEmpty checkIsEmpty - valType reflect2.Type -} - -func (encoder *marshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - obj := encoder.valType.UnsafeIndirect(ptr) - if encoder.valType.IsNullable() && reflect2.IsNil(obj) { - stream.WriteNil() - return - } - bytes, err := json.Marshal(obj) - if err != nil { - stream.Error = err - } else { - stream.Write(bytes) - } -} - -func (encoder *marshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.checkIsEmpty.IsEmpty(ptr) -} - -type directMarshalerEncoder struct { - checkIsEmpty checkIsEmpty -} - -func (encoder *directMarshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - marshaler := *(*json.Marshaler)(ptr) - if marshaler == nil { - stream.WriteNil() - return - } - bytes, err := marshaler.MarshalJSON() - if err != nil { - stream.Error = err - } else { - stream.Write(bytes) - } -} - -func (encoder *directMarshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.checkIsEmpty.IsEmpty(ptr) -} - -type textMarshalerEncoder struct { - valType reflect2.Type - stringEncoder ValEncoder - checkIsEmpty checkIsEmpty -} - -func (encoder *textMarshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - obj := encoder.valType.UnsafeIndirect(ptr) - if encoder.valType.IsNullable() && reflect2.IsNil(obj) { - stream.WriteNil() - return - } - marshaler := (obj).(encoding.TextMarshaler) - bytes, err := marshaler.MarshalText() - if err != nil { - stream.Error = err - } else { - str := string(bytes) - encoder.stringEncoder.Encode(unsafe.Pointer(&str), stream) - } -} - -func (encoder *textMarshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.checkIsEmpty.IsEmpty(ptr) -} - -type directTextMarshalerEncoder struct { - stringEncoder ValEncoder - checkIsEmpty checkIsEmpty -} - -func (encoder *directTextMarshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - marshaler := *(*encoding.TextMarshaler)(ptr) - if marshaler == nil { - stream.WriteNil() - return - } - bytes, err := marshaler.MarshalText() - if err != nil { - stream.Error = err - } else { - str := string(bytes) - encoder.stringEncoder.Encode(unsafe.Pointer(&str), stream) - } -} - -func (encoder *directTextMarshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.checkIsEmpty.IsEmpty(ptr) -} - -type unmarshalerDecoder struct { - valType reflect2.Type -} - -func (decoder *unmarshalerDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - valType := decoder.valType - obj := valType.UnsafeIndirect(ptr) - unmarshaler := obj.(json.Unmarshaler) - iter.nextToken() - iter.unreadByte() // skip spaces - bytes := iter.SkipAndReturnBytes() - err := unmarshaler.UnmarshalJSON(bytes) - if err != nil { - iter.ReportError("unmarshalerDecoder", err.Error()) - } -} - -type textUnmarshalerDecoder struct { - valType reflect2.Type -} - -func (decoder *textUnmarshalerDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - valType := decoder.valType - obj := valType.UnsafeIndirect(ptr) - if reflect2.IsNil(obj) { - ptrType := valType.(*reflect2.UnsafePtrType) - elemType := ptrType.Elem() - elem := elemType.UnsafeNew() - ptrType.UnsafeSet(ptr, unsafe.Pointer(&elem)) - obj = valType.UnsafeIndirect(ptr) - } - unmarshaler := (obj).(encoding.TextUnmarshaler) - str := iter.ReadString() - err := unmarshaler.UnmarshalText([]byte(str)) - if err != nil { - iter.ReportError("textUnmarshalerDecoder", err.Error()) - } -} diff --git a/vendor/github.com/json-iterator/go/reflect_native.go b/vendor/github.com/json-iterator/go/reflect_native.go deleted file mode 100644 index 9042eb0c..00000000 --- a/vendor/github.com/json-iterator/go/reflect_native.go +++ /dev/null @@ -1,451 +0,0 @@ -package jsoniter - -import ( - "encoding/base64" - "reflect" - "strconv" - "unsafe" - - "github.com/modern-go/reflect2" -) - -const ptrSize = 32 << uintptr(^uintptr(0)>>63) - -func createEncoderOfNative(ctx *ctx, typ reflect2.Type) ValEncoder { - if typ.Kind() == reflect.Slice && typ.(reflect2.SliceType).Elem().Kind() == reflect.Uint8 { - sliceDecoder := decoderOfSlice(ctx, typ) - return &base64Codec{sliceDecoder: sliceDecoder} - } - typeName := typ.String() - kind := typ.Kind() - switch kind { - case reflect.String: - if typeName != "string" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*string)(nil)).Elem()) - } - return &stringCodec{} - case reflect.Int: - if typeName != "int" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*int)(nil)).Elem()) - } - if strconv.IntSize == 32 { - return &int32Codec{} - } - return &int64Codec{} - case reflect.Int8: - if typeName != "int8" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*int8)(nil)).Elem()) - } - return &int8Codec{} - case reflect.Int16: - if typeName != "int16" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*int16)(nil)).Elem()) - } - return &int16Codec{} - case reflect.Int32: - if typeName != "int32" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*int32)(nil)).Elem()) - } - return &int32Codec{} - case reflect.Int64: - if typeName != "int64" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*int64)(nil)).Elem()) - } - return &int64Codec{} - case reflect.Uint: - if typeName != "uint" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*uint)(nil)).Elem()) - } - if strconv.IntSize == 32 { - return &uint32Codec{} - } - return &uint64Codec{} - case reflect.Uint8: - if typeName != "uint8" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*uint8)(nil)).Elem()) - } - return &uint8Codec{} - case reflect.Uint16: - if typeName != "uint16" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*uint16)(nil)).Elem()) - } - return &uint16Codec{} - case reflect.Uint32: - if typeName != "uint32" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*uint32)(nil)).Elem()) - } - return &uint32Codec{} - case reflect.Uintptr: - if typeName != "uintptr" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*uintptr)(nil)).Elem()) - } - if ptrSize == 32 { - return &uint32Codec{} - } - return &uint64Codec{} - case reflect.Uint64: - if typeName != "uint64" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*uint64)(nil)).Elem()) - } - return &uint64Codec{} - case reflect.Float32: - if typeName != "float32" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*float32)(nil)).Elem()) - } - return &float32Codec{} - case reflect.Float64: - if typeName != "float64" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*float64)(nil)).Elem()) - } - return &float64Codec{} - case reflect.Bool: - if typeName != "bool" { - return encoderOfType(ctx, reflect2.TypeOfPtr((*bool)(nil)).Elem()) - } - return &boolCodec{} - } - return nil -} - -func createDecoderOfNative(ctx *ctx, typ reflect2.Type) ValDecoder { - if typ.Kind() == reflect.Slice && typ.(reflect2.SliceType).Elem().Kind() == reflect.Uint8 { - sliceDecoder := decoderOfSlice(ctx, typ) - return &base64Codec{sliceDecoder: sliceDecoder} - } - typeName := typ.String() - switch typ.Kind() { - case reflect.String: - if typeName != "string" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*string)(nil)).Elem()) - } - return &stringCodec{} - case reflect.Int: - if typeName != "int" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*int)(nil)).Elem()) - } - if strconv.IntSize == 32 { - return &int32Codec{} - } - return &int64Codec{} - case reflect.Int8: - if typeName != "int8" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*int8)(nil)).Elem()) - } - return &int8Codec{} - case reflect.Int16: - if typeName != "int16" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*int16)(nil)).Elem()) - } - return &int16Codec{} - case reflect.Int32: - if typeName != "int32" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*int32)(nil)).Elem()) - } - return &int32Codec{} - case reflect.Int64: - if typeName != "int64" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*int64)(nil)).Elem()) - } - return &int64Codec{} - case reflect.Uint: - if typeName != "uint" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*uint)(nil)).Elem()) - } - if strconv.IntSize == 32 { - return &uint32Codec{} - } - return &uint64Codec{} - case reflect.Uint8: - if typeName != "uint8" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*uint8)(nil)).Elem()) - } - return &uint8Codec{} - case reflect.Uint16: - if typeName != "uint16" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*uint16)(nil)).Elem()) - } - return &uint16Codec{} - case reflect.Uint32: - if typeName != "uint32" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*uint32)(nil)).Elem()) - } - return &uint32Codec{} - case reflect.Uintptr: - if typeName != "uintptr" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*uintptr)(nil)).Elem()) - } - if ptrSize == 32 { - return &uint32Codec{} - } - return &uint64Codec{} - case reflect.Uint64: - if typeName != "uint64" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*uint64)(nil)).Elem()) - } - return &uint64Codec{} - case reflect.Float32: - if typeName != "float32" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*float32)(nil)).Elem()) - } - return &float32Codec{} - case reflect.Float64: - if typeName != "float64" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*float64)(nil)).Elem()) - } - return &float64Codec{} - case reflect.Bool: - if typeName != "bool" { - return decoderOfType(ctx, reflect2.TypeOfPtr((*bool)(nil)).Elem()) - } - return &boolCodec{} - } - return nil -} - -type stringCodec struct { -} - -func (codec *stringCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - *((*string)(ptr)) = iter.ReadString() -} - -func (codec *stringCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - str := *((*string)(ptr)) - stream.WriteString(str) -} - -func (codec *stringCodec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*string)(ptr)) == "" -} - -type int8Codec struct { -} - -func (codec *int8Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*int8)(ptr)) = iter.ReadInt8() - } -} - -func (codec *int8Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteInt8(*((*int8)(ptr))) -} - -func (codec *int8Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*int8)(ptr)) == 0 -} - -type int16Codec struct { -} - -func (codec *int16Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*int16)(ptr)) = iter.ReadInt16() - } -} - -func (codec *int16Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteInt16(*((*int16)(ptr))) -} - -func (codec *int16Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*int16)(ptr)) == 0 -} - -type int32Codec struct { -} - -func (codec *int32Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*int32)(ptr)) = iter.ReadInt32() - } -} - -func (codec *int32Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteInt32(*((*int32)(ptr))) -} - -func (codec *int32Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*int32)(ptr)) == 0 -} - -type int64Codec struct { -} - -func (codec *int64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*int64)(ptr)) = iter.ReadInt64() - } -} - -func (codec *int64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteInt64(*((*int64)(ptr))) -} - -func (codec *int64Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*int64)(ptr)) == 0 -} - -type uint8Codec struct { -} - -func (codec *uint8Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*uint8)(ptr)) = iter.ReadUint8() - } -} - -func (codec *uint8Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteUint8(*((*uint8)(ptr))) -} - -func (codec *uint8Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*uint8)(ptr)) == 0 -} - -type uint16Codec struct { -} - -func (codec *uint16Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*uint16)(ptr)) = iter.ReadUint16() - } -} - -func (codec *uint16Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteUint16(*((*uint16)(ptr))) -} - -func (codec *uint16Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*uint16)(ptr)) == 0 -} - -type uint32Codec struct { -} - -func (codec *uint32Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*uint32)(ptr)) = iter.ReadUint32() - } -} - -func (codec *uint32Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteUint32(*((*uint32)(ptr))) -} - -func (codec *uint32Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*uint32)(ptr)) == 0 -} - -type uint64Codec struct { -} - -func (codec *uint64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*uint64)(ptr)) = iter.ReadUint64() - } -} - -func (codec *uint64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteUint64(*((*uint64)(ptr))) -} - -func (codec *uint64Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*uint64)(ptr)) == 0 -} - -type float32Codec struct { -} - -func (codec *float32Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*float32)(ptr)) = iter.ReadFloat32() - } -} - -func (codec *float32Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteFloat32(*((*float32)(ptr))) -} - -func (codec *float32Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*float32)(ptr)) == 0 -} - -type float64Codec struct { -} - -func (codec *float64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*float64)(ptr)) = iter.ReadFloat64() - } -} - -func (codec *float64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteFloat64(*((*float64)(ptr))) -} - -func (codec *float64Codec) IsEmpty(ptr unsafe.Pointer) bool { - return *((*float64)(ptr)) == 0 -} - -type boolCodec struct { -} - -func (codec *boolCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.ReadNil() { - *((*bool)(ptr)) = iter.ReadBool() - } -} - -func (codec *boolCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteBool(*((*bool)(ptr))) -} - -func (codec *boolCodec) IsEmpty(ptr unsafe.Pointer) bool { - return !(*((*bool)(ptr))) -} - -type base64Codec struct { - sliceType *reflect2.UnsafeSliceType - sliceDecoder ValDecoder -} - -func (codec *base64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { - if iter.ReadNil() { - codec.sliceType.UnsafeSetNil(ptr) - return - } - switch iter.WhatIsNext() { - case StringValue: - src := iter.ReadString() - dst, err := base64.StdEncoding.DecodeString(src) - if err != nil { - iter.ReportError("decode base64", err.Error()) - } else { - codec.sliceType.UnsafeSet(ptr, unsafe.Pointer(&dst)) - } - case ArrayValue: - codec.sliceDecoder.Decode(ptr, iter) - default: - iter.ReportError("base64Codec", "invalid input") - } -} - -func (codec *base64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { - src := *((*[]byte)(ptr)) - if len(src) == 0 { - stream.WriteNil() - return - } - encoding := base64.StdEncoding - stream.writeByte('"') - size := encoding.EncodedLen(len(src)) - buf := make([]byte, size) - encoding.Encode(buf, src) - stream.buf = append(stream.buf, buf...) - stream.writeByte('"') -} - -func (codec *base64Codec) IsEmpty(ptr unsafe.Pointer) bool { - return len(*((*[]byte)(ptr))) == 0 -} diff --git a/vendor/github.com/json-iterator/go/reflect_optional.go b/vendor/github.com/json-iterator/go/reflect_optional.go deleted file mode 100644 index 43ec71d6..00000000 --- a/vendor/github.com/json-iterator/go/reflect_optional.go +++ /dev/null @@ -1,133 +0,0 @@ -package jsoniter - -import ( - "github.com/modern-go/reflect2" - "reflect" - "unsafe" -) - -func decoderOfOptional(ctx *ctx, typ reflect2.Type) ValDecoder { - ptrType := typ.(*reflect2.UnsafePtrType) - elemType := ptrType.Elem() - decoder := decoderOfType(ctx, elemType) - if ctx.prefix == "" && elemType.Kind() == reflect.Ptr { - return &dereferenceDecoder{elemType, decoder} - } - return &OptionalDecoder{elemType, decoder} -} - -func encoderOfOptional(ctx *ctx, typ reflect2.Type) ValEncoder { - ptrType := typ.(*reflect2.UnsafePtrType) - elemType := ptrType.Elem() - elemEncoder := encoderOfType(ctx, elemType) - encoder := &OptionalEncoder{elemEncoder} - return encoder -} - -type OptionalDecoder struct { - ValueType reflect2.Type - ValueDecoder ValDecoder -} - -func (decoder *OptionalDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if iter.ReadNil() { - *((*unsafe.Pointer)(ptr)) = nil - } else { - if *((*unsafe.Pointer)(ptr)) == nil { - //pointer to null, we have to allocate memory to hold the value - newPtr := decoder.ValueType.UnsafeNew() - decoder.ValueDecoder.Decode(newPtr, iter) - *((*unsafe.Pointer)(ptr)) = newPtr - } else { - //reuse existing instance - decoder.ValueDecoder.Decode(*((*unsafe.Pointer)(ptr)), iter) - } - } -} - -type dereferenceDecoder struct { - // only to deference a pointer - valueType reflect2.Type - valueDecoder ValDecoder -} - -func (decoder *dereferenceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if *((*unsafe.Pointer)(ptr)) == nil { - //pointer to null, we have to allocate memory to hold the value - newPtr := decoder.valueType.UnsafeNew() - decoder.valueDecoder.Decode(newPtr, iter) - *((*unsafe.Pointer)(ptr)) = newPtr - } else { - //reuse existing instance - decoder.valueDecoder.Decode(*((*unsafe.Pointer)(ptr)), iter) - } -} - -type OptionalEncoder struct { - ValueEncoder ValEncoder -} - -func (encoder *OptionalEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - if *((*unsafe.Pointer)(ptr)) == nil { - stream.WriteNil() - } else { - encoder.ValueEncoder.Encode(*((*unsafe.Pointer)(ptr)), stream) - } -} - -func (encoder *OptionalEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return *((*unsafe.Pointer)(ptr)) == nil -} - -type dereferenceEncoder struct { - ValueEncoder ValEncoder -} - -func (encoder *dereferenceEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - if *((*unsafe.Pointer)(ptr)) == nil { - stream.WriteNil() - } else { - encoder.ValueEncoder.Encode(*((*unsafe.Pointer)(ptr)), stream) - } -} - -func (encoder *dereferenceEncoder) IsEmpty(ptr unsafe.Pointer) bool { - dePtr := *((*unsafe.Pointer)(ptr)) - if dePtr == nil { - return true - } - return encoder.ValueEncoder.IsEmpty(dePtr) -} - -func (encoder *dereferenceEncoder) IsEmbeddedPtrNil(ptr unsafe.Pointer) bool { - deReferenced := *((*unsafe.Pointer)(ptr)) - if deReferenced == nil { - return true - } - isEmbeddedPtrNil, converted := encoder.ValueEncoder.(IsEmbeddedPtrNil) - if !converted { - return false - } - fieldPtr := unsafe.Pointer(deReferenced) - return isEmbeddedPtrNil.IsEmbeddedPtrNil(fieldPtr) -} - -type referenceEncoder struct { - encoder ValEncoder -} - -func (encoder *referenceEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - encoder.encoder.Encode(unsafe.Pointer(&ptr), stream) -} - -func (encoder *referenceEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.encoder.IsEmpty(unsafe.Pointer(&ptr)) -} - -type referenceDecoder struct { - decoder ValDecoder -} - -func (decoder *referenceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - decoder.decoder.Decode(unsafe.Pointer(&ptr), iter) -} diff --git a/vendor/github.com/json-iterator/go/reflect_slice.go b/vendor/github.com/json-iterator/go/reflect_slice.go deleted file mode 100644 index 9441d79d..00000000 --- a/vendor/github.com/json-iterator/go/reflect_slice.go +++ /dev/null @@ -1,99 +0,0 @@ -package jsoniter - -import ( - "fmt" - "github.com/modern-go/reflect2" - "io" - "unsafe" -) - -func decoderOfSlice(ctx *ctx, typ reflect2.Type) ValDecoder { - sliceType := typ.(*reflect2.UnsafeSliceType) - decoder := decoderOfType(ctx.append("[sliceElem]"), sliceType.Elem()) - return &sliceDecoder{sliceType, decoder} -} - -func encoderOfSlice(ctx *ctx, typ reflect2.Type) ValEncoder { - sliceType := typ.(*reflect2.UnsafeSliceType) - encoder := encoderOfType(ctx.append("[sliceElem]"), sliceType.Elem()) - return &sliceEncoder{sliceType, encoder} -} - -type sliceEncoder struct { - sliceType *reflect2.UnsafeSliceType - elemEncoder ValEncoder -} - -func (encoder *sliceEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - if encoder.sliceType.UnsafeIsNil(ptr) { - stream.WriteNil() - return - } - length := encoder.sliceType.UnsafeLengthOf(ptr) - if length == 0 { - stream.WriteEmptyArray() - return - } - stream.WriteArrayStart() - encoder.elemEncoder.Encode(encoder.sliceType.UnsafeGetIndex(ptr, 0), stream) - for i := 1; i < length; i++ { - stream.WriteMore() - elemPtr := encoder.sliceType.UnsafeGetIndex(ptr, i) - encoder.elemEncoder.Encode(elemPtr, stream) - } - stream.WriteArrayEnd() - if stream.Error != nil && stream.Error != io.EOF { - stream.Error = fmt.Errorf("%v: %s", encoder.sliceType, stream.Error.Error()) - } -} - -func (encoder *sliceEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.sliceType.UnsafeLengthOf(ptr) == 0 -} - -type sliceDecoder struct { - sliceType *reflect2.UnsafeSliceType - elemDecoder ValDecoder -} - -func (decoder *sliceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - decoder.doDecode(ptr, iter) - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v: %s", decoder.sliceType, iter.Error.Error()) - } -} - -func (decoder *sliceDecoder) doDecode(ptr unsafe.Pointer, iter *Iterator) { - c := iter.nextToken() - sliceType := decoder.sliceType - if c == 'n' { - iter.skipThreeBytes('u', 'l', 'l') - sliceType.UnsafeSetNil(ptr) - return - } - if c != '[' { - iter.ReportError("decode slice", "expect [ or n, but found "+string([]byte{c})) - return - } - c = iter.nextToken() - if c == ']' { - sliceType.UnsafeSet(ptr, sliceType.UnsafeMakeSlice(0, 0)) - return - } - iter.unreadByte() - sliceType.UnsafeGrow(ptr, 1) - elemPtr := sliceType.UnsafeGetIndex(ptr, 0) - decoder.elemDecoder.Decode(elemPtr, iter) - length := 1 - for c = iter.nextToken(); c == ','; c = iter.nextToken() { - idx := length - length += 1 - sliceType.UnsafeGrow(ptr, length) - elemPtr = sliceType.UnsafeGetIndex(ptr, idx) - decoder.elemDecoder.Decode(elemPtr, iter) - } - if c != ']' { - iter.ReportError("decode slice", "expect ], but found "+string([]byte{c})) - return - } -} diff --git a/vendor/github.com/json-iterator/go/reflect_struct_decoder.go b/vendor/github.com/json-iterator/go/reflect_struct_decoder.go deleted file mode 100644 index 355d2d11..00000000 --- a/vendor/github.com/json-iterator/go/reflect_struct_decoder.go +++ /dev/null @@ -1,1048 +0,0 @@ -package jsoniter - -import ( - "fmt" - "io" - "strings" - "unsafe" - - "github.com/modern-go/reflect2" -) - -func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder { - bindings := map[string]*Binding{} - structDescriptor := describeStruct(ctx, typ) - for _, binding := range structDescriptor.Fields { - for _, fromName := range binding.FromNames { - old := bindings[fromName] - if old == nil { - bindings[fromName] = binding - continue - } - ignoreOld, ignoreNew := resolveConflictBinding(ctx.frozenConfig, old, binding) - if ignoreOld { - delete(bindings, fromName) - } - if !ignoreNew { - bindings[fromName] = binding - } - } - } - fields := map[string]*structFieldDecoder{} - for k, binding := range bindings { - fields[k] = binding.Decoder.(*structFieldDecoder) - } - - if !ctx.caseSensitive() { - for k, binding := range bindings { - if _, found := fields[strings.ToLower(k)]; !found { - fields[strings.ToLower(k)] = binding.Decoder.(*structFieldDecoder) - } - } - } - - return createStructDecoder(ctx, typ, fields) -} - -func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structFieldDecoder) ValDecoder { - if ctx.disallowUnknownFields { - return &generalStructDecoder{typ: typ, fields: fields, disallowUnknownFields: true} - } - knownHash := map[int64]struct{}{ - 0: {}, - } - - switch len(fields) { - case 0: - return &skipObjectDecoder{typ} - case 1: - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - return &oneFieldStructDecoder{typ, fieldHash, fieldDecoder} - } - case 2: - var fieldHash1 int64 - var fieldHash2 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldHash1 == 0 { - fieldHash1 = fieldHash - fieldDecoder1 = fieldDecoder - } else { - fieldHash2 = fieldHash - fieldDecoder2 = fieldDecoder - } - } - return &twoFieldsStructDecoder{typ, fieldHash1, fieldDecoder1, fieldHash2, fieldDecoder2} - case 3: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } - } - return &threeFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3} - case 4: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } - } - return &fourFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4} - case 5: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldName5 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - var fieldDecoder5 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else if fieldName4 == 0 { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } else { - fieldName5 = fieldHash - fieldDecoder5 = fieldDecoder - } - } - return &fiveFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4, - fieldName5, fieldDecoder5} - case 6: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldName5 int64 - var fieldName6 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - var fieldDecoder5 *structFieldDecoder - var fieldDecoder6 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else if fieldName4 == 0 { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } else if fieldName5 == 0 { - fieldName5 = fieldHash - fieldDecoder5 = fieldDecoder - } else { - fieldName6 = fieldHash - fieldDecoder6 = fieldDecoder - } - } - return &sixFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4, - fieldName5, fieldDecoder5, - fieldName6, fieldDecoder6} - case 7: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldName5 int64 - var fieldName6 int64 - var fieldName7 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - var fieldDecoder5 *structFieldDecoder - var fieldDecoder6 *structFieldDecoder - var fieldDecoder7 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else if fieldName4 == 0 { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } else if fieldName5 == 0 { - fieldName5 = fieldHash - fieldDecoder5 = fieldDecoder - } else if fieldName6 == 0 { - fieldName6 = fieldHash - fieldDecoder6 = fieldDecoder - } else { - fieldName7 = fieldHash - fieldDecoder7 = fieldDecoder - } - } - return &sevenFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4, - fieldName5, fieldDecoder5, - fieldName6, fieldDecoder6, - fieldName7, fieldDecoder7} - case 8: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldName5 int64 - var fieldName6 int64 - var fieldName7 int64 - var fieldName8 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - var fieldDecoder5 *structFieldDecoder - var fieldDecoder6 *structFieldDecoder - var fieldDecoder7 *structFieldDecoder - var fieldDecoder8 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else if fieldName4 == 0 { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } else if fieldName5 == 0 { - fieldName5 = fieldHash - fieldDecoder5 = fieldDecoder - } else if fieldName6 == 0 { - fieldName6 = fieldHash - fieldDecoder6 = fieldDecoder - } else if fieldName7 == 0 { - fieldName7 = fieldHash - fieldDecoder7 = fieldDecoder - } else { - fieldName8 = fieldHash - fieldDecoder8 = fieldDecoder - } - } - return &eightFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4, - fieldName5, fieldDecoder5, - fieldName6, fieldDecoder6, - fieldName7, fieldDecoder7, - fieldName8, fieldDecoder8} - case 9: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldName5 int64 - var fieldName6 int64 - var fieldName7 int64 - var fieldName8 int64 - var fieldName9 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - var fieldDecoder5 *structFieldDecoder - var fieldDecoder6 *structFieldDecoder - var fieldDecoder7 *structFieldDecoder - var fieldDecoder8 *structFieldDecoder - var fieldDecoder9 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else if fieldName4 == 0 { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } else if fieldName5 == 0 { - fieldName5 = fieldHash - fieldDecoder5 = fieldDecoder - } else if fieldName6 == 0 { - fieldName6 = fieldHash - fieldDecoder6 = fieldDecoder - } else if fieldName7 == 0 { - fieldName7 = fieldHash - fieldDecoder7 = fieldDecoder - } else if fieldName8 == 0 { - fieldName8 = fieldHash - fieldDecoder8 = fieldDecoder - } else { - fieldName9 = fieldHash - fieldDecoder9 = fieldDecoder - } - } - return &nineFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4, - fieldName5, fieldDecoder5, - fieldName6, fieldDecoder6, - fieldName7, fieldDecoder7, - fieldName8, fieldDecoder8, - fieldName9, fieldDecoder9} - case 10: - var fieldName1 int64 - var fieldName2 int64 - var fieldName3 int64 - var fieldName4 int64 - var fieldName5 int64 - var fieldName6 int64 - var fieldName7 int64 - var fieldName8 int64 - var fieldName9 int64 - var fieldName10 int64 - var fieldDecoder1 *structFieldDecoder - var fieldDecoder2 *structFieldDecoder - var fieldDecoder3 *structFieldDecoder - var fieldDecoder4 *structFieldDecoder - var fieldDecoder5 *structFieldDecoder - var fieldDecoder6 *structFieldDecoder - var fieldDecoder7 *structFieldDecoder - var fieldDecoder8 *structFieldDecoder - var fieldDecoder9 *structFieldDecoder - var fieldDecoder10 *structFieldDecoder - for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) - _, known := knownHash[fieldHash] - if known { - return &generalStructDecoder{typ, fields, false} - } - knownHash[fieldHash] = struct{}{} - if fieldName1 == 0 { - fieldName1 = fieldHash - fieldDecoder1 = fieldDecoder - } else if fieldName2 == 0 { - fieldName2 = fieldHash - fieldDecoder2 = fieldDecoder - } else if fieldName3 == 0 { - fieldName3 = fieldHash - fieldDecoder3 = fieldDecoder - } else if fieldName4 == 0 { - fieldName4 = fieldHash - fieldDecoder4 = fieldDecoder - } else if fieldName5 == 0 { - fieldName5 = fieldHash - fieldDecoder5 = fieldDecoder - } else if fieldName6 == 0 { - fieldName6 = fieldHash - fieldDecoder6 = fieldDecoder - } else if fieldName7 == 0 { - fieldName7 = fieldHash - fieldDecoder7 = fieldDecoder - } else if fieldName8 == 0 { - fieldName8 = fieldHash - fieldDecoder8 = fieldDecoder - } else if fieldName9 == 0 { - fieldName9 = fieldHash - fieldDecoder9 = fieldDecoder - } else { - fieldName10 = fieldHash - fieldDecoder10 = fieldDecoder - } - } - return &tenFieldsStructDecoder{typ, - fieldName1, fieldDecoder1, - fieldName2, fieldDecoder2, - fieldName3, fieldDecoder3, - fieldName4, fieldDecoder4, - fieldName5, fieldDecoder5, - fieldName6, fieldDecoder6, - fieldName7, fieldDecoder7, - fieldName8, fieldDecoder8, - fieldName9, fieldDecoder9, - fieldName10, fieldDecoder10} - } - return &generalStructDecoder{typ, fields, false} -} - -type generalStructDecoder struct { - typ reflect2.Type - fields map[string]*structFieldDecoder - disallowUnknownFields bool -} - -func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - var c byte - for c = ','; c == ','; c = iter.nextToken() { - decoder.decodeOneField(ptr, iter) - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } - if c != '}' { - iter.ReportError("struct Decode", `expect }, but found `+string([]byte{c})) - } -} - -func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *Iterator) { - var field string - var fieldDecoder *structFieldDecoder - if iter.cfg.objectFieldMustBeSimpleString { - fieldBytes := iter.ReadStringAsSlice() - field = *(*string)(unsafe.Pointer(&fieldBytes)) - fieldDecoder = decoder.fields[field] - if fieldDecoder == nil && !iter.cfg.caseSensitive { - fieldDecoder = decoder.fields[strings.ToLower(field)] - } - } else { - field = iter.ReadString() - fieldDecoder = decoder.fields[field] - if fieldDecoder == nil && !iter.cfg.caseSensitive { - fieldDecoder = decoder.fields[strings.ToLower(field)] - } - } - if fieldDecoder == nil { - msg := "found unknown field: " + field - if decoder.disallowUnknownFields { - iter.ReportError("ReadObject", msg) - } - c := iter.nextToken() - if c != ':' { - iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) - } - iter.Skip() - return - } - c := iter.nextToken() - if c != ':' { - iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) - } - fieldDecoder.Decode(ptr, iter) -} - -type skipObjectDecoder struct { - typ reflect2.Type -} - -func (decoder *skipObjectDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - valueType := iter.WhatIsNext() - if valueType != ObjectValue && valueType != NilValue { - iter.ReportError("skipObjectDecoder", "expect object or null") - return - } - iter.Skip() -} - -type oneFieldStructDecoder struct { - typ reflect2.Type - fieldHash int64 - fieldDecoder *structFieldDecoder -} - -func (decoder *oneFieldStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - if iter.readFieldHash() == decoder.fieldHash { - decoder.fieldDecoder.Decode(ptr, iter) - } else { - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type twoFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder -} - -func (decoder *twoFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type threeFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder -} - -func (decoder *threeFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type fourFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder -} - -func (decoder *fourFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type fiveFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder - fieldHash5 int64 - fieldDecoder5 *structFieldDecoder -} - -func (decoder *fiveFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - case decoder.fieldHash5: - decoder.fieldDecoder5.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type sixFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder - fieldHash5 int64 - fieldDecoder5 *structFieldDecoder - fieldHash6 int64 - fieldDecoder6 *structFieldDecoder -} - -func (decoder *sixFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - case decoder.fieldHash5: - decoder.fieldDecoder5.Decode(ptr, iter) - case decoder.fieldHash6: - decoder.fieldDecoder6.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type sevenFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder - fieldHash5 int64 - fieldDecoder5 *structFieldDecoder - fieldHash6 int64 - fieldDecoder6 *structFieldDecoder - fieldHash7 int64 - fieldDecoder7 *structFieldDecoder -} - -func (decoder *sevenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - case decoder.fieldHash5: - decoder.fieldDecoder5.Decode(ptr, iter) - case decoder.fieldHash6: - decoder.fieldDecoder6.Decode(ptr, iter) - case decoder.fieldHash7: - decoder.fieldDecoder7.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type eightFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder - fieldHash5 int64 - fieldDecoder5 *structFieldDecoder - fieldHash6 int64 - fieldDecoder6 *structFieldDecoder - fieldHash7 int64 - fieldDecoder7 *structFieldDecoder - fieldHash8 int64 - fieldDecoder8 *structFieldDecoder -} - -func (decoder *eightFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - case decoder.fieldHash5: - decoder.fieldDecoder5.Decode(ptr, iter) - case decoder.fieldHash6: - decoder.fieldDecoder6.Decode(ptr, iter) - case decoder.fieldHash7: - decoder.fieldDecoder7.Decode(ptr, iter) - case decoder.fieldHash8: - decoder.fieldDecoder8.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type nineFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder - fieldHash5 int64 - fieldDecoder5 *structFieldDecoder - fieldHash6 int64 - fieldDecoder6 *structFieldDecoder - fieldHash7 int64 - fieldDecoder7 *structFieldDecoder - fieldHash8 int64 - fieldDecoder8 *structFieldDecoder - fieldHash9 int64 - fieldDecoder9 *structFieldDecoder -} - -func (decoder *nineFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - case decoder.fieldHash5: - decoder.fieldDecoder5.Decode(ptr, iter) - case decoder.fieldHash6: - decoder.fieldDecoder6.Decode(ptr, iter) - case decoder.fieldHash7: - decoder.fieldDecoder7.Decode(ptr, iter) - case decoder.fieldHash8: - decoder.fieldDecoder8.Decode(ptr, iter) - case decoder.fieldHash9: - decoder.fieldDecoder9.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type tenFieldsStructDecoder struct { - typ reflect2.Type - fieldHash1 int64 - fieldDecoder1 *structFieldDecoder - fieldHash2 int64 - fieldDecoder2 *structFieldDecoder - fieldHash3 int64 - fieldDecoder3 *structFieldDecoder - fieldHash4 int64 - fieldDecoder4 *structFieldDecoder - fieldHash5 int64 - fieldDecoder5 *structFieldDecoder - fieldHash6 int64 - fieldDecoder6 *structFieldDecoder - fieldHash7 int64 - fieldDecoder7 *structFieldDecoder - fieldHash8 int64 - fieldDecoder8 *structFieldDecoder - fieldHash9 int64 - fieldDecoder9 *structFieldDecoder - fieldHash10 int64 - fieldDecoder10 *structFieldDecoder -} - -func (decoder *tenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - if !iter.readObjectStart() { - return - } - for { - switch iter.readFieldHash() { - case decoder.fieldHash1: - decoder.fieldDecoder1.Decode(ptr, iter) - case decoder.fieldHash2: - decoder.fieldDecoder2.Decode(ptr, iter) - case decoder.fieldHash3: - decoder.fieldDecoder3.Decode(ptr, iter) - case decoder.fieldHash4: - decoder.fieldDecoder4.Decode(ptr, iter) - case decoder.fieldHash5: - decoder.fieldDecoder5.Decode(ptr, iter) - case decoder.fieldHash6: - decoder.fieldDecoder6.Decode(ptr, iter) - case decoder.fieldHash7: - decoder.fieldDecoder7.Decode(ptr, iter) - case decoder.fieldHash8: - decoder.fieldDecoder8.Decode(ptr, iter) - case decoder.fieldHash9: - decoder.fieldDecoder9.Decode(ptr, iter) - case decoder.fieldHash10: - decoder.fieldDecoder10.Decode(ptr, iter) - default: - iter.Skip() - } - if iter.isObjectEnd() { - break - } - } - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) - } -} - -type structFieldDecoder struct { - field reflect2.StructField - fieldDecoder ValDecoder -} - -func (decoder *structFieldDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - fieldPtr := decoder.field.UnsafeGet(ptr) - decoder.fieldDecoder.Decode(fieldPtr, iter) - if iter.Error != nil && iter.Error != io.EOF { - iter.Error = fmt.Errorf("%s: %s", decoder.field.Name(), iter.Error.Error()) - } -} - -type stringModeStringDecoder struct { - elemDecoder ValDecoder - cfg *frozenConfig -} - -func (decoder *stringModeStringDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - decoder.elemDecoder.Decode(ptr, iter) - str := *((*string)(ptr)) - tempIter := decoder.cfg.BorrowIterator([]byte(str)) - defer decoder.cfg.ReturnIterator(tempIter) - *((*string)(ptr)) = tempIter.ReadString() -} - -type stringModeNumberDecoder struct { - elemDecoder ValDecoder -} - -func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { - c := iter.nextToken() - if c != '"' { - iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c})) - return - } - decoder.elemDecoder.Decode(ptr, iter) - if iter.Error != nil { - return - } - c = iter.readByte() - if c != '"' { - iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c})) - return - } -} diff --git a/vendor/github.com/json-iterator/go/reflect_struct_encoder.go b/vendor/github.com/json-iterator/go/reflect_struct_encoder.go deleted file mode 100644 index d0759cf6..00000000 --- a/vendor/github.com/json-iterator/go/reflect_struct_encoder.go +++ /dev/null @@ -1,210 +0,0 @@ -package jsoniter - -import ( - "fmt" - "github.com/modern-go/reflect2" - "io" - "reflect" - "unsafe" -) - -func encoderOfStruct(ctx *ctx, typ reflect2.Type) ValEncoder { - type bindingTo struct { - binding *Binding - toName string - ignored bool - } - orderedBindings := []*bindingTo{} - structDescriptor := describeStruct(ctx, typ) - for _, binding := range structDescriptor.Fields { - for _, toName := range binding.ToNames { - new := &bindingTo{ - binding: binding, - toName: toName, - } - for _, old := range orderedBindings { - if old.toName != toName { - continue - } - old.ignored, new.ignored = resolveConflictBinding(ctx.frozenConfig, old.binding, new.binding) - } - orderedBindings = append(orderedBindings, new) - } - } - if len(orderedBindings) == 0 { - return &emptyStructEncoder{} - } - finalOrderedFields := []structFieldTo{} - for _, bindingTo := range orderedBindings { - if !bindingTo.ignored { - finalOrderedFields = append(finalOrderedFields, structFieldTo{ - encoder: bindingTo.binding.Encoder.(*structFieldEncoder), - toName: bindingTo.toName, - }) - } - } - return &structEncoder{typ, finalOrderedFields} -} - -func createCheckIsEmpty(ctx *ctx, typ reflect2.Type) checkIsEmpty { - encoder := createEncoderOfNative(ctx, typ) - if encoder != nil { - return encoder - } - kind := typ.Kind() - switch kind { - case reflect.Interface: - return &dynamicEncoder{typ} - case reflect.Struct: - return &structEncoder{typ: typ} - case reflect.Array: - return &arrayEncoder{} - case reflect.Slice: - return &sliceEncoder{} - case reflect.Map: - return encoderOfMap(ctx, typ) - case reflect.Ptr: - return &OptionalEncoder{} - default: - return &lazyErrorEncoder{err: fmt.Errorf("unsupported type: %v", typ)} - } -} - -func resolveConflictBinding(cfg *frozenConfig, old, new *Binding) (ignoreOld, ignoreNew bool) { - newTagged := new.Field.Tag().Get(cfg.getTagKey()) != "" - oldTagged := old.Field.Tag().Get(cfg.getTagKey()) != "" - if newTagged { - if oldTagged { - if len(old.levels) > len(new.levels) { - return true, false - } else if len(new.levels) > len(old.levels) { - return false, true - } else { - return true, true - } - } else { - return true, false - } - } else { - if oldTagged { - return true, false - } - if len(old.levels) > len(new.levels) { - return true, false - } else if len(new.levels) > len(old.levels) { - return false, true - } else { - return true, true - } - } -} - -type structFieldEncoder struct { - field reflect2.StructField - fieldEncoder ValEncoder - omitempty bool -} - -func (encoder *structFieldEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - fieldPtr := encoder.field.UnsafeGet(ptr) - encoder.fieldEncoder.Encode(fieldPtr, stream) - if stream.Error != nil && stream.Error != io.EOF { - stream.Error = fmt.Errorf("%s: %s", encoder.field.Name(), stream.Error.Error()) - } -} - -func (encoder *structFieldEncoder) IsEmpty(ptr unsafe.Pointer) bool { - fieldPtr := encoder.field.UnsafeGet(ptr) - return encoder.fieldEncoder.IsEmpty(fieldPtr) -} - -func (encoder *structFieldEncoder) IsEmbeddedPtrNil(ptr unsafe.Pointer) bool { - isEmbeddedPtrNil, converted := encoder.fieldEncoder.(IsEmbeddedPtrNil) - if !converted { - return false - } - fieldPtr := encoder.field.UnsafeGet(ptr) - return isEmbeddedPtrNil.IsEmbeddedPtrNil(fieldPtr) -} - -type IsEmbeddedPtrNil interface { - IsEmbeddedPtrNil(ptr unsafe.Pointer) bool -} - -type structEncoder struct { - typ reflect2.Type - fields []structFieldTo -} - -type structFieldTo struct { - encoder *structFieldEncoder - toName string -} - -func (encoder *structEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteObjectStart() - isNotFirst := false - for _, field := range encoder.fields { - if field.encoder.omitempty && field.encoder.IsEmpty(ptr) { - continue - } - if field.encoder.IsEmbeddedPtrNil(ptr) { - continue - } - if isNotFirst { - stream.WriteMore() - } - stream.WriteObjectField(field.toName) - field.encoder.Encode(ptr, stream) - isNotFirst = true - } - stream.WriteObjectEnd() - if stream.Error != nil && stream.Error != io.EOF { - stream.Error = fmt.Errorf("%v.%s", encoder.typ, stream.Error.Error()) - } -} - -func (encoder *structEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return false -} - -type emptyStructEncoder struct { -} - -func (encoder *emptyStructEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteEmptyObject() -} - -func (encoder *emptyStructEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return false -} - -type stringModeNumberEncoder struct { - elemEncoder ValEncoder -} - -func (encoder *stringModeNumberEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.writeByte('"') - encoder.elemEncoder.Encode(ptr, stream) - stream.writeByte('"') -} - -func (encoder *stringModeNumberEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.elemEncoder.IsEmpty(ptr) -} - -type stringModeStringEncoder struct { - elemEncoder ValEncoder - cfg *frozenConfig -} - -func (encoder *stringModeStringEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { - tempStream := encoder.cfg.BorrowStream(nil) - defer encoder.cfg.ReturnStream(tempStream) - encoder.elemEncoder.Encode(ptr, tempStream) - stream.WriteString(string(tempStream.Buffer())) -} - -func (encoder *stringModeStringEncoder) IsEmpty(ptr unsafe.Pointer) bool { - return encoder.elemEncoder.IsEmpty(ptr) -} diff --git a/vendor/github.com/json-iterator/go/stream.go b/vendor/github.com/json-iterator/go/stream.go deleted file mode 100644 index 17662fde..00000000 --- a/vendor/github.com/json-iterator/go/stream.go +++ /dev/null @@ -1,211 +0,0 @@ -package jsoniter - -import ( - "io" -) - -// stream is a io.Writer like object, with JSON specific write functions. -// Error is not returned as return value, but stored as Error member on this stream instance. -type Stream struct { - cfg *frozenConfig - out io.Writer - buf []byte - Error error - indention int - Attachment interface{} // open for customized encoder -} - -// NewStream create new stream instance. -// cfg can be jsoniter.ConfigDefault. -// out can be nil if write to internal buffer. -// bufSize is the initial size for the internal buffer in bytes. -func NewStream(cfg API, out io.Writer, bufSize int) *Stream { - return &Stream{ - cfg: cfg.(*frozenConfig), - out: out, - buf: make([]byte, 0, bufSize), - Error: nil, - indention: 0, - } -} - -// Pool returns a pool can provide more stream with same configuration -func (stream *Stream) Pool() StreamPool { - return stream.cfg -} - -// Reset reuse this stream instance by assign a new writer -func (stream *Stream) Reset(out io.Writer) { - stream.out = out - stream.buf = stream.buf[:0] -} - -// Available returns how many bytes are unused in the buffer. -func (stream *Stream) Available() int { - return cap(stream.buf) - len(stream.buf) -} - -// Buffered returns the number of bytes that have been written into the current buffer. -func (stream *Stream) Buffered() int { - return len(stream.buf) -} - -// Buffer if writer is nil, use this method to take the result -func (stream *Stream) Buffer() []byte { - return stream.buf -} - -// SetBuffer allows to append to the internal buffer directly -func (stream *Stream) SetBuffer(buf []byte) { - stream.buf = buf -} - -// Write writes the contents of p into the buffer. -// It returns the number of bytes written. -// If nn < len(p), it also returns an error explaining -// why the write is short. -func (stream *Stream) Write(p []byte) (nn int, err error) { - stream.buf = append(stream.buf, p...) - if stream.out != nil { - nn, err = stream.out.Write(stream.buf) - stream.buf = stream.buf[nn:] - return - } - return len(p), nil -} - -// WriteByte writes a single byte. -func (stream *Stream) writeByte(c byte) { - stream.buf = append(stream.buf, c) -} - -func (stream *Stream) writeTwoBytes(c1 byte, c2 byte) { - stream.buf = append(stream.buf, c1, c2) -} - -func (stream *Stream) writeThreeBytes(c1 byte, c2 byte, c3 byte) { - stream.buf = append(stream.buf, c1, c2, c3) -} - -func (stream *Stream) writeFourBytes(c1 byte, c2 byte, c3 byte, c4 byte) { - stream.buf = append(stream.buf, c1, c2, c3, c4) -} - -func (stream *Stream) writeFiveBytes(c1 byte, c2 byte, c3 byte, c4 byte, c5 byte) { - stream.buf = append(stream.buf, c1, c2, c3, c4, c5) -} - -// Flush writes any buffered data to the underlying io.Writer. -func (stream *Stream) Flush() error { - if stream.out == nil { - return nil - } - if stream.Error != nil { - return stream.Error - } - n, err := stream.out.Write(stream.buf) - if err != nil { - if stream.Error == nil { - stream.Error = err - } - return err - } - stream.buf = stream.buf[n:] - return nil -} - -// WriteRaw write string out without quotes, just like []byte -func (stream *Stream) WriteRaw(s string) { - stream.buf = append(stream.buf, s...) -} - -// WriteNil write null to stream -func (stream *Stream) WriteNil() { - stream.writeFourBytes('n', 'u', 'l', 'l') -} - -// WriteTrue write true to stream -func (stream *Stream) WriteTrue() { - stream.writeFourBytes('t', 'r', 'u', 'e') -} - -// WriteFalse write false to stream -func (stream *Stream) WriteFalse() { - stream.writeFiveBytes('f', 'a', 'l', 's', 'e') -} - -// WriteBool write true or false into stream -func (stream *Stream) WriteBool(val bool) { - if val { - stream.WriteTrue() - } else { - stream.WriteFalse() - } -} - -// WriteObjectStart write { with possible indention -func (stream *Stream) WriteObjectStart() { - stream.indention += stream.cfg.indentionStep - stream.writeByte('{') - stream.writeIndention(0) -} - -// WriteObjectField write "field": with possible indention -func (stream *Stream) WriteObjectField(field string) { - stream.WriteString(field) - if stream.indention > 0 { - stream.writeTwoBytes(':', ' ') - } else { - stream.writeByte(':') - } -} - -// WriteObjectEnd write } with possible indention -func (stream *Stream) WriteObjectEnd() { - stream.writeIndention(stream.cfg.indentionStep) - stream.indention -= stream.cfg.indentionStep - stream.writeByte('}') -} - -// WriteEmptyObject write {} -func (stream *Stream) WriteEmptyObject() { - stream.writeByte('{') - stream.writeByte('}') -} - -// WriteMore write , with possible indention -func (stream *Stream) WriteMore() { - stream.writeByte(',') - stream.writeIndention(0) - stream.Flush() -} - -// WriteArrayStart write [ with possible indention -func (stream *Stream) WriteArrayStart() { - stream.indention += stream.cfg.indentionStep - stream.writeByte('[') - stream.writeIndention(0) -} - -// WriteEmptyArray write [] -func (stream *Stream) WriteEmptyArray() { - stream.writeTwoBytes('[', ']') -} - -// WriteArrayEnd write ] with possible indention -func (stream *Stream) WriteArrayEnd() { - stream.writeIndention(stream.cfg.indentionStep) - stream.indention -= stream.cfg.indentionStep - stream.writeByte(']') -} - -func (stream *Stream) writeIndention(delta int) { - if stream.indention == 0 { - return - } - stream.writeByte('\n') - toWrite := stream.indention - delta - for i := 0; i < toWrite; i++ { - stream.buf = append(stream.buf, ' ') - } -} diff --git a/vendor/github.com/json-iterator/go/stream_float.go b/vendor/github.com/json-iterator/go/stream_float.go deleted file mode 100644 index f318d2c5..00000000 --- a/vendor/github.com/json-iterator/go/stream_float.go +++ /dev/null @@ -1,94 +0,0 @@ -package jsoniter - -import ( - "math" - "strconv" -) - -var pow10 []uint64 - -func init() { - pow10 = []uint64{1, 10, 100, 1000, 10000, 100000, 1000000} -} - -// WriteFloat32 write float32 to stream -func (stream *Stream) WriteFloat32(val float32) { - abs := math.Abs(float64(val)) - fmt := byte('f') - // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. - if abs != 0 { - if float32(abs) < 1e-6 || float32(abs) >= 1e21 { - fmt = 'e' - } - } - stream.buf = strconv.AppendFloat(stream.buf, float64(val), fmt, -1, 32) -} - -// WriteFloat32Lossy write float32 to stream with ONLY 6 digits precision although much much faster -func (stream *Stream) WriteFloat32Lossy(val float32) { - if val < 0 { - stream.writeByte('-') - val = -val - } - if val > 0x4ffffff { - stream.WriteFloat32(val) - return - } - precision := 6 - exp := uint64(1000000) // 6 - lval := uint64(float64(val)*float64(exp) + 0.5) - stream.WriteUint64(lval / exp) - fval := lval % exp - if fval == 0 { - return - } - stream.writeByte('.') - for p := precision - 1; p > 0 && fval < pow10[p]; p-- { - stream.writeByte('0') - } - stream.WriteUint64(fval) - for stream.buf[len(stream.buf)-1] == '0' { - stream.buf = stream.buf[:len(stream.buf)-1] - } -} - -// WriteFloat64 write float64 to stream -func (stream *Stream) WriteFloat64(val float64) { - abs := math.Abs(val) - fmt := byte('f') - // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. - if abs != 0 { - if abs < 1e-6 || abs >= 1e21 { - fmt = 'e' - } - } - stream.buf = strconv.AppendFloat(stream.buf, float64(val), fmt, -1, 64) -} - -// WriteFloat64Lossy write float64 to stream with ONLY 6 digits precision although much much faster -func (stream *Stream) WriteFloat64Lossy(val float64) { - if val < 0 { - stream.writeByte('-') - val = -val - } - if val > 0x4ffffff { - stream.WriteFloat64(val) - return - } - precision := 6 - exp := uint64(1000000) // 6 - lval := uint64(val*float64(exp) + 0.5) - stream.WriteUint64(lval / exp) - fval := lval % exp - if fval == 0 { - return - } - stream.writeByte('.') - for p := precision - 1; p > 0 && fval < pow10[p]; p-- { - stream.writeByte('0') - } - stream.WriteUint64(fval) - for stream.buf[len(stream.buf)-1] == '0' { - stream.buf = stream.buf[:len(stream.buf)-1] - } -} diff --git a/vendor/github.com/json-iterator/go/stream_int.go b/vendor/github.com/json-iterator/go/stream_int.go deleted file mode 100644 index d1059ee4..00000000 --- a/vendor/github.com/json-iterator/go/stream_int.go +++ /dev/null @@ -1,190 +0,0 @@ -package jsoniter - -var digits []uint32 - -func init() { - digits = make([]uint32, 1000) - for i := uint32(0); i < 1000; i++ { - digits[i] = (((i / 100) + '0') << 16) + ((((i / 10) % 10) + '0') << 8) + i%10 + '0' - if i < 10 { - digits[i] += 2 << 24 - } else if i < 100 { - digits[i] += 1 << 24 - } - } -} - -func writeFirstBuf(space []byte, v uint32) []byte { - start := v >> 24 - if start == 0 { - space = append(space, byte(v>>16), byte(v>>8)) - } else if start == 1 { - space = append(space, byte(v>>8)) - } - space = append(space, byte(v)) - return space -} - -func writeBuf(buf []byte, v uint32) []byte { - return append(buf, byte(v>>16), byte(v>>8), byte(v)) -} - -// WriteUint8 write uint8 to stream -func (stream *Stream) WriteUint8(val uint8) { - stream.buf = writeFirstBuf(stream.buf, digits[val]) -} - -// WriteInt8 write int8 to stream -func (stream *Stream) WriteInt8(nval int8) { - var val uint8 - if nval < 0 { - val = uint8(-nval) - stream.buf = append(stream.buf, '-') - } else { - val = uint8(nval) - } - stream.buf = writeFirstBuf(stream.buf, digits[val]) -} - -// WriteUint16 write uint16 to stream -func (stream *Stream) WriteUint16(val uint16) { - q1 := val / 1000 - if q1 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[val]) - return - } - r1 := val - q1*1000 - stream.buf = writeFirstBuf(stream.buf, digits[q1]) - stream.buf = writeBuf(stream.buf, digits[r1]) - return -} - -// WriteInt16 write int16 to stream -func (stream *Stream) WriteInt16(nval int16) { - var val uint16 - if nval < 0 { - val = uint16(-nval) - stream.buf = append(stream.buf, '-') - } else { - val = uint16(nval) - } - stream.WriteUint16(val) -} - -// WriteUint32 write uint32 to stream -func (stream *Stream) WriteUint32(val uint32) { - q1 := val / 1000 - if q1 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[val]) - return - } - r1 := val - q1*1000 - q2 := q1 / 1000 - if q2 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q1]) - stream.buf = writeBuf(stream.buf, digits[r1]) - return - } - r2 := q1 - q2*1000 - q3 := q2 / 1000 - if q3 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q2]) - } else { - r3 := q2 - q3*1000 - stream.buf = append(stream.buf, byte(q3+'0')) - stream.buf = writeBuf(stream.buf, digits[r3]) - } - stream.buf = writeBuf(stream.buf, digits[r2]) - stream.buf = writeBuf(stream.buf, digits[r1]) -} - -// WriteInt32 write int32 to stream -func (stream *Stream) WriteInt32(nval int32) { - var val uint32 - if nval < 0 { - val = uint32(-nval) - stream.buf = append(stream.buf, '-') - } else { - val = uint32(nval) - } - stream.WriteUint32(val) -} - -// WriteUint64 write uint64 to stream -func (stream *Stream) WriteUint64(val uint64) { - q1 := val / 1000 - if q1 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[val]) - return - } - r1 := val - q1*1000 - q2 := q1 / 1000 - if q2 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q1]) - stream.buf = writeBuf(stream.buf, digits[r1]) - return - } - r2 := q1 - q2*1000 - q3 := q2 / 1000 - if q3 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q2]) - stream.buf = writeBuf(stream.buf, digits[r2]) - stream.buf = writeBuf(stream.buf, digits[r1]) - return - } - r3 := q2 - q3*1000 - q4 := q3 / 1000 - if q4 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q3]) - stream.buf = writeBuf(stream.buf, digits[r3]) - stream.buf = writeBuf(stream.buf, digits[r2]) - stream.buf = writeBuf(stream.buf, digits[r1]) - return - } - r4 := q3 - q4*1000 - q5 := q4 / 1000 - if q5 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q4]) - stream.buf = writeBuf(stream.buf, digits[r4]) - stream.buf = writeBuf(stream.buf, digits[r3]) - stream.buf = writeBuf(stream.buf, digits[r2]) - stream.buf = writeBuf(stream.buf, digits[r1]) - return - } - r5 := q4 - q5*1000 - q6 := q5 / 1000 - if q6 == 0 { - stream.buf = writeFirstBuf(stream.buf, digits[q5]) - } else { - stream.buf = writeFirstBuf(stream.buf, digits[q6]) - r6 := q5 - q6*1000 - stream.buf = writeBuf(stream.buf, digits[r6]) - } - stream.buf = writeBuf(stream.buf, digits[r5]) - stream.buf = writeBuf(stream.buf, digits[r4]) - stream.buf = writeBuf(stream.buf, digits[r3]) - stream.buf = writeBuf(stream.buf, digits[r2]) - stream.buf = writeBuf(stream.buf, digits[r1]) -} - -// WriteInt64 write int64 to stream -func (stream *Stream) WriteInt64(nval int64) { - var val uint64 - if nval < 0 { - val = uint64(-nval) - stream.buf = append(stream.buf, '-') - } else { - val = uint64(nval) - } - stream.WriteUint64(val) -} - -// WriteInt write int to stream -func (stream *Stream) WriteInt(val int) { - stream.WriteInt64(int64(val)) -} - -// WriteUint write uint to stream -func (stream *Stream) WriteUint(val uint) { - stream.WriteUint64(uint64(val)) -} diff --git a/vendor/github.com/json-iterator/go/stream_str.go b/vendor/github.com/json-iterator/go/stream_str.go deleted file mode 100644 index 54c2ba0b..00000000 --- a/vendor/github.com/json-iterator/go/stream_str.go +++ /dev/null @@ -1,372 +0,0 @@ -package jsoniter - -import ( - "unicode/utf8" -) - -// htmlSafeSet holds the value true if the ASCII character with the given -// array position can be safely represented inside a JSON string, embedded -// inside of HTML