Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 1 | /* |
| 2 | Copyright 2014 The Kubernetes Authors. |
| 3 | |
| 4 | Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | you may not use this file except in compliance with the License. |
| 6 | You may obtain a copy of the License at |
| 7 | |
| 8 | http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | Unless required by applicable law or agreed to in writing, software |
| 11 | distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | See the License for the specific language governing permissions and |
| 14 | limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package json |
| 18 | |
| 19 | import ( |
| 20 | "encoding/json" |
| 21 | "io" |
| 22 | "strconv" |
| 23 | "unsafe" |
| 24 | |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 25 | jsoniter "github.com/json-iterator/go" |
| 26 | "github.com/modern-go/reflect2" |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 27 | "sigs.k8s.io/yaml" |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 28 | |
| 29 | "k8s.io/apimachinery/pkg/runtime" |
| 30 | "k8s.io/apimachinery/pkg/runtime/schema" |
| 31 | "k8s.io/apimachinery/pkg/runtime/serializer/recognizer" |
| 32 | "k8s.io/apimachinery/pkg/util/framer" |
| 33 | utilyaml "k8s.io/apimachinery/pkg/util/yaml" |
| 34 | ) |
| 35 | |
| 36 | // NewSerializer creates a JSON serializer that handles encoding versioned objects into the proper JSON form. If typer |
| 37 | // is not nil, the object has the group, version, and kind fields set. |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 38 | // Deprecated: use NewSerializerWithOptions instead. |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 39 | func NewSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper, pretty bool) *Serializer { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 40 | return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{false, pretty, false}) |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 41 | } |
| 42 | |
| 43 | // NewYAMLSerializer creates a YAML serializer that handles encoding versioned objects into the proper YAML form. If typer |
| 44 | // is not nil, the object has the group, version, and kind fields set. This serializer supports only the subset of YAML that |
| 45 | // matches JSON, and will error if constructs are used that do not serialize to JSON. |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 46 | // Deprecated: use NewSerializerWithOptions instead. |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 47 | func NewYAMLSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper) *Serializer { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 48 | return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{true, false, false}) |
| 49 | } |
| 50 | |
| 51 | // NewSerializerWithOptions creates a JSON/YAML serializer that handles encoding versioned objects into the proper JSON/YAML |
| 52 | // form. If typer is not nil, the object has the group, version, and kind fields set. Options are copied into the Serializer |
| 53 | // and are immutable. |
| 54 | func NewSerializerWithOptions(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper, options SerializerOptions) *Serializer { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 55 | return &Serializer{ |
| 56 | meta: meta, |
| 57 | creater: creater, |
| 58 | typer: typer, |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 59 | options: options, |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 60 | } |
| 61 | } |
| 62 | |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 63 | // SerializerOptions holds the options which are used to configure a JSON/YAML serializer. |
| 64 | // example: |
| 65 | // (1) To configure a JSON serializer, set `Yaml` to `false`. |
| 66 | // (2) To configure a YAML serializer, set `Yaml` to `true`. |
| 67 | // (3) To configure a strict serializer that can return strictDecodingError, set `Strict` to `true`. |
| 68 | type SerializerOptions struct { |
| 69 | // Yaml: configures the Serializer to work with JSON(false) or YAML(true). |
| 70 | // When `Yaml` is enabled, this serializer only supports the subset of YAML that |
| 71 | // matches JSON, and will error if constructs are used that do not serialize to JSON. |
| 72 | Yaml bool |
| 73 | |
| 74 | // Pretty: configures a JSON enabled Serializer(`Yaml: false`) to produce human-readable output. |
| 75 | // This option is silently ignored when `Yaml` is `true`. |
| 76 | Pretty bool |
| 77 | |
| 78 | // Strict: configures the Serializer to return strictDecodingError's when duplicate fields are present decoding JSON or YAML. |
| 79 | // Note that enabling this option is not as performant as the non-strict variant, and should not be used in fast paths. |
| 80 | Strict bool |
| 81 | } |
| 82 | |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 83 | type Serializer struct { |
| 84 | meta MetaFactory |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 85 | options SerializerOptions |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 86 | creater runtime.ObjectCreater |
| 87 | typer runtime.ObjectTyper |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 88 | } |
| 89 | |
| 90 | // Serializer implements Serializer |
| 91 | var _ runtime.Serializer = &Serializer{} |
| 92 | var _ recognizer.RecognizingDecoder = &Serializer{} |
| 93 | |
| 94 | type customNumberExtension struct { |
| 95 | jsoniter.DummyExtension |
| 96 | } |
| 97 | |
| 98 | func (cne *customNumberExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { |
| 99 | if typ.String() == "interface {}" { |
| 100 | return customNumberDecoder{} |
| 101 | } |
| 102 | return nil |
| 103 | } |
| 104 | |
| 105 | type customNumberDecoder struct { |
| 106 | } |
| 107 | |
| 108 | func (customNumberDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { |
| 109 | switch iter.WhatIsNext() { |
| 110 | case jsoniter.NumberValue: |
| 111 | var number jsoniter.Number |
| 112 | iter.ReadVal(&number) |
| 113 | i64, err := strconv.ParseInt(string(number), 10, 64) |
| 114 | if err == nil { |
| 115 | *(*interface{})(ptr) = i64 |
| 116 | return |
| 117 | } |
| 118 | f64, err := strconv.ParseFloat(string(number), 64) |
| 119 | if err == nil { |
| 120 | *(*interface{})(ptr) = f64 |
| 121 | return |
| 122 | } |
| 123 | iter.ReportError("DecodeNumber", err.Error()) |
| 124 | default: |
| 125 | *(*interface{})(ptr) = iter.Read() |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | // CaseSensitiveJsonIterator returns a jsoniterator API that's configured to be |
| 130 | // case-sensitive when unmarshalling, and otherwise compatible with |
| 131 | // the encoding/json standard library. |
| 132 | func CaseSensitiveJsonIterator() jsoniter.API { |
| 133 | config := jsoniter.Config{ |
| 134 | EscapeHTML: true, |
| 135 | SortMapKeys: true, |
| 136 | ValidateJsonRawMessage: true, |
| 137 | CaseSensitive: true, |
| 138 | }.Froze() |
| 139 | // Force jsoniter to decode number to interface{} via int64/float64, if possible. |
| 140 | config.RegisterExtension(&customNumberExtension{}) |
| 141 | return config |
| 142 | } |
| 143 | |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 144 | // StrictCaseSensitiveJsonIterator returns a jsoniterator API that's configured to be |
| 145 | // case-sensitive, but also disallows unknown fields when unmarshalling. It is compatible with |
| 146 | // the encoding/json standard library. |
| 147 | func StrictCaseSensitiveJsonIterator() jsoniter.API { |
| 148 | config := jsoniter.Config{ |
| 149 | EscapeHTML: true, |
| 150 | SortMapKeys: true, |
| 151 | ValidateJsonRawMessage: true, |
| 152 | CaseSensitive: true, |
| 153 | DisallowUnknownFields: true, |
| 154 | }.Froze() |
| 155 | // Force jsoniter to decode number to interface{} via int64/float64, if possible. |
| 156 | config.RegisterExtension(&customNumberExtension{}) |
| 157 | return config |
| 158 | } |
| 159 | |
| 160 | // Private copies of jsoniter to try to shield against possible mutations |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 161 | // from outside. Still does not protect from package level jsoniter.Register*() functions - someone calling them |
| 162 | // in some other library will mess with every usage of the jsoniter library in the whole program. |
| 163 | // See https://github.com/json-iterator/go/issues/265 |
| 164 | var caseSensitiveJsonIterator = CaseSensitiveJsonIterator() |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 165 | var strictCaseSensitiveJsonIterator = StrictCaseSensitiveJsonIterator() |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 166 | |
| 167 | // gvkWithDefaults returns group kind and version defaulting from provided default |
| 168 | func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind { |
| 169 | if len(actual.Kind) == 0 { |
| 170 | actual.Kind = defaultGVK.Kind |
| 171 | } |
| 172 | if len(actual.Version) == 0 && len(actual.Group) == 0 { |
| 173 | actual.Group = defaultGVK.Group |
| 174 | actual.Version = defaultGVK.Version |
| 175 | } |
| 176 | if len(actual.Version) == 0 && actual.Group == defaultGVK.Group { |
| 177 | actual.Version = defaultGVK.Version |
| 178 | } |
| 179 | return actual |
| 180 | } |
| 181 | |
| 182 | // Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then |
| 183 | // load that data into an object matching the desired schema kind or the provided into. |
| 184 | // If into is *runtime.Unknown, the raw data will be extracted and no decoding will be performed. |
| 185 | // If into is not registered with the typer, then the object will be straight decoded using normal JSON/YAML unmarshalling. |
| 186 | // If into is provided and the original data is not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. |
| 187 | // If into is nil or data's gvk different from into's gvk, it will generate a new Object with ObjectCreater.New(gvk) |
| 188 | // On success or most errors, the method will return the calculated schema kind. |
| 189 | // The gvk calculate priority will be originalData > default gvk > into |
| 190 | func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { |
| 191 | if versioned, ok := into.(*runtime.VersionedObjects); ok { |
| 192 | into = versioned.Last() |
| 193 | obj, actual, err := s.Decode(originalData, gvk, into) |
| 194 | if err != nil { |
| 195 | return nil, actual, err |
| 196 | } |
| 197 | versioned.Objects = []runtime.Object{obj} |
| 198 | return versioned, actual, nil |
| 199 | } |
| 200 | |
| 201 | data := originalData |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 202 | if s.options.Yaml { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 203 | altered, err := yaml.YAMLToJSON(data) |
| 204 | if err != nil { |
| 205 | return nil, nil, err |
| 206 | } |
| 207 | data = altered |
| 208 | } |
| 209 | |
| 210 | actual, err := s.meta.Interpret(data) |
| 211 | if err != nil { |
| 212 | return nil, nil, err |
| 213 | } |
| 214 | |
| 215 | if gvk != nil { |
| 216 | *actual = gvkWithDefaults(*actual, *gvk) |
| 217 | } |
| 218 | |
| 219 | if unk, ok := into.(*runtime.Unknown); ok && unk != nil { |
| 220 | unk.Raw = originalData |
| 221 | unk.ContentType = runtime.ContentTypeJSON |
| 222 | unk.GetObjectKind().SetGroupVersionKind(*actual) |
| 223 | return unk, actual, nil |
| 224 | } |
| 225 | |
| 226 | if into != nil { |
| 227 | _, isUnstructured := into.(runtime.Unstructured) |
| 228 | types, _, err := s.typer.ObjectKinds(into) |
| 229 | switch { |
| 230 | case runtime.IsNotRegisteredError(err), isUnstructured: |
| 231 | if err := caseSensitiveJsonIterator.Unmarshal(data, into); err != nil { |
| 232 | return nil, actual, err |
| 233 | } |
| 234 | return into, actual, nil |
| 235 | case err != nil: |
| 236 | return nil, actual, err |
| 237 | default: |
| 238 | *actual = gvkWithDefaults(*actual, types[0]) |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | if len(actual.Kind) == 0 { |
| 243 | return nil, actual, runtime.NewMissingKindErr(string(originalData)) |
| 244 | } |
| 245 | if len(actual.Version) == 0 { |
| 246 | return nil, actual, runtime.NewMissingVersionErr(string(originalData)) |
| 247 | } |
| 248 | |
| 249 | // use the target if necessary |
| 250 | obj, err := runtime.UseOrCreateObject(s.typer, s.creater, *actual, into) |
| 251 | if err != nil { |
| 252 | return nil, actual, err |
| 253 | } |
| 254 | |
| 255 | if err := caseSensitiveJsonIterator.Unmarshal(data, obj); err != nil { |
| 256 | return nil, actual, err |
| 257 | } |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 258 | |
| 259 | // If the deserializer is non-strict, return successfully here. |
| 260 | if !s.options.Strict { |
| 261 | return obj, actual, nil |
| 262 | } |
| 263 | |
| 264 | // In strict mode pass the data trough the YAMLToJSONStrict converter. |
| 265 | // This is done to catch duplicate fields regardless of encoding (JSON or YAML). For JSON data, |
| 266 | // the output would equal the input, unless there is a parsing error such as duplicate fields. |
| 267 | // As we know this was successful in the non-strict case, the only error that may be returned here |
| 268 | // is because of the newly-added strictness. hence we know we can return the typed strictDecoderError |
| 269 | // the actual error is that the object contains duplicate fields. |
| 270 | altered, err := yaml.YAMLToJSONStrict(originalData) |
| 271 | if err != nil { |
| 272 | return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData)) |
| 273 | } |
| 274 | // As performance is not an issue for now for the strict deserializer (one has regardless to do |
| 275 | // the unmarshal twice), we take the sanitized, altered data that is guaranteed to have no duplicated |
| 276 | // fields, and unmarshal this into a copy of the already-populated obj. Any error that occurs here is |
| 277 | // due to that a matching field doesn't exist in the object. hence we can return a typed strictDecoderError, |
| 278 | // the actual error is that the object contains unknown field. |
| 279 | strictObj := obj.DeepCopyObject() |
| 280 | if err := strictCaseSensitiveJsonIterator.Unmarshal(altered, strictObj); err != nil { |
| 281 | return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData)) |
| 282 | } |
| 283 | // Always return the same object as the non-strict serializer to avoid any deviations. |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 284 | return obj, actual, nil |
| 285 | } |
| 286 | |
| 287 | // Encode serializes the provided object to the given writer. |
| 288 | func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 289 | if s.options.Yaml { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 290 | json, err := caseSensitiveJsonIterator.Marshal(obj) |
| 291 | if err != nil { |
| 292 | return err |
| 293 | } |
| 294 | data, err := yaml.JSONToYAML(json) |
| 295 | if err != nil { |
| 296 | return err |
| 297 | } |
| 298 | _, err = w.Write(data) |
| 299 | return err |
| 300 | } |
| 301 | |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 302 | if s.options.Pretty { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 303 | data, err := caseSensitiveJsonIterator.MarshalIndent(obj, "", " ") |
| 304 | if err != nil { |
| 305 | return err |
| 306 | } |
| 307 | _, err = w.Write(data) |
| 308 | return err |
| 309 | } |
| 310 | encoder := json.NewEncoder(w) |
| 311 | return encoder.Encode(obj) |
| 312 | } |
| 313 | |
| 314 | // RecognizesData implements the RecognizingDecoder interface. |
| 315 | func (s *Serializer) RecognizesData(peek io.Reader) (ok, unknown bool, err error) { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 316 | if s.options.Yaml { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 317 | // we could potentially look for '---' |
| 318 | return false, true, nil |
| 319 | } |
| 320 | _, _, ok = utilyaml.GuessJSONStream(peek, 2048) |
| 321 | return ok, false, nil |
| 322 | } |
| 323 | |
| 324 | // Framer is the default JSON framing behavior, with newlines delimiting individual objects. |
| 325 | var Framer = jsonFramer{} |
| 326 | |
| 327 | type jsonFramer struct{} |
| 328 | |
| 329 | // NewFrameWriter implements stream framing for this serializer |
| 330 | func (jsonFramer) NewFrameWriter(w io.Writer) io.Writer { |
| 331 | // we can write JSON objects directly to the writer, because they are self-framing |
| 332 | return w |
| 333 | } |
| 334 | |
| 335 | // NewFrameReader implements stream framing for this serializer |
| 336 | func (jsonFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser { |
| 337 | // we need to extract the JSON chunks of data to pass to Decode() |
| 338 | return framer.NewJSONFramedReader(r) |
| 339 | } |
| 340 | |
| 341 | // YAMLFramer is the default JSON framing behavior, with newlines delimiting individual objects. |
| 342 | var YAMLFramer = yamlFramer{} |
| 343 | |
| 344 | type yamlFramer struct{} |
| 345 | |
| 346 | // NewFrameWriter implements stream framing for this serializer |
| 347 | func (yamlFramer) NewFrameWriter(w io.Writer) io.Writer { |
| 348 | return yamlFrameWriter{w} |
| 349 | } |
| 350 | |
| 351 | // NewFrameReader implements stream framing for this serializer |
| 352 | func (yamlFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser { |
| 353 | // extract the YAML document chunks directly |
| 354 | return utilyaml.NewDocumentDecoder(r) |
| 355 | } |
| 356 | |
| 357 | type yamlFrameWriter struct { |
| 358 | w io.Writer |
| 359 | } |
| 360 | |
| 361 | // Write separates each document with the YAML document separator (`---` followed by line |
| 362 | // break). Writers must write well formed YAML documents (include a final line break). |
| 363 | func (w yamlFrameWriter) Write(data []byte) (n int, err error) { |
| 364 | if _, err := w.w.Write([]byte("---\n")); err != nil { |
| 365 | return 0, err |
| 366 | } |
| 367 | return w.w.Write(data) |
| 368 | } |