Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 1 | /* |
| 2 | Copyright 2015 The Kubernetes Authors. |
| 3 | |
| 4 | Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | you may not use this file except in compliance with the License. |
| 6 | You may obtain a copy of the License at |
| 7 | |
| 8 | http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | Unless required by applicable law or agreed to in writing, software |
| 11 | distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | See the License for the specific language governing permissions and |
| 14 | limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package protobuf |
| 18 | |
| 19 | import ( |
| 20 | "bytes" |
| 21 | "fmt" |
| 22 | "io" |
| 23 | "net/http" |
| 24 | "reflect" |
| 25 | |
| 26 | "github.com/gogo/protobuf/proto" |
| 27 | |
| 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 29 | "k8s.io/apimachinery/pkg/runtime" |
| 30 | "k8s.io/apimachinery/pkg/runtime/schema" |
| 31 | "k8s.io/apimachinery/pkg/runtime/serializer/recognizer" |
| 32 | "k8s.io/apimachinery/pkg/util/framer" |
| 33 | ) |
| 34 | |
| 35 | var ( |
| 36 | // protoEncodingPrefix serves as a magic number for an encoded protobuf message on this serializer. All |
| 37 | // proto messages serialized by this schema will be preceded by the bytes 0x6b 0x38 0x73, with the fourth |
| 38 | // byte being reserved for the encoding style. The only encoding style defined is 0x00, which means that |
| 39 | // the rest of the byte stream is a message of type k8s.io.kubernetes.pkg.runtime.Unknown (proto2). |
| 40 | // |
| 41 | // See k8s.io/apimachinery/pkg/runtime/generated.proto for details of the runtime.Unknown message. |
| 42 | // |
| 43 | // This encoding scheme is experimental, and is subject to change at any time. |
| 44 | protoEncodingPrefix = []byte{0x6b, 0x38, 0x73, 0x00} |
| 45 | ) |
| 46 | |
| 47 | type errNotMarshalable struct { |
| 48 | t reflect.Type |
| 49 | } |
| 50 | |
| 51 | func (e errNotMarshalable) Error() string { |
| 52 | return fmt.Sprintf("object %v does not implement the protobuf marshalling interface and cannot be encoded to a protobuf message", e.t) |
| 53 | } |
| 54 | |
| 55 | func (e errNotMarshalable) Status() metav1.Status { |
| 56 | return metav1.Status{ |
| 57 | Status: metav1.StatusFailure, |
| 58 | Code: http.StatusNotAcceptable, |
| 59 | Reason: metav1.StatusReason("NotAcceptable"), |
| 60 | Message: e.Error(), |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func IsNotMarshalable(err error) bool { |
| 65 | _, ok := err.(errNotMarshalable) |
| 66 | return err != nil && ok |
| 67 | } |
| 68 | |
| 69 | // NewSerializer creates a Protobuf serializer that handles encoding versioned objects into the proper wire form. If a typer |
| 70 | // is passed, the encoded object will have group, version, and kind fields set. If typer is nil, the objects will be written |
| 71 | // as-is (any type info passed with the object will be used). |
| 72 | // |
| 73 | // This encoding scheme is experimental, and is subject to change at any time. |
| 74 | func NewSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper, defaultContentType string) *Serializer { |
| 75 | return &Serializer{ |
| 76 | prefix: protoEncodingPrefix, |
| 77 | creater: creater, |
| 78 | typer: typer, |
| 79 | contentType: defaultContentType, |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | type Serializer struct { |
| 84 | prefix []byte |
| 85 | creater runtime.ObjectCreater |
| 86 | typer runtime.ObjectTyper |
| 87 | contentType string |
| 88 | } |
| 89 | |
| 90 | var _ runtime.Serializer = &Serializer{} |
| 91 | var _ recognizer.RecognizingDecoder = &Serializer{} |
| 92 | |
| 93 | // Decode attempts to convert the provided data into a protobuf message, extract the stored schema kind, apply the provided default |
| 94 | // gvk, and then load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, |
| 95 | // the raw data will be extracted and no decoding will be performed. If into is not registered with the typer, then the object will |
| 96 | // be straight decoded using normal protobuf unmarshalling (the MarshalTo interface). If into is provided and the original data is |
| 97 | // not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. On success or most |
| 98 | // errors, the method will return the calculated schema kind. |
| 99 | func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { |
| 100 | if versioned, ok := into.(*runtime.VersionedObjects); ok { |
| 101 | into = versioned.Last() |
| 102 | obj, actual, err := s.Decode(originalData, gvk, into) |
| 103 | if err != nil { |
| 104 | return nil, actual, err |
| 105 | } |
| 106 | // the last item in versioned becomes into, so if versioned was not originally empty we reset the object |
| 107 | // array so the first position is the decoded object and the second position is the outermost object. |
| 108 | // if there were no objects in the versioned list passed to us, only add ourselves. |
| 109 | if into != nil && into != obj { |
| 110 | versioned.Objects = []runtime.Object{obj, into} |
| 111 | } else { |
| 112 | versioned.Objects = []runtime.Object{obj} |
| 113 | } |
| 114 | return versioned, actual, err |
| 115 | } |
| 116 | |
| 117 | prefixLen := len(s.prefix) |
| 118 | switch { |
| 119 | case len(originalData) == 0: |
| 120 | // TODO: treat like decoding {} from JSON with defaulting |
| 121 | return nil, nil, fmt.Errorf("empty data") |
| 122 | case len(originalData) < prefixLen || !bytes.Equal(s.prefix, originalData[:prefixLen]): |
| 123 | return nil, nil, fmt.Errorf("provided data does not appear to be a protobuf message, expected prefix %v", s.prefix) |
| 124 | case len(originalData) == prefixLen: |
| 125 | // TODO: treat like decoding {} from JSON with defaulting |
| 126 | return nil, nil, fmt.Errorf("empty body") |
| 127 | } |
| 128 | |
| 129 | data := originalData[prefixLen:] |
| 130 | unk := runtime.Unknown{} |
| 131 | if err := unk.Unmarshal(data); err != nil { |
| 132 | return nil, nil, err |
| 133 | } |
| 134 | |
| 135 | actual := unk.GroupVersionKind() |
| 136 | copyKindDefaults(&actual, gvk) |
| 137 | |
| 138 | if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil { |
| 139 | *intoUnknown = unk |
| 140 | if ok, _, _ := s.RecognizesData(bytes.NewBuffer(unk.Raw)); ok { |
| 141 | intoUnknown.ContentType = s.contentType |
| 142 | } |
| 143 | return intoUnknown, &actual, nil |
| 144 | } |
| 145 | |
| 146 | if into != nil { |
| 147 | types, _, err := s.typer.ObjectKinds(into) |
| 148 | switch { |
| 149 | case runtime.IsNotRegisteredError(err): |
| 150 | pb, ok := into.(proto.Message) |
| 151 | if !ok { |
| 152 | return nil, &actual, errNotMarshalable{reflect.TypeOf(into)} |
| 153 | } |
| 154 | if err := proto.Unmarshal(unk.Raw, pb); err != nil { |
| 155 | return nil, &actual, err |
| 156 | } |
| 157 | return into, &actual, nil |
| 158 | case err != nil: |
| 159 | return nil, &actual, err |
| 160 | default: |
| 161 | copyKindDefaults(&actual, &types[0]) |
| 162 | // if the result of defaulting did not set a version or group, ensure that at least group is set |
| 163 | // (copyKindDefaults will not assign Group if version is already set). This guarantees that the group |
| 164 | // of into is set if there is no better information from the caller or object. |
| 165 | if len(actual.Version) == 0 && len(actual.Group) == 0 { |
| 166 | actual.Group = types[0].Group |
| 167 | } |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | if len(actual.Kind) == 0 { |
| 172 | return nil, &actual, runtime.NewMissingKindErr(fmt.Sprintf("%#v", unk.TypeMeta)) |
| 173 | } |
| 174 | if len(actual.Version) == 0 { |
| 175 | return nil, &actual, runtime.NewMissingVersionErr(fmt.Sprintf("%#v", unk.TypeMeta)) |
| 176 | } |
| 177 | |
| 178 | return unmarshalToObject(s.typer, s.creater, &actual, into, unk.Raw) |
| 179 | } |
| 180 | |
| 181 | // Encode serializes the provided object to the given writer. |
| 182 | func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error { |
| 183 | prefixSize := uint64(len(s.prefix)) |
| 184 | |
| 185 | var unk runtime.Unknown |
| 186 | switch t := obj.(type) { |
| 187 | case *runtime.Unknown: |
| 188 | estimatedSize := prefixSize + uint64(t.Size()) |
| 189 | data := make([]byte, estimatedSize) |
| 190 | i, err := t.MarshalTo(data[prefixSize:]) |
| 191 | if err != nil { |
| 192 | return err |
| 193 | } |
| 194 | copy(data, s.prefix) |
| 195 | _, err = w.Write(data[:prefixSize+uint64(i)]) |
| 196 | return err |
| 197 | default: |
| 198 | kind := obj.GetObjectKind().GroupVersionKind() |
| 199 | unk = runtime.Unknown{ |
| 200 | TypeMeta: runtime.TypeMeta{ |
| 201 | Kind: kind.Kind, |
| 202 | APIVersion: kind.GroupVersion().String(), |
| 203 | }, |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | switch t := obj.(type) { |
| 208 | case bufferedMarshaller: |
| 209 | // this path performs a single allocation during write but requires the caller to implement |
| 210 | // the more efficient Size and MarshalTo methods |
| 211 | encodedSize := uint64(t.Size()) |
| 212 | estimatedSize := prefixSize + estimateUnknownSize(&unk, encodedSize) |
| 213 | data := make([]byte, estimatedSize) |
| 214 | |
| 215 | i, err := unk.NestedMarshalTo(data[prefixSize:], t, encodedSize) |
| 216 | if err != nil { |
| 217 | return err |
| 218 | } |
| 219 | |
| 220 | copy(data, s.prefix) |
| 221 | |
| 222 | _, err = w.Write(data[:prefixSize+uint64(i)]) |
| 223 | return err |
| 224 | |
| 225 | case proto.Marshaler: |
| 226 | // this path performs extra allocations |
| 227 | data, err := t.Marshal() |
| 228 | if err != nil { |
| 229 | return err |
| 230 | } |
| 231 | unk.Raw = data |
| 232 | |
| 233 | estimatedSize := prefixSize + uint64(unk.Size()) |
| 234 | data = make([]byte, estimatedSize) |
| 235 | |
| 236 | i, err := unk.MarshalTo(data[prefixSize:]) |
| 237 | if err != nil { |
| 238 | return err |
| 239 | } |
| 240 | |
| 241 | copy(data, s.prefix) |
| 242 | |
| 243 | _, err = w.Write(data[:prefixSize+uint64(i)]) |
| 244 | return err |
| 245 | |
| 246 | default: |
| 247 | // TODO: marshal with a different content type and serializer (JSON for third party objects) |
| 248 | return errNotMarshalable{reflect.TypeOf(obj)} |
| 249 | } |
| 250 | } |
| 251 | |
| 252 | // RecognizesData implements the RecognizingDecoder interface. |
| 253 | func (s *Serializer) RecognizesData(peek io.Reader) (bool, bool, error) { |
| 254 | prefix := make([]byte, 4) |
| 255 | n, err := peek.Read(prefix) |
| 256 | if err != nil { |
| 257 | if err == io.EOF { |
| 258 | return false, false, nil |
| 259 | } |
| 260 | return false, false, err |
| 261 | } |
| 262 | if n != 4 { |
| 263 | return false, false, nil |
| 264 | } |
| 265 | return bytes.Equal(s.prefix, prefix), false, nil |
| 266 | } |
| 267 | |
| 268 | // copyKindDefaults defaults dst to the value in src if dst does not have a value set. |
| 269 | func copyKindDefaults(dst, src *schema.GroupVersionKind) { |
| 270 | if src == nil { |
| 271 | return |
| 272 | } |
| 273 | // apply kind and version defaulting from provided default |
| 274 | if len(dst.Kind) == 0 { |
| 275 | dst.Kind = src.Kind |
| 276 | } |
| 277 | if len(dst.Version) == 0 && len(src.Version) > 0 { |
| 278 | dst.Group = src.Group |
| 279 | dst.Version = src.Version |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | // bufferedMarshaller describes a more efficient marshalling interface that can avoid allocating multiple |
| 284 | // byte buffers by pre-calculating the size of the final buffer needed. |
| 285 | type bufferedMarshaller interface { |
| 286 | proto.Sizer |
| 287 | runtime.ProtobufMarshaller |
| 288 | } |
| 289 | |
| 290 | // estimateUnknownSize returns the expected bytes consumed by a given runtime.Unknown |
| 291 | // object with a nil RawJSON struct and the expected size of the provided buffer. The |
| 292 | // returned size will not be correct if RawJSOn is set on unk. |
| 293 | func estimateUnknownSize(unk *runtime.Unknown, byteSize uint64) uint64 { |
| 294 | size := uint64(unk.Size()) |
| 295 | // protobuf uses 1 byte for the tag, a varint for the length of the array (at most 8 bytes - uint64 - here), |
| 296 | // and the size of the array. |
| 297 | size += 1 + 8 + byteSize |
| 298 | return size |
| 299 | } |
| 300 | |
| 301 | // NewRawSerializer creates a Protobuf serializer that handles encoding versioned objects into the proper wire form. If typer |
| 302 | // is not nil, the object has the group, version, and kind fields set. This serializer does not provide type information for the |
| 303 | // encoded object, and thus is not self describing (callers must know what type is being described in order to decode). |
| 304 | // |
| 305 | // This encoding scheme is experimental, and is subject to change at any time. |
| 306 | func NewRawSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper, defaultContentType string) *RawSerializer { |
| 307 | return &RawSerializer{ |
| 308 | creater: creater, |
| 309 | typer: typer, |
| 310 | contentType: defaultContentType, |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | // RawSerializer encodes and decodes objects without adding a runtime.Unknown wrapper (objects are encoded without identifying |
| 315 | // type). |
| 316 | type RawSerializer struct { |
| 317 | creater runtime.ObjectCreater |
| 318 | typer runtime.ObjectTyper |
| 319 | contentType string |
| 320 | } |
| 321 | |
| 322 | var _ runtime.Serializer = &RawSerializer{} |
| 323 | |
| 324 | // Decode attempts to convert the provided data into a protobuf message, extract the stored schema kind, apply the provided default |
| 325 | // gvk, and then load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, |
| 326 | // the raw data will be extracted and no decoding will be performed. If into is not registered with the typer, then the object will |
| 327 | // be straight decoded using normal protobuf unmarshalling (the MarshalTo interface). If into is provided and the original data is |
| 328 | // not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. On success or most |
| 329 | // errors, the method will return the calculated schema kind. |
| 330 | func (s *RawSerializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { |
| 331 | if into == nil { |
| 332 | return nil, nil, fmt.Errorf("this serializer requires an object to decode into: %#v", s) |
| 333 | } |
| 334 | |
| 335 | if versioned, ok := into.(*runtime.VersionedObjects); ok { |
| 336 | into = versioned.Last() |
| 337 | obj, actual, err := s.Decode(originalData, gvk, into) |
| 338 | if err != nil { |
| 339 | return nil, actual, err |
| 340 | } |
| 341 | if into != nil && into != obj { |
| 342 | versioned.Objects = []runtime.Object{obj, into} |
| 343 | } else { |
| 344 | versioned.Objects = []runtime.Object{obj} |
| 345 | } |
| 346 | return versioned, actual, err |
| 347 | } |
| 348 | |
| 349 | if len(originalData) == 0 { |
| 350 | // TODO: treat like decoding {} from JSON with defaulting |
| 351 | return nil, nil, fmt.Errorf("empty data") |
| 352 | } |
| 353 | data := originalData |
| 354 | |
| 355 | actual := &schema.GroupVersionKind{} |
| 356 | copyKindDefaults(actual, gvk) |
| 357 | |
| 358 | if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil { |
| 359 | intoUnknown.Raw = data |
| 360 | intoUnknown.ContentEncoding = "" |
| 361 | intoUnknown.ContentType = s.contentType |
| 362 | intoUnknown.SetGroupVersionKind(*actual) |
| 363 | return intoUnknown, actual, nil |
| 364 | } |
| 365 | |
| 366 | types, _, err := s.typer.ObjectKinds(into) |
| 367 | switch { |
| 368 | case runtime.IsNotRegisteredError(err): |
| 369 | pb, ok := into.(proto.Message) |
| 370 | if !ok { |
| 371 | return nil, actual, errNotMarshalable{reflect.TypeOf(into)} |
| 372 | } |
| 373 | if err := proto.Unmarshal(data, pb); err != nil { |
| 374 | return nil, actual, err |
| 375 | } |
| 376 | return into, actual, nil |
| 377 | case err != nil: |
| 378 | return nil, actual, err |
| 379 | default: |
| 380 | copyKindDefaults(actual, &types[0]) |
| 381 | // if the result of defaulting did not set a version or group, ensure that at least group is set |
| 382 | // (copyKindDefaults will not assign Group if version is already set). This guarantees that the group |
| 383 | // of into is set if there is no better information from the caller or object. |
| 384 | if len(actual.Version) == 0 && len(actual.Group) == 0 { |
| 385 | actual.Group = types[0].Group |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | if len(actual.Kind) == 0 { |
| 390 | return nil, actual, runtime.NewMissingKindErr("<protobuf encoded body - must provide default type>") |
| 391 | } |
| 392 | if len(actual.Version) == 0 { |
| 393 | return nil, actual, runtime.NewMissingVersionErr("<protobuf encoded body - must provide default type>") |
| 394 | } |
| 395 | |
| 396 | return unmarshalToObject(s.typer, s.creater, actual, into, data) |
| 397 | } |
| 398 | |
| 399 | // unmarshalToObject is the common code between decode in the raw and normal serializer. |
| 400 | func unmarshalToObject(typer runtime.ObjectTyper, creater runtime.ObjectCreater, actual *schema.GroupVersionKind, into runtime.Object, data []byte) (runtime.Object, *schema.GroupVersionKind, error) { |
| 401 | // use the target if necessary |
| 402 | obj, err := runtime.UseOrCreateObject(typer, creater, *actual, into) |
| 403 | if err != nil { |
| 404 | return nil, actual, err |
| 405 | } |
| 406 | |
| 407 | pb, ok := obj.(proto.Message) |
| 408 | if !ok { |
| 409 | return nil, actual, errNotMarshalable{reflect.TypeOf(obj)} |
| 410 | } |
| 411 | if err := proto.Unmarshal(data, pb); err != nil { |
| 412 | return nil, actual, err |
| 413 | } |
| 414 | return obj, actual, nil |
| 415 | } |
| 416 | |
| 417 | // Encode serializes the provided object to the given writer. Overrides is ignored. |
| 418 | func (s *RawSerializer) Encode(obj runtime.Object, w io.Writer) error { |
| 419 | switch t := obj.(type) { |
| 420 | case bufferedMarshaller: |
| 421 | // this path performs a single allocation during write but requires the caller to implement |
| 422 | // the more efficient Size and MarshalTo methods |
| 423 | encodedSize := uint64(t.Size()) |
| 424 | data := make([]byte, encodedSize) |
| 425 | |
| 426 | n, err := t.MarshalTo(data) |
| 427 | if err != nil { |
| 428 | return err |
| 429 | } |
| 430 | _, err = w.Write(data[:n]) |
| 431 | return err |
| 432 | |
| 433 | case proto.Marshaler: |
| 434 | // this path performs extra allocations |
| 435 | data, err := t.Marshal() |
| 436 | if err != nil { |
| 437 | return err |
| 438 | } |
| 439 | _, err = w.Write(data) |
| 440 | return err |
| 441 | |
| 442 | default: |
| 443 | return errNotMarshalable{reflect.TypeOf(obj)} |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | var LengthDelimitedFramer = lengthDelimitedFramer{} |
| 448 | |
| 449 | type lengthDelimitedFramer struct{} |
| 450 | |
| 451 | // NewFrameWriter implements stream framing for this serializer |
| 452 | func (lengthDelimitedFramer) NewFrameWriter(w io.Writer) io.Writer { |
| 453 | return framer.NewLengthDelimitedFrameWriter(w) |
| 454 | } |
| 455 | |
| 456 | // NewFrameReader implements stream framing for this serializer |
| 457 | func (lengthDelimitedFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser { |
| 458 | return framer.NewLengthDelimitedFrameReader(r) |
| 459 | } |