Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 1 | /* |
| 2 | Copyright 2015 The Kubernetes Authors. |
| 3 | |
| 4 | Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | you may not use this file except in compliance with the License. |
| 6 | You may obtain a copy of the License at |
| 7 | |
| 8 | http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | Unless required by applicable law or agreed to in writing, software |
| 11 | distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | See the License for the specific language governing permissions and |
| 14 | limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package protobuf |
| 18 | |
| 19 | import ( |
| 20 | "bytes" |
| 21 | "fmt" |
| 22 | "io" |
| 23 | "net/http" |
| 24 | "reflect" |
| 25 | |
| 26 | "github.com/gogo/protobuf/proto" |
| 27 | |
| 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 29 | "k8s.io/apimachinery/pkg/runtime" |
| 30 | "k8s.io/apimachinery/pkg/runtime/schema" |
| 31 | "k8s.io/apimachinery/pkg/runtime/serializer/recognizer" |
| 32 | "k8s.io/apimachinery/pkg/util/framer" |
| 33 | ) |
| 34 | |
| 35 | var ( |
| 36 | // protoEncodingPrefix serves as a magic number for an encoded protobuf message on this serializer. All |
| 37 | // proto messages serialized by this schema will be preceded by the bytes 0x6b 0x38 0x73, with the fourth |
| 38 | // byte being reserved for the encoding style. The only encoding style defined is 0x00, which means that |
| 39 | // the rest of the byte stream is a message of type k8s.io.kubernetes.pkg.runtime.Unknown (proto2). |
| 40 | // |
| 41 | // See k8s.io/apimachinery/pkg/runtime/generated.proto for details of the runtime.Unknown message. |
| 42 | // |
| 43 | // This encoding scheme is experimental, and is subject to change at any time. |
| 44 | protoEncodingPrefix = []byte{0x6b, 0x38, 0x73, 0x00} |
| 45 | ) |
| 46 | |
| 47 | type errNotMarshalable struct { |
| 48 | t reflect.Type |
| 49 | } |
| 50 | |
| 51 | func (e errNotMarshalable) Error() string { |
| 52 | return fmt.Sprintf("object %v does not implement the protobuf marshalling interface and cannot be encoded to a protobuf message", e.t) |
| 53 | } |
| 54 | |
| 55 | func (e errNotMarshalable) Status() metav1.Status { |
| 56 | return metav1.Status{ |
| 57 | Status: metav1.StatusFailure, |
| 58 | Code: http.StatusNotAcceptable, |
| 59 | Reason: metav1.StatusReason("NotAcceptable"), |
| 60 | Message: e.Error(), |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func IsNotMarshalable(err error) bool { |
| 65 | _, ok := err.(errNotMarshalable) |
| 66 | return err != nil && ok |
| 67 | } |
| 68 | |
| 69 | // NewSerializer creates a Protobuf serializer that handles encoding versioned objects into the proper wire form. If a typer |
| 70 | // is passed, the encoded object will have group, version, and kind fields set. If typer is nil, the objects will be written |
| 71 | // as-is (any type info passed with the object will be used). |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 72 | func NewSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper) *Serializer { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 73 | return &Serializer{ |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 74 | prefix: protoEncodingPrefix, |
| 75 | creater: creater, |
| 76 | typer: typer, |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 77 | } |
| 78 | } |
| 79 | |
| 80 | type Serializer struct { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 81 | prefix []byte |
| 82 | creater runtime.ObjectCreater |
| 83 | typer runtime.ObjectTyper |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 84 | } |
| 85 | |
| 86 | var _ runtime.Serializer = &Serializer{} |
| 87 | var _ recognizer.RecognizingDecoder = &Serializer{} |
| 88 | |
| 89 | // Decode attempts to convert the provided data into a protobuf message, extract the stored schema kind, apply the provided default |
| 90 | // gvk, and then load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, |
| 91 | // the raw data will be extracted and no decoding will be performed. If into is not registered with the typer, then the object will |
| 92 | // be straight decoded using normal protobuf unmarshalling (the MarshalTo interface). If into is provided and the original data is |
| 93 | // not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. On success or most |
| 94 | // errors, the method will return the calculated schema kind. |
| 95 | func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { |
| 96 | if versioned, ok := into.(*runtime.VersionedObjects); ok { |
| 97 | into = versioned.Last() |
| 98 | obj, actual, err := s.Decode(originalData, gvk, into) |
| 99 | if err != nil { |
| 100 | return nil, actual, err |
| 101 | } |
| 102 | // the last item in versioned becomes into, so if versioned was not originally empty we reset the object |
| 103 | // array so the first position is the decoded object and the second position is the outermost object. |
| 104 | // if there were no objects in the versioned list passed to us, only add ourselves. |
| 105 | if into != nil && into != obj { |
| 106 | versioned.Objects = []runtime.Object{obj, into} |
| 107 | } else { |
| 108 | versioned.Objects = []runtime.Object{obj} |
| 109 | } |
| 110 | return versioned, actual, err |
| 111 | } |
| 112 | |
| 113 | prefixLen := len(s.prefix) |
| 114 | switch { |
| 115 | case len(originalData) == 0: |
| 116 | // TODO: treat like decoding {} from JSON with defaulting |
| 117 | return nil, nil, fmt.Errorf("empty data") |
| 118 | case len(originalData) < prefixLen || !bytes.Equal(s.prefix, originalData[:prefixLen]): |
| 119 | return nil, nil, fmt.Errorf("provided data does not appear to be a protobuf message, expected prefix %v", s.prefix) |
| 120 | case len(originalData) == prefixLen: |
| 121 | // TODO: treat like decoding {} from JSON with defaulting |
| 122 | return nil, nil, fmt.Errorf("empty body") |
| 123 | } |
| 124 | |
| 125 | data := originalData[prefixLen:] |
| 126 | unk := runtime.Unknown{} |
| 127 | if err := unk.Unmarshal(data); err != nil { |
| 128 | return nil, nil, err |
| 129 | } |
| 130 | |
| 131 | actual := unk.GroupVersionKind() |
| 132 | copyKindDefaults(&actual, gvk) |
| 133 | |
| 134 | if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil { |
| 135 | *intoUnknown = unk |
| 136 | if ok, _, _ := s.RecognizesData(bytes.NewBuffer(unk.Raw)); ok { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 137 | intoUnknown.ContentType = runtime.ContentTypeProtobuf |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 138 | } |
| 139 | return intoUnknown, &actual, nil |
| 140 | } |
| 141 | |
| 142 | if into != nil { |
| 143 | types, _, err := s.typer.ObjectKinds(into) |
| 144 | switch { |
| 145 | case runtime.IsNotRegisteredError(err): |
| 146 | pb, ok := into.(proto.Message) |
| 147 | if !ok { |
| 148 | return nil, &actual, errNotMarshalable{reflect.TypeOf(into)} |
| 149 | } |
| 150 | if err := proto.Unmarshal(unk.Raw, pb); err != nil { |
| 151 | return nil, &actual, err |
| 152 | } |
| 153 | return into, &actual, nil |
| 154 | case err != nil: |
| 155 | return nil, &actual, err |
| 156 | default: |
| 157 | copyKindDefaults(&actual, &types[0]) |
| 158 | // if the result of defaulting did not set a version or group, ensure that at least group is set |
| 159 | // (copyKindDefaults will not assign Group if version is already set). This guarantees that the group |
| 160 | // of into is set if there is no better information from the caller or object. |
| 161 | if len(actual.Version) == 0 && len(actual.Group) == 0 { |
| 162 | actual.Group = types[0].Group |
| 163 | } |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | if len(actual.Kind) == 0 { |
| 168 | return nil, &actual, runtime.NewMissingKindErr(fmt.Sprintf("%#v", unk.TypeMeta)) |
| 169 | } |
| 170 | if len(actual.Version) == 0 { |
| 171 | return nil, &actual, runtime.NewMissingVersionErr(fmt.Sprintf("%#v", unk.TypeMeta)) |
| 172 | } |
| 173 | |
| 174 | return unmarshalToObject(s.typer, s.creater, &actual, into, unk.Raw) |
| 175 | } |
| 176 | |
| 177 | // Encode serializes the provided object to the given writer. |
| 178 | func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error { |
| 179 | prefixSize := uint64(len(s.prefix)) |
| 180 | |
| 181 | var unk runtime.Unknown |
| 182 | switch t := obj.(type) { |
| 183 | case *runtime.Unknown: |
| 184 | estimatedSize := prefixSize + uint64(t.Size()) |
| 185 | data := make([]byte, estimatedSize) |
| 186 | i, err := t.MarshalTo(data[prefixSize:]) |
| 187 | if err != nil { |
| 188 | return err |
| 189 | } |
| 190 | copy(data, s.prefix) |
| 191 | _, err = w.Write(data[:prefixSize+uint64(i)]) |
| 192 | return err |
| 193 | default: |
| 194 | kind := obj.GetObjectKind().GroupVersionKind() |
| 195 | unk = runtime.Unknown{ |
| 196 | TypeMeta: runtime.TypeMeta{ |
| 197 | Kind: kind.Kind, |
| 198 | APIVersion: kind.GroupVersion().String(), |
| 199 | }, |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | switch t := obj.(type) { |
| 204 | case bufferedMarshaller: |
| 205 | // this path performs a single allocation during write but requires the caller to implement |
| 206 | // the more efficient Size and MarshalTo methods |
| 207 | encodedSize := uint64(t.Size()) |
| 208 | estimatedSize := prefixSize + estimateUnknownSize(&unk, encodedSize) |
| 209 | data := make([]byte, estimatedSize) |
| 210 | |
| 211 | i, err := unk.NestedMarshalTo(data[prefixSize:], t, encodedSize) |
| 212 | if err != nil { |
| 213 | return err |
| 214 | } |
| 215 | |
| 216 | copy(data, s.prefix) |
| 217 | |
| 218 | _, err = w.Write(data[:prefixSize+uint64(i)]) |
| 219 | return err |
| 220 | |
| 221 | case proto.Marshaler: |
| 222 | // this path performs extra allocations |
| 223 | data, err := t.Marshal() |
| 224 | if err != nil { |
| 225 | return err |
| 226 | } |
| 227 | unk.Raw = data |
| 228 | |
| 229 | estimatedSize := prefixSize + uint64(unk.Size()) |
| 230 | data = make([]byte, estimatedSize) |
| 231 | |
| 232 | i, err := unk.MarshalTo(data[prefixSize:]) |
| 233 | if err != nil { |
| 234 | return err |
| 235 | } |
| 236 | |
| 237 | copy(data, s.prefix) |
| 238 | |
| 239 | _, err = w.Write(data[:prefixSize+uint64(i)]) |
| 240 | return err |
| 241 | |
| 242 | default: |
| 243 | // TODO: marshal with a different content type and serializer (JSON for third party objects) |
| 244 | return errNotMarshalable{reflect.TypeOf(obj)} |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | // RecognizesData implements the RecognizingDecoder interface. |
| 249 | func (s *Serializer) RecognizesData(peek io.Reader) (bool, bool, error) { |
| 250 | prefix := make([]byte, 4) |
| 251 | n, err := peek.Read(prefix) |
| 252 | if err != nil { |
| 253 | if err == io.EOF { |
| 254 | return false, false, nil |
| 255 | } |
| 256 | return false, false, err |
| 257 | } |
| 258 | if n != 4 { |
| 259 | return false, false, nil |
| 260 | } |
| 261 | return bytes.Equal(s.prefix, prefix), false, nil |
| 262 | } |
| 263 | |
| 264 | // copyKindDefaults defaults dst to the value in src if dst does not have a value set. |
| 265 | func copyKindDefaults(dst, src *schema.GroupVersionKind) { |
| 266 | if src == nil { |
| 267 | return |
| 268 | } |
| 269 | // apply kind and version defaulting from provided default |
| 270 | if len(dst.Kind) == 0 { |
| 271 | dst.Kind = src.Kind |
| 272 | } |
| 273 | if len(dst.Version) == 0 && len(src.Version) > 0 { |
| 274 | dst.Group = src.Group |
| 275 | dst.Version = src.Version |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // bufferedMarshaller describes a more efficient marshalling interface that can avoid allocating multiple |
| 280 | // byte buffers by pre-calculating the size of the final buffer needed. |
| 281 | type bufferedMarshaller interface { |
| 282 | proto.Sizer |
| 283 | runtime.ProtobufMarshaller |
| 284 | } |
| 285 | |
| 286 | // estimateUnknownSize returns the expected bytes consumed by a given runtime.Unknown |
| 287 | // object with a nil RawJSON struct and the expected size of the provided buffer. The |
| 288 | // returned size will not be correct if RawJSOn is set on unk. |
| 289 | func estimateUnknownSize(unk *runtime.Unknown, byteSize uint64) uint64 { |
| 290 | size := uint64(unk.Size()) |
| 291 | // protobuf uses 1 byte for the tag, a varint for the length of the array (at most 8 bytes - uint64 - here), |
| 292 | // and the size of the array. |
| 293 | size += 1 + 8 + byteSize |
| 294 | return size |
| 295 | } |
| 296 | |
| 297 | // NewRawSerializer creates a Protobuf serializer that handles encoding versioned objects into the proper wire form. If typer |
| 298 | // is not nil, the object has the group, version, and kind fields set. This serializer does not provide type information for the |
| 299 | // encoded object, and thus is not self describing (callers must know what type is being described in order to decode). |
| 300 | // |
| 301 | // This encoding scheme is experimental, and is subject to change at any time. |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 302 | func NewRawSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper) *RawSerializer { |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 303 | return &RawSerializer{ |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 304 | creater: creater, |
| 305 | typer: typer, |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 306 | } |
| 307 | } |
| 308 | |
| 309 | // RawSerializer encodes and decodes objects without adding a runtime.Unknown wrapper (objects are encoded without identifying |
| 310 | // type). |
| 311 | type RawSerializer struct { |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 312 | creater runtime.ObjectCreater |
| 313 | typer runtime.ObjectTyper |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 314 | } |
| 315 | |
| 316 | var _ runtime.Serializer = &RawSerializer{} |
| 317 | |
| 318 | // Decode attempts to convert the provided data into a protobuf message, extract the stored schema kind, apply the provided default |
| 319 | // gvk, and then load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, |
| 320 | // the raw data will be extracted and no decoding will be performed. If into is not registered with the typer, then the object will |
| 321 | // be straight decoded using normal protobuf unmarshalling (the MarshalTo interface). If into is provided and the original data is |
| 322 | // not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. On success or most |
| 323 | // errors, the method will return the calculated schema kind. |
| 324 | func (s *RawSerializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { |
| 325 | if into == nil { |
| 326 | return nil, nil, fmt.Errorf("this serializer requires an object to decode into: %#v", s) |
| 327 | } |
| 328 | |
| 329 | if versioned, ok := into.(*runtime.VersionedObjects); ok { |
| 330 | into = versioned.Last() |
| 331 | obj, actual, err := s.Decode(originalData, gvk, into) |
| 332 | if err != nil { |
| 333 | return nil, actual, err |
| 334 | } |
| 335 | if into != nil && into != obj { |
| 336 | versioned.Objects = []runtime.Object{obj, into} |
| 337 | } else { |
| 338 | versioned.Objects = []runtime.Object{obj} |
| 339 | } |
| 340 | return versioned, actual, err |
| 341 | } |
| 342 | |
| 343 | if len(originalData) == 0 { |
| 344 | // TODO: treat like decoding {} from JSON with defaulting |
| 345 | return nil, nil, fmt.Errorf("empty data") |
| 346 | } |
| 347 | data := originalData |
| 348 | |
| 349 | actual := &schema.GroupVersionKind{} |
| 350 | copyKindDefaults(actual, gvk) |
| 351 | |
| 352 | if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil { |
| 353 | intoUnknown.Raw = data |
| 354 | intoUnknown.ContentEncoding = "" |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 355 | intoUnknown.ContentType = runtime.ContentTypeProtobuf |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 356 | intoUnknown.SetGroupVersionKind(*actual) |
| 357 | return intoUnknown, actual, nil |
| 358 | } |
| 359 | |
| 360 | types, _, err := s.typer.ObjectKinds(into) |
| 361 | switch { |
| 362 | case runtime.IsNotRegisteredError(err): |
| 363 | pb, ok := into.(proto.Message) |
| 364 | if !ok { |
| 365 | return nil, actual, errNotMarshalable{reflect.TypeOf(into)} |
| 366 | } |
| 367 | if err := proto.Unmarshal(data, pb); err != nil { |
| 368 | return nil, actual, err |
| 369 | } |
| 370 | return into, actual, nil |
| 371 | case err != nil: |
| 372 | return nil, actual, err |
| 373 | default: |
| 374 | copyKindDefaults(actual, &types[0]) |
| 375 | // if the result of defaulting did not set a version or group, ensure that at least group is set |
| 376 | // (copyKindDefaults will not assign Group if version is already set). This guarantees that the group |
| 377 | // of into is set if there is no better information from the caller or object. |
| 378 | if len(actual.Version) == 0 && len(actual.Group) == 0 { |
| 379 | actual.Group = types[0].Group |
| 380 | } |
| 381 | } |
| 382 | |
| 383 | if len(actual.Kind) == 0 { |
| 384 | return nil, actual, runtime.NewMissingKindErr("<protobuf encoded body - must provide default type>") |
| 385 | } |
| 386 | if len(actual.Version) == 0 { |
| 387 | return nil, actual, runtime.NewMissingVersionErr("<protobuf encoded body - must provide default type>") |
| 388 | } |
| 389 | |
| 390 | return unmarshalToObject(s.typer, s.creater, actual, into, data) |
| 391 | } |
| 392 | |
| 393 | // unmarshalToObject is the common code between decode in the raw and normal serializer. |
| 394 | func unmarshalToObject(typer runtime.ObjectTyper, creater runtime.ObjectCreater, actual *schema.GroupVersionKind, into runtime.Object, data []byte) (runtime.Object, *schema.GroupVersionKind, error) { |
| 395 | // use the target if necessary |
| 396 | obj, err := runtime.UseOrCreateObject(typer, creater, *actual, into) |
| 397 | if err != nil { |
| 398 | return nil, actual, err |
| 399 | } |
| 400 | |
| 401 | pb, ok := obj.(proto.Message) |
| 402 | if !ok { |
| 403 | return nil, actual, errNotMarshalable{reflect.TypeOf(obj)} |
| 404 | } |
| 405 | if err := proto.Unmarshal(data, pb); err != nil { |
| 406 | return nil, actual, err |
| 407 | } |
David Bainbridge | 8697152 | 2019-09-26 22:09:39 +0000 | [diff] [blame] | 408 | if actual != nil { |
| 409 | obj.GetObjectKind().SetGroupVersionKind(*actual) |
| 410 | } |
Zack Williams | e940c7a | 2019-08-21 14:25:39 -0700 | [diff] [blame] | 411 | return obj, actual, nil |
| 412 | } |
| 413 | |
| 414 | // Encode serializes the provided object to the given writer. Overrides is ignored. |
| 415 | func (s *RawSerializer) Encode(obj runtime.Object, w io.Writer) error { |
| 416 | switch t := obj.(type) { |
| 417 | case bufferedMarshaller: |
| 418 | // this path performs a single allocation during write but requires the caller to implement |
| 419 | // the more efficient Size and MarshalTo methods |
| 420 | encodedSize := uint64(t.Size()) |
| 421 | data := make([]byte, encodedSize) |
| 422 | |
| 423 | n, err := t.MarshalTo(data) |
| 424 | if err != nil { |
| 425 | return err |
| 426 | } |
| 427 | _, err = w.Write(data[:n]) |
| 428 | return err |
| 429 | |
| 430 | case proto.Marshaler: |
| 431 | // this path performs extra allocations |
| 432 | data, err := t.Marshal() |
| 433 | if err != nil { |
| 434 | return err |
| 435 | } |
| 436 | _, err = w.Write(data) |
| 437 | return err |
| 438 | |
| 439 | default: |
| 440 | return errNotMarshalable{reflect.TypeOf(obj)} |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | var LengthDelimitedFramer = lengthDelimitedFramer{} |
| 445 | |
| 446 | type lengthDelimitedFramer struct{} |
| 447 | |
| 448 | // NewFrameWriter implements stream framing for this serializer |
| 449 | func (lengthDelimitedFramer) NewFrameWriter(w io.Writer) io.Writer { |
| 450 | return framer.NewLengthDelimitedFrameWriter(w) |
| 451 | } |
| 452 | |
| 453 | // NewFrameReader implements stream framing for this serializer |
| 454 | func (lengthDelimitedFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser { |
| 455 | return framer.NewLengthDelimitedFrameReader(r) |
| 456 | } |