Matteo Scandolo | a428586 | 2020-12-01 18:10:10 -0800 | [diff] [blame^] | 1 | // Copyright 2019 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package proto |
| 6 | |
| 7 | import ( |
| 8 | "sort" |
| 9 | |
| 10 | "google.golang.org/protobuf/encoding/protowire" |
| 11 | "google.golang.org/protobuf/internal/encoding/messageset" |
| 12 | "google.golang.org/protobuf/internal/fieldsort" |
| 13 | "google.golang.org/protobuf/internal/mapsort" |
| 14 | "google.golang.org/protobuf/internal/pragma" |
| 15 | "google.golang.org/protobuf/reflect/protoreflect" |
| 16 | "google.golang.org/protobuf/runtime/protoiface" |
| 17 | ) |
| 18 | |
| 19 | // MarshalOptions configures the marshaler. |
| 20 | // |
| 21 | // Example usage: |
| 22 | // b, err := MarshalOptions{Deterministic: true}.Marshal(m) |
| 23 | type MarshalOptions struct { |
| 24 | pragma.NoUnkeyedLiterals |
| 25 | |
| 26 | // AllowPartial allows messages that have missing required fields to marshal |
| 27 | // without returning an error. If AllowPartial is false (the default), |
| 28 | // Marshal will return an error if there are any missing required fields. |
| 29 | AllowPartial bool |
| 30 | |
| 31 | // Deterministic controls whether the same message will always be |
| 32 | // serialized to the same bytes within the same binary. |
| 33 | // |
| 34 | // Setting this option guarantees that repeated serialization of |
| 35 | // the same message will return the same bytes, and that different |
| 36 | // processes of the same binary (which may be executing on different |
| 37 | // machines) will serialize equal messages to the same bytes. |
| 38 | // It has no effect on the resulting size of the encoded message compared |
| 39 | // to a non-deterministic marshal. |
| 40 | // |
| 41 | // Note that the deterministic serialization is NOT canonical across |
| 42 | // languages. It is not guaranteed to remain stable over time. It is |
| 43 | // unstable across different builds with schema changes due to unknown |
| 44 | // fields. Users who need canonical serialization (e.g., persistent |
| 45 | // storage in a canonical form, fingerprinting, etc.) must define |
| 46 | // their own canonicalization specification and implement their own |
| 47 | // serializer rather than relying on this API. |
| 48 | // |
| 49 | // If deterministic serialization is requested, map entries will be |
| 50 | // sorted by keys in lexographical order. This is an implementation |
| 51 | // detail and subject to change. |
| 52 | Deterministic bool |
| 53 | |
| 54 | // UseCachedSize indicates that the result of a previous Size call |
| 55 | // may be reused. |
| 56 | // |
| 57 | // Setting this option asserts that: |
| 58 | // |
| 59 | // 1. Size has previously been called on this message with identical |
| 60 | // options (except for UseCachedSize itself). |
| 61 | // |
| 62 | // 2. The message and all its submessages have not changed in any |
| 63 | // way since the Size call. |
| 64 | // |
| 65 | // If either of these invariants is violated, |
| 66 | // the results are undefined and may include panics or corrupted output. |
| 67 | // |
| 68 | // Implementations MAY take this option into account to provide |
| 69 | // better performance, but there is no guarantee that they will do so. |
| 70 | // There is absolutely no guarantee that Size followed by Marshal with |
| 71 | // UseCachedSize set will perform equivalently to Marshal alone. |
| 72 | UseCachedSize bool |
| 73 | } |
| 74 | |
| 75 | // Marshal returns the wire-format encoding of m. |
| 76 | func Marshal(m Message) ([]byte, error) { |
| 77 | // Treat nil message interface as an empty message; nothing to output. |
| 78 | if m == nil { |
| 79 | return nil, nil |
| 80 | } |
| 81 | |
| 82 | out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect()) |
| 83 | if len(out.Buf) == 0 && err == nil { |
| 84 | out.Buf = emptyBytesForMessage(m) |
| 85 | } |
| 86 | return out.Buf, err |
| 87 | } |
| 88 | |
| 89 | // Marshal returns the wire-format encoding of m. |
| 90 | func (o MarshalOptions) Marshal(m Message) ([]byte, error) { |
| 91 | // Treat nil message interface as an empty message; nothing to output. |
| 92 | if m == nil { |
| 93 | return nil, nil |
| 94 | } |
| 95 | |
| 96 | out, err := o.marshal(nil, m.ProtoReflect()) |
| 97 | if len(out.Buf) == 0 && err == nil { |
| 98 | out.Buf = emptyBytesForMessage(m) |
| 99 | } |
| 100 | return out.Buf, err |
| 101 | } |
| 102 | |
| 103 | // emptyBytesForMessage returns a nil buffer if and only if m is invalid, |
| 104 | // otherwise it returns a non-nil empty buffer. |
| 105 | // |
| 106 | // This is to assist the edge-case where user-code does the following: |
| 107 | // m1.OptionalBytes, _ = proto.Marshal(m2) |
| 108 | // where they expect the proto2 "optional_bytes" field to be populated |
| 109 | // if any only if m2 is a valid message. |
| 110 | func emptyBytesForMessage(m Message) []byte { |
| 111 | if m == nil || !m.ProtoReflect().IsValid() { |
| 112 | return nil |
| 113 | } |
| 114 | return emptyBuf[:] |
| 115 | } |
| 116 | |
| 117 | // MarshalAppend appends the wire-format encoding of m to b, |
| 118 | // returning the result. |
| 119 | func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) { |
| 120 | // Treat nil message interface as an empty message; nothing to append. |
| 121 | if m == nil { |
| 122 | return b, nil |
| 123 | } |
| 124 | |
| 125 | out, err := o.marshal(b, m.ProtoReflect()) |
| 126 | return out.Buf, err |
| 127 | } |
| 128 | |
| 129 | // MarshalState returns the wire-format encoding of a message. |
| 130 | // |
| 131 | // This method permits fine-grained control over the marshaler. |
| 132 | // Most users should use Marshal instead. |
| 133 | func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) { |
| 134 | return o.marshal(in.Buf, in.Message) |
| 135 | } |
| 136 | |
| 137 | // marshal is a centralized function that all marshal operations go through. |
| 138 | // For profiling purposes, avoid changing the name of this function or |
| 139 | // introducing other code paths for marshal that do not go through this. |
| 140 | func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) { |
| 141 | allowPartial := o.AllowPartial |
| 142 | o.AllowPartial = true |
| 143 | if methods := protoMethods(m); methods != nil && methods.Marshal != nil && |
| 144 | !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) { |
| 145 | in := protoiface.MarshalInput{ |
| 146 | Message: m, |
| 147 | Buf: b, |
| 148 | } |
| 149 | if o.Deterministic { |
| 150 | in.Flags |= protoiface.MarshalDeterministic |
| 151 | } |
| 152 | if o.UseCachedSize { |
| 153 | in.Flags |= protoiface.MarshalUseCachedSize |
| 154 | } |
| 155 | if methods.Size != nil { |
| 156 | sout := methods.Size(protoiface.SizeInput{ |
| 157 | Message: m, |
| 158 | Flags: in.Flags, |
| 159 | }) |
| 160 | if cap(b) < len(b)+sout.Size { |
| 161 | in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size)) |
| 162 | copy(in.Buf, b) |
| 163 | } |
| 164 | in.Flags |= protoiface.MarshalUseCachedSize |
| 165 | } |
| 166 | out, err = methods.Marshal(in) |
| 167 | } else { |
| 168 | out.Buf, err = o.marshalMessageSlow(b, m) |
| 169 | } |
| 170 | if err != nil { |
| 171 | return out, err |
| 172 | } |
| 173 | if allowPartial { |
| 174 | return out, nil |
| 175 | } |
| 176 | return out, checkInitialized(m) |
| 177 | } |
| 178 | |
| 179 | func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) { |
| 180 | out, err := o.marshal(b, m) |
| 181 | return out.Buf, err |
| 182 | } |
| 183 | |
| 184 | // growcap scales up the capacity of a slice. |
| 185 | // |
| 186 | // Given a slice with a current capacity of oldcap and a desired |
| 187 | // capacity of wantcap, growcap returns a new capacity >= wantcap. |
| 188 | // |
| 189 | // The algorithm is mostly identical to the one used by append as of Go 1.14. |
| 190 | func growcap(oldcap, wantcap int) (newcap int) { |
| 191 | if wantcap > oldcap*2 { |
| 192 | newcap = wantcap |
| 193 | } else if oldcap < 1024 { |
| 194 | // The Go 1.14 runtime takes this case when len(s) < 1024, |
| 195 | // not when cap(s) < 1024. The difference doesn't seem |
| 196 | // significant here. |
| 197 | newcap = oldcap * 2 |
| 198 | } else { |
| 199 | newcap = oldcap |
| 200 | for 0 < newcap && newcap < wantcap { |
| 201 | newcap += newcap / 4 |
| 202 | } |
| 203 | if newcap <= 0 { |
| 204 | newcap = wantcap |
| 205 | } |
| 206 | } |
| 207 | return newcap |
| 208 | } |
| 209 | |
| 210 | func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) { |
| 211 | if messageset.IsMessageSet(m.Descriptor()) { |
| 212 | return o.marshalMessageSet(b, m) |
| 213 | } |
| 214 | // There are many choices for what order we visit fields in. The default one here |
| 215 | // is chosen for reasonable efficiency and simplicity given the protoreflect API. |
| 216 | // It is not deterministic, since Message.Range does not return fields in any |
| 217 | // defined order. |
| 218 | // |
| 219 | // When using deterministic serialization, we sort the known fields. |
| 220 | var err error |
| 221 | o.rangeFields(m, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { |
| 222 | b, err = o.marshalField(b, fd, v) |
| 223 | return err == nil |
| 224 | }) |
| 225 | if err != nil { |
| 226 | return b, err |
| 227 | } |
| 228 | b = append(b, m.GetUnknown()...) |
| 229 | return b, nil |
| 230 | } |
| 231 | |
| 232 | // rangeFields visits fields in a defined order when deterministic serialization is enabled. |
| 233 | func (o MarshalOptions) rangeFields(m protoreflect.Message, f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) { |
| 234 | if !o.Deterministic { |
| 235 | m.Range(f) |
| 236 | return |
| 237 | } |
| 238 | var fds []protoreflect.FieldDescriptor |
| 239 | m.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool { |
| 240 | fds = append(fds, fd) |
| 241 | return true |
| 242 | }) |
| 243 | sort.Slice(fds, func(a, b int) bool { |
| 244 | return fieldsort.Less(fds[a], fds[b]) |
| 245 | }) |
| 246 | for _, fd := range fds { |
| 247 | if !f(fd, m.Get(fd)) { |
| 248 | break |
| 249 | } |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) { |
| 254 | switch { |
| 255 | case fd.IsList(): |
| 256 | return o.marshalList(b, fd, value.List()) |
| 257 | case fd.IsMap(): |
| 258 | return o.marshalMap(b, fd, value.Map()) |
| 259 | default: |
| 260 | b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()]) |
| 261 | return o.marshalSingular(b, fd, value) |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) { |
| 266 | if fd.IsPacked() && list.Len() > 0 { |
| 267 | b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) |
| 268 | b, pos := appendSpeculativeLength(b) |
| 269 | for i, llen := 0, list.Len(); i < llen; i++ { |
| 270 | var err error |
| 271 | b, err = o.marshalSingular(b, fd, list.Get(i)) |
| 272 | if err != nil { |
| 273 | return b, err |
| 274 | } |
| 275 | } |
| 276 | b = finishSpeculativeLength(b, pos) |
| 277 | return b, nil |
| 278 | } |
| 279 | |
| 280 | kind := fd.Kind() |
| 281 | for i, llen := 0, list.Len(); i < llen; i++ { |
| 282 | var err error |
| 283 | b = protowire.AppendTag(b, fd.Number(), wireTypes[kind]) |
| 284 | b, err = o.marshalSingular(b, fd, list.Get(i)) |
| 285 | if err != nil { |
| 286 | return b, err |
| 287 | } |
| 288 | } |
| 289 | return b, nil |
| 290 | } |
| 291 | |
| 292 | func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) { |
| 293 | keyf := fd.MapKey() |
| 294 | valf := fd.MapValue() |
| 295 | var err error |
| 296 | o.rangeMap(mapv, keyf.Kind(), func(key protoreflect.MapKey, value protoreflect.Value) bool { |
| 297 | b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) |
| 298 | var pos int |
| 299 | b, pos = appendSpeculativeLength(b) |
| 300 | |
| 301 | b, err = o.marshalField(b, keyf, key.Value()) |
| 302 | if err != nil { |
| 303 | return false |
| 304 | } |
| 305 | b, err = o.marshalField(b, valf, value) |
| 306 | if err != nil { |
| 307 | return false |
| 308 | } |
| 309 | b = finishSpeculativeLength(b, pos) |
| 310 | return true |
| 311 | }) |
| 312 | return b, err |
| 313 | } |
| 314 | |
| 315 | func (o MarshalOptions) rangeMap(mapv protoreflect.Map, kind protoreflect.Kind, f func(protoreflect.MapKey, protoreflect.Value) bool) { |
| 316 | if !o.Deterministic { |
| 317 | mapv.Range(f) |
| 318 | return |
| 319 | } |
| 320 | mapsort.Range(mapv, kind, f) |
| 321 | } |
| 322 | |
| 323 | // When encoding length-prefixed fields, we speculatively set aside some number of bytes |
| 324 | // for the length, encode the data, and then encode the length (shifting the data if necessary |
| 325 | // to make room). |
| 326 | const speculativeLength = 1 |
| 327 | |
| 328 | func appendSpeculativeLength(b []byte) ([]byte, int) { |
| 329 | pos := len(b) |
| 330 | b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...) |
| 331 | return b, pos |
| 332 | } |
| 333 | |
| 334 | func finishSpeculativeLength(b []byte, pos int) []byte { |
| 335 | mlen := len(b) - pos - speculativeLength |
| 336 | msiz := protowire.SizeVarint(uint64(mlen)) |
| 337 | if msiz != speculativeLength { |
| 338 | for i := 0; i < msiz-speculativeLength; i++ { |
| 339 | b = append(b, 0) |
| 340 | } |
| 341 | copy(b[pos+msiz:], b[pos+speculativeLength:]) |
| 342 | b = b[:pos+msiz+mlen] |
| 343 | } |
| 344 | protowire.AppendVarint(b[:pos], uint64(mlen)) |
| 345 | return b |
| 346 | } |