blob: 7b47a1180e4d555518d64903975f267273e0cd11 [file] [log] [blame]
Matteo Scandoloa4285862020-12-01 18:10:10 -08001// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package proto
6
7import (
8 "sort"
9
10 "google.golang.org/protobuf/encoding/protowire"
11 "google.golang.org/protobuf/internal/encoding/messageset"
12 "google.golang.org/protobuf/internal/fieldsort"
13 "google.golang.org/protobuf/internal/mapsort"
14 "google.golang.org/protobuf/internal/pragma"
15 "google.golang.org/protobuf/reflect/protoreflect"
16 "google.golang.org/protobuf/runtime/protoiface"
17)
18
19// MarshalOptions configures the marshaler.
20//
21// Example usage:
22// b, err := MarshalOptions{Deterministic: true}.Marshal(m)
23type MarshalOptions struct {
24 pragma.NoUnkeyedLiterals
25
26 // AllowPartial allows messages that have missing required fields to marshal
27 // without returning an error. If AllowPartial is false (the default),
28 // Marshal will return an error if there are any missing required fields.
29 AllowPartial bool
30
31 // Deterministic controls whether the same message will always be
32 // serialized to the same bytes within the same binary.
33 //
34 // Setting this option guarantees that repeated serialization of
35 // the same message will return the same bytes, and that different
36 // processes of the same binary (which may be executing on different
37 // machines) will serialize equal messages to the same bytes.
38 // It has no effect on the resulting size of the encoded message compared
39 // to a non-deterministic marshal.
40 //
41 // Note that the deterministic serialization is NOT canonical across
42 // languages. It is not guaranteed to remain stable over time. It is
43 // unstable across different builds with schema changes due to unknown
44 // fields. Users who need canonical serialization (e.g., persistent
45 // storage in a canonical form, fingerprinting, etc.) must define
46 // their own canonicalization specification and implement their own
47 // serializer rather than relying on this API.
48 //
49 // If deterministic serialization is requested, map entries will be
50 // sorted by keys in lexographical order. This is an implementation
51 // detail and subject to change.
52 Deterministic bool
53
54 // UseCachedSize indicates that the result of a previous Size call
55 // may be reused.
56 //
57 // Setting this option asserts that:
58 //
59 // 1. Size has previously been called on this message with identical
60 // options (except for UseCachedSize itself).
61 //
62 // 2. The message and all its submessages have not changed in any
63 // way since the Size call.
64 //
65 // If either of these invariants is violated,
66 // the results are undefined and may include panics or corrupted output.
67 //
68 // Implementations MAY take this option into account to provide
69 // better performance, but there is no guarantee that they will do so.
70 // There is absolutely no guarantee that Size followed by Marshal with
71 // UseCachedSize set will perform equivalently to Marshal alone.
72 UseCachedSize bool
73}
74
75// Marshal returns the wire-format encoding of m.
76func Marshal(m Message) ([]byte, error) {
77 // Treat nil message interface as an empty message; nothing to output.
78 if m == nil {
79 return nil, nil
80 }
81
82 out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
83 if len(out.Buf) == 0 && err == nil {
84 out.Buf = emptyBytesForMessage(m)
85 }
86 return out.Buf, err
87}
88
89// Marshal returns the wire-format encoding of m.
90func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
91 // Treat nil message interface as an empty message; nothing to output.
92 if m == nil {
93 return nil, nil
94 }
95
96 out, err := o.marshal(nil, m.ProtoReflect())
97 if len(out.Buf) == 0 && err == nil {
98 out.Buf = emptyBytesForMessage(m)
99 }
100 return out.Buf, err
101}
102
103// emptyBytesForMessage returns a nil buffer if and only if m is invalid,
104// otherwise it returns a non-nil empty buffer.
105//
106// This is to assist the edge-case where user-code does the following:
107// m1.OptionalBytes, _ = proto.Marshal(m2)
108// where they expect the proto2 "optional_bytes" field to be populated
109// if any only if m2 is a valid message.
110func emptyBytesForMessage(m Message) []byte {
111 if m == nil || !m.ProtoReflect().IsValid() {
112 return nil
113 }
114 return emptyBuf[:]
115}
116
117// MarshalAppend appends the wire-format encoding of m to b,
118// returning the result.
119func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
120 // Treat nil message interface as an empty message; nothing to append.
121 if m == nil {
122 return b, nil
123 }
124
125 out, err := o.marshal(b, m.ProtoReflect())
126 return out.Buf, err
127}
128
129// MarshalState returns the wire-format encoding of a message.
130//
131// This method permits fine-grained control over the marshaler.
132// Most users should use Marshal instead.
133func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
134 return o.marshal(in.Buf, in.Message)
135}
136
137// marshal is a centralized function that all marshal operations go through.
138// For profiling purposes, avoid changing the name of this function or
139// introducing other code paths for marshal that do not go through this.
140func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
141 allowPartial := o.AllowPartial
142 o.AllowPartial = true
143 if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
144 !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
145 in := protoiface.MarshalInput{
146 Message: m,
147 Buf: b,
148 }
149 if o.Deterministic {
150 in.Flags |= protoiface.MarshalDeterministic
151 }
152 if o.UseCachedSize {
153 in.Flags |= protoiface.MarshalUseCachedSize
154 }
155 if methods.Size != nil {
156 sout := methods.Size(protoiface.SizeInput{
157 Message: m,
158 Flags: in.Flags,
159 })
160 if cap(b) < len(b)+sout.Size {
161 in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
162 copy(in.Buf, b)
163 }
164 in.Flags |= protoiface.MarshalUseCachedSize
165 }
166 out, err = methods.Marshal(in)
167 } else {
168 out.Buf, err = o.marshalMessageSlow(b, m)
169 }
170 if err != nil {
171 return out, err
172 }
173 if allowPartial {
174 return out, nil
175 }
176 return out, checkInitialized(m)
177}
178
179func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
180 out, err := o.marshal(b, m)
181 return out.Buf, err
182}
183
184// growcap scales up the capacity of a slice.
185//
186// Given a slice with a current capacity of oldcap and a desired
187// capacity of wantcap, growcap returns a new capacity >= wantcap.
188//
189// The algorithm is mostly identical to the one used by append as of Go 1.14.
190func growcap(oldcap, wantcap int) (newcap int) {
191 if wantcap > oldcap*2 {
192 newcap = wantcap
193 } else if oldcap < 1024 {
194 // The Go 1.14 runtime takes this case when len(s) < 1024,
195 // not when cap(s) < 1024. The difference doesn't seem
196 // significant here.
197 newcap = oldcap * 2
198 } else {
199 newcap = oldcap
200 for 0 < newcap && newcap < wantcap {
201 newcap += newcap / 4
202 }
203 if newcap <= 0 {
204 newcap = wantcap
205 }
206 }
207 return newcap
208}
209
210func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
211 if messageset.IsMessageSet(m.Descriptor()) {
212 return o.marshalMessageSet(b, m)
213 }
214 // There are many choices for what order we visit fields in. The default one here
215 // is chosen for reasonable efficiency and simplicity given the protoreflect API.
216 // It is not deterministic, since Message.Range does not return fields in any
217 // defined order.
218 //
219 // When using deterministic serialization, we sort the known fields.
220 var err error
221 o.rangeFields(m, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
222 b, err = o.marshalField(b, fd, v)
223 return err == nil
224 })
225 if err != nil {
226 return b, err
227 }
228 b = append(b, m.GetUnknown()...)
229 return b, nil
230}
231
232// rangeFields visits fields in a defined order when deterministic serialization is enabled.
233func (o MarshalOptions) rangeFields(m protoreflect.Message, f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) {
234 if !o.Deterministic {
235 m.Range(f)
236 return
237 }
238 var fds []protoreflect.FieldDescriptor
239 m.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool {
240 fds = append(fds, fd)
241 return true
242 })
243 sort.Slice(fds, func(a, b int) bool {
244 return fieldsort.Less(fds[a], fds[b])
245 })
246 for _, fd := range fds {
247 if !f(fd, m.Get(fd)) {
248 break
249 }
250 }
251}
252
253func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
254 switch {
255 case fd.IsList():
256 return o.marshalList(b, fd, value.List())
257 case fd.IsMap():
258 return o.marshalMap(b, fd, value.Map())
259 default:
260 b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
261 return o.marshalSingular(b, fd, value)
262 }
263}
264
265func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
266 if fd.IsPacked() && list.Len() > 0 {
267 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
268 b, pos := appendSpeculativeLength(b)
269 for i, llen := 0, list.Len(); i < llen; i++ {
270 var err error
271 b, err = o.marshalSingular(b, fd, list.Get(i))
272 if err != nil {
273 return b, err
274 }
275 }
276 b = finishSpeculativeLength(b, pos)
277 return b, nil
278 }
279
280 kind := fd.Kind()
281 for i, llen := 0, list.Len(); i < llen; i++ {
282 var err error
283 b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
284 b, err = o.marshalSingular(b, fd, list.Get(i))
285 if err != nil {
286 return b, err
287 }
288 }
289 return b, nil
290}
291
292func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
293 keyf := fd.MapKey()
294 valf := fd.MapValue()
295 var err error
296 o.rangeMap(mapv, keyf.Kind(), func(key protoreflect.MapKey, value protoreflect.Value) bool {
297 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
298 var pos int
299 b, pos = appendSpeculativeLength(b)
300
301 b, err = o.marshalField(b, keyf, key.Value())
302 if err != nil {
303 return false
304 }
305 b, err = o.marshalField(b, valf, value)
306 if err != nil {
307 return false
308 }
309 b = finishSpeculativeLength(b, pos)
310 return true
311 })
312 return b, err
313}
314
315func (o MarshalOptions) rangeMap(mapv protoreflect.Map, kind protoreflect.Kind, f func(protoreflect.MapKey, protoreflect.Value) bool) {
316 if !o.Deterministic {
317 mapv.Range(f)
318 return
319 }
320 mapsort.Range(mapv, kind, f)
321}
322
323// When encoding length-prefixed fields, we speculatively set aside some number of bytes
324// for the length, encode the data, and then encode the length (shifting the data if necessary
325// to make room).
326const speculativeLength = 1
327
328func appendSpeculativeLength(b []byte) ([]byte, int) {
329 pos := len(b)
330 b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
331 return b, pos
332}
333
334func finishSpeculativeLength(b []byte, pos int) []byte {
335 mlen := len(b) - pos - speculativeLength
336 msiz := protowire.SizeVarint(uint64(mlen))
337 if msiz != speculativeLength {
338 for i := 0; i < msiz-speculativeLength; i++ {
339 b = append(b, 0)
340 }
341 copy(b[pos+msiz:], b[pos+speculativeLength:])
342 b = b[:pos+msiz+mlen]
343 }
344 protowire.AppendVarint(b[:pos], uint64(mlen))
345 return b
346}