blob: 456bfda47892addd67c924fb045c411823508321 [file] [log] [blame]
amit.ghosh258d14c2020-10-02 15:13:38 +02001// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package proto
6
7import (
8 "sort"
9
10 "google.golang.org/protobuf/encoding/protowire"
11 "google.golang.org/protobuf/internal/encoding/messageset"
12 "google.golang.org/protobuf/internal/fieldsort"
13 "google.golang.org/protobuf/internal/mapsort"
14 "google.golang.org/protobuf/internal/pragma"
15 "google.golang.org/protobuf/reflect/protoreflect"
16 "google.golang.org/protobuf/runtime/protoiface"
17)
18
19// MarshalOptions configures the marshaler.
20//
21// Example usage:
22// b, err := MarshalOptions{Deterministic: true}.Marshal(m)
23type MarshalOptions struct {
24 pragma.NoUnkeyedLiterals
25
26 // AllowPartial allows messages that have missing required fields to marshal
27 // without returning an error. If AllowPartial is false (the default),
28 // Marshal will return an error if there are any missing required fields.
29 AllowPartial bool
30
31 // Deterministic controls whether the same message will always be
32 // serialized to the same bytes within the same binary.
33 //
34 // Setting this option guarantees that repeated serialization of
35 // the same message will return the same bytes, and that different
36 // processes of the same binary (which may be executing on different
37 // machines) will serialize equal messages to the same bytes.
38 // It has no effect on the resulting size of the encoded message compared
39 // to a non-deterministic marshal.
40 //
41 // Note that the deterministic serialization is NOT canonical across
42 // languages. It is not guaranteed to remain stable over time. It is
43 // unstable across different builds with schema changes due to unknown
44 // fields. Users who need canonical serialization (e.g., persistent
45 // storage in a canonical form, fingerprinting, etc.) must define
46 // their own canonicalization specification and implement their own
47 // serializer rather than relying on this API.
48 //
49 // If deterministic serialization is requested, map entries will be
50 // sorted by keys in lexographical order. This is an implementation
51 // detail and subject to change.
52 Deterministic bool
53
54 // UseCachedSize indicates that the result of a previous Size call
55 // may be reused.
56 //
57 // Setting this option asserts that:
58 //
59 // 1. Size has previously been called on this message with identical
60 // options (except for UseCachedSize itself).
61 //
62 // 2. The message and all its submessages have not changed in any
63 // way since the Size call.
64 //
65 // If either of these invariants is violated,
66 // the results are undefined and may include panics or corrupted output.
67 //
68 // Implementations MAY take this option into account to provide
69 // better performance, but there is no guarantee that they will do so.
70 // There is absolutely no guarantee that Size followed by Marshal with
71 // UseCachedSize set will perform equivalently to Marshal alone.
72 UseCachedSize bool
73}
74
75// Marshal returns the wire-format encoding of m.
76func Marshal(m Message) ([]byte, error) {
77 // Treat nil message interface as an empty message; nothing to output.
78 if m == nil {
79 return nil, nil
80 }
81
82 out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
83 if len(out.Buf) == 0 && err == nil {
84 out.Buf = emptyBytesForMessage(m)
85 }
86 return out.Buf, err
87}
88
89// Marshal returns the wire-format encoding of m.
90func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
91 // Treat nil message interface as an empty message; nothing to output.
92 if m == nil {
93 return nil, nil
94 }
95
96 out, err := o.marshal(nil, m.ProtoReflect())
97 if len(out.Buf) == 0 && err == nil {
98 out.Buf = emptyBytesForMessage(m)
99 }
100 return out.Buf, err
101}
102
103// emptyBytesForMessage returns a nil buffer if and only if m is invalid,
104// otherwise it returns a non-nil empty buffer.
105//
106// This is to assist the edge-case where user-code does the following:
107// m1.OptionalBytes, _ = proto.Marshal(m2)
108// where they expect the proto2 "optional_bytes" field to be populated
109// if any only if m2 is a valid message.
110func emptyBytesForMessage(m Message) []byte {
111 if m == nil || !m.ProtoReflect().IsValid() {
112 return nil
113 }
114 return emptyBuf[:]
115}
116
117// MarshalAppend appends the wire-format encoding of m to b,
118// returning the result.
119func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
120 // Treat nil message interface as an empty message; nothing to append.
121 if m == nil {
122 return b, nil
123 }
124
125 out, err := o.marshal(b, m.ProtoReflect())
126 return out.Buf, err
127}
128
129// MarshalState returns the wire-format encoding of a message.
130//
131// This method permits fine-grained control over the marshaler.
132// Most users should use Marshal instead.
133func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
134 return o.marshal(in.Buf, in.Message)
135}
136
137func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
138 allowPartial := o.AllowPartial
139 o.AllowPartial = true
140 if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
141 !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
142 in := protoiface.MarshalInput{
143 Message: m,
144 Buf: b,
145 }
146 if o.Deterministic {
147 in.Flags |= protoiface.MarshalDeterministic
148 }
149 if o.UseCachedSize {
150 in.Flags |= protoiface.MarshalUseCachedSize
151 }
152 if methods.Size != nil {
153 sout := methods.Size(protoiface.SizeInput{
154 Message: m,
155 Flags: in.Flags,
156 })
157 if cap(b) < len(b)+sout.Size {
158 in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
159 copy(in.Buf, b)
160 }
161 in.Flags |= protoiface.MarshalUseCachedSize
162 }
163 out, err = methods.Marshal(in)
164 } else {
165 out.Buf, err = o.marshalMessageSlow(b, m)
166 }
167 if err != nil {
168 return out, err
169 }
170 if allowPartial {
171 return out, nil
172 }
173 return out, checkInitialized(m)
174}
175
176func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
177 out, err := o.marshal(b, m)
178 return out.Buf, err
179}
180
181// growcap scales up the capacity of a slice.
182//
183// Given a slice with a current capacity of oldcap and a desired
184// capacity of wantcap, growcap returns a new capacity >= wantcap.
185//
186// The algorithm is mostly identical to the one used by append as of Go 1.14.
187func growcap(oldcap, wantcap int) (newcap int) {
188 if wantcap > oldcap*2 {
189 newcap = wantcap
190 } else if oldcap < 1024 {
191 // The Go 1.14 runtime takes this case when len(s) < 1024,
192 // not when cap(s) < 1024. The difference doesn't seem
193 // significant here.
194 newcap = oldcap * 2
195 } else {
196 newcap = oldcap
197 for 0 < newcap && newcap < wantcap {
198 newcap += newcap / 4
199 }
200 if newcap <= 0 {
201 newcap = wantcap
202 }
203 }
204 return newcap
205}
206
207func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
208 if messageset.IsMessageSet(m.Descriptor()) {
209 return marshalMessageSet(b, m, o)
210 }
211 // There are many choices for what order we visit fields in. The default one here
212 // is chosen for reasonable efficiency and simplicity given the protoreflect API.
213 // It is not deterministic, since Message.Range does not return fields in any
214 // defined order.
215 //
216 // When using deterministic serialization, we sort the known fields.
217 var err error
218 o.rangeFields(m, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
219 b, err = o.marshalField(b, fd, v)
220 return err == nil
221 })
222 if err != nil {
223 return b, err
224 }
225 b = append(b, m.GetUnknown()...)
226 return b, nil
227}
228
229// rangeFields visits fields in a defined order when deterministic serialization is enabled.
230func (o MarshalOptions) rangeFields(m protoreflect.Message, f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) {
231 if !o.Deterministic {
232 m.Range(f)
233 return
234 }
235 var fds []protoreflect.FieldDescriptor
236 m.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool {
237 fds = append(fds, fd)
238 return true
239 })
240 sort.Slice(fds, func(a, b int) bool {
241 return fieldsort.Less(fds[a], fds[b])
242 })
243 for _, fd := range fds {
244 if !f(fd, m.Get(fd)) {
245 break
246 }
247 }
248}
249
250func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
251 switch {
252 case fd.IsList():
253 return o.marshalList(b, fd, value.List())
254 case fd.IsMap():
255 return o.marshalMap(b, fd, value.Map())
256 default:
257 b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
258 return o.marshalSingular(b, fd, value)
259 }
260}
261
262func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
263 if fd.IsPacked() && list.Len() > 0 {
264 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
265 b, pos := appendSpeculativeLength(b)
266 for i, llen := 0, list.Len(); i < llen; i++ {
267 var err error
268 b, err = o.marshalSingular(b, fd, list.Get(i))
269 if err != nil {
270 return b, err
271 }
272 }
273 b = finishSpeculativeLength(b, pos)
274 return b, nil
275 }
276
277 kind := fd.Kind()
278 for i, llen := 0, list.Len(); i < llen; i++ {
279 var err error
280 b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
281 b, err = o.marshalSingular(b, fd, list.Get(i))
282 if err != nil {
283 return b, err
284 }
285 }
286 return b, nil
287}
288
289func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
290 keyf := fd.MapKey()
291 valf := fd.MapValue()
292 var err error
293 o.rangeMap(mapv, keyf.Kind(), func(key protoreflect.MapKey, value protoreflect.Value) bool {
294 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
295 var pos int
296 b, pos = appendSpeculativeLength(b)
297
298 b, err = o.marshalField(b, keyf, key.Value())
299 if err != nil {
300 return false
301 }
302 b, err = o.marshalField(b, valf, value)
303 if err != nil {
304 return false
305 }
306 b = finishSpeculativeLength(b, pos)
307 return true
308 })
309 return b, err
310}
311
312func (o MarshalOptions) rangeMap(mapv protoreflect.Map, kind protoreflect.Kind, f func(protoreflect.MapKey, protoreflect.Value) bool) {
313 if !o.Deterministic {
314 mapv.Range(f)
315 return
316 }
317 mapsort.Range(mapv, kind, f)
318}
319
320// When encoding length-prefixed fields, we speculatively set aside some number of bytes
321// for the length, encode the data, and then encode the length (shifting the data if necessary
322// to make room).
323const speculativeLength = 1
324
325func appendSpeculativeLength(b []byte) ([]byte, int) {
326 pos := len(b)
327 b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
328 return b, pos
329}
330
331func finishSpeculativeLength(b []byte, pos int) []byte {
332 mlen := len(b) - pos - speculativeLength
333 msiz := protowire.SizeVarint(uint64(mlen))
334 if msiz != speculativeLength {
335 for i := 0; i < msiz-speculativeLength; i++ {
336 b = append(b, 0)
337 }
338 copy(b[pos+msiz:], b[pos+speculativeLength:])
339 b = b[:pos+msiz+mlen]
340 }
341 protowire.AppendVarint(b[:pos], uint64(mlen))
342 return b
343}