blob: d18239c23723a8768a38d9bdb22cad8d8131a799 [file] [log] [blame]
David K. Bainbridgebd6b2882021-08-26 13:31:02 +00001// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package proto
6
7import (
8 "google.golang.org/protobuf/encoding/protowire"
9 "google.golang.org/protobuf/internal/encoding/messageset"
10 "google.golang.org/protobuf/internal/order"
11 "google.golang.org/protobuf/internal/pragma"
12 "google.golang.org/protobuf/reflect/protoreflect"
13 "google.golang.org/protobuf/runtime/protoiface"
14)
15
16// MarshalOptions configures the marshaler.
17//
18// Example usage:
19// b, err := MarshalOptions{Deterministic: true}.Marshal(m)
20type MarshalOptions struct {
21 pragma.NoUnkeyedLiterals
22
23 // AllowPartial allows messages that have missing required fields to marshal
24 // without returning an error. If AllowPartial is false (the default),
25 // Marshal will return an error if there are any missing required fields.
26 AllowPartial bool
27
28 // Deterministic controls whether the same message will always be
29 // serialized to the same bytes within the same binary.
30 //
31 // Setting this option guarantees that repeated serialization of
32 // the same message will return the same bytes, and that different
33 // processes of the same binary (which may be executing on different
34 // machines) will serialize equal messages to the same bytes.
35 // It has no effect on the resulting size of the encoded message compared
36 // to a non-deterministic marshal.
37 //
38 // Note that the deterministic serialization is NOT canonical across
39 // languages. It is not guaranteed to remain stable over time. It is
40 // unstable across different builds with schema changes due to unknown
41 // fields. Users who need canonical serialization (e.g., persistent
42 // storage in a canonical form, fingerprinting, etc.) must define
43 // their own canonicalization specification and implement their own
44 // serializer rather than relying on this API.
45 //
46 // If deterministic serialization is requested, map entries will be
47 // sorted by keys in lexographical order. This is an implementation
48 // detail and subject to change.
49 Deterministic bool
50
51 // UseCachedSize indicates that the result of a previous Size call
52 // may be reused.
53 //
54 // Setting this option asserts that:
55 //
56 // 1. Size has previously been called on this message with identical
57 // options (except for UseCachedSize itself).
58 //
59 // 2. The message and all its submessages have not changed in any
60 // way since the Size call.
61 //
62 // If either of these invariants is violated,
63 // the results are undefined and may include panics or corrupted output.
64 //
65 // Implementations MAY take this option into account to provide
66 // better performance, but there is no guarantee that they will do so.
67 // There is absolutely no guarantee that Size followed by Marshal with
68 // UseCachedSize set will perform equivalently to Marshal alone.
69 UseCachedSize bool
70}
71
72// Marshal returns the wire-format encoding of m.
73func Marshal(m Message) ([]byte, error) {
74 // Treat nil message interface as an empty message; nothing to output.
75 if m == nil {
76 return nil, nil
77 }
78
79 out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
80 if len(out.Buf) == 0 && err == nil {
81 out.Buf = emptyBytesForMessage(m)
82 }
83 return out.Buf, err
84}
85
86// Marshal returns the wire-format encoding of m.
87func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
88 // Treat nil message interface as an empty message; nothing to output.
89 if m == nil {
90 return nil, nil
91 }
92
93 out, err := o.marshal(nil, m.ProtoReflect())
94 if len(out.Buf) == 0 && err == nil {
95 out.Buf = emptyBytesForMessage(m)
96 }
97 return out.Buf, err
98}
99
100// emptyBytesForMessage returns a nil buffer if and only if m is invalid,
101// otherwise it returns a non-nil empty buffer.
102//
103// This is to assist the edge-case where user-code does the following:
104// m1.OptionalBytes, _ = proto.Marshal(m2)
105// where they expect the proto2 "optional_bytes" field to be populated
106// if any only if m2 is a valid message.
107func emptyBytesForMessage(m Message) []byte {
108 if m == nil || !m.ProtoReflect().IsValid() {
109 return nil
110 }
111 return emptyBuf[:]
112}
113
114// MarshalAppend appends the wire-format encoding of m to b,
115// returning the result.
116func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
117 // Treat nil message interface as an empty message; nothing to append.
118 if m == nil {
119 return b, nil
120 }
121
122 out, err := o.marshal(b, m.ProtoReflect())
123 return out.Buf, err
124}
125
126// MarshalState returns the wire-format encoding of a message.
127//
128// This method permits fine-grained control over the marshaler.
129// Most users should use Marshal instead.
130func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
131 return o.marshal(in.Buf, in.Message)
132}
133
134// marshal is a centralized function that all marshal operations go through.
135// For profiling purposes, avoid changing the name of this function or
136// introducing other code paths for marshal that do not go through this.
137func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
138 allowPartial := o.AllowPartial
139 o.AllowPartial = true
140 if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
141 !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
142 in := protoiface.MarshalInput{
143 Message: m,
144 Buf: b,
145 }
146 if o.Deterministic {
147 in.Flags |= protoiface.MarshalDeterministic
148 }
149 if o.UseCachedSize {
150 in.Flags |= protoiface.MarshalUseCachedSize
151 }
152 if methods.Size != nil {
153 sout := methods.Size(protoiface.SizeInput{
154 Message: m,
155 Flags: in.Flags,
156 })
157 if cap(b) < len(b)+sout.Size {
158 in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
159 copy(in.Buf, b)
160 }
161 in.Flags |= protoiface.MarshalUseCachedSize
162 }
163 out, err = methods.Marshal(in)
164 } else {
165 out.Buf, err = o.marshalMessageSlow(b, m)
166 }
167 if err != nil {
168 return out, err
169 }
170 if allowPartial {
171 return out, nil
172 }
173 return out, checkInitialized(m)
174}
175
176func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
177 out, err := o.marshal(b, m)
178 return out.Buf, err
179}
180
181// growcap scales up the capacity of a slice.
182//
183// Given a slice with a current capacity of oldcap and a desired
184// capacity of wantcap, growcap returns a new capacity >= wantcap.
185//
186// The algorithm is mostly identical to the one used by append as of Go 1.14.
187func growcap(oldcap, wantcap int) (newcap int) {
188 if wantcap > oldcap*2 {
189 newcap = wantcap
190 } else if oldcap < 1024 {
191 // The Go 1.14 runtime takes this case when len(s) < 1024,
192 // not when cap(s) < 1024. The difference doesn't seem
193 // significant here.
194 newcap = oldcap * 2
195 } else {
196 newcap = oldcap
197 for 0 < newcap && newcap < wantcap {
198 newcap += newcap / 4
199 }
200 if newcap <= 0 {
201 newcap = wantcap
202 }
203 }
204 return newcap
205}
206
207func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
208 if messageset.IsMessageSet(m.Descriptor()) {
209 return o.marshalMessageSet(b, m)
210 }
211 fieldOrder := order.AnyFieldOrder
212 if o.Deterministic {
213 // TODO: This should use a more natural ordering like NumberFieldOrder,
214 // but doing so breaks golden tests that make invalid assumption about
215 // output stability of this implementation.
216 fieldOrder = order.LegacyFieldOrder
217 }
218 var err error
219 order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
220 b, err = o.marshalField(b, fd, v)
221 return err == nil
222 })
223 if err != nil {
224 return b, err
225 }
226 b = append(b, m.GetUnknown()...)
227 return b, nil
228}
229
230func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
231 switch {
232 case fd.IsList():
233 return o.marshalList(b, fd, value.List())
234 case fd.IsMap():
235 return o.marshalMap(b, fd, value.Map())
236 default:
237 b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
238 return o.marshalSingular(b, fd, value)
239 }
240}
241
242func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
243 if fd.IsPacked() && list.Len() > 0 {
244 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
245 b, pos := appendSpeculativeLength(b)
246 for i, llen := 0, list.Len(); i < llen; i++ {
247 var err error
248 b, err = o.marshalSingular(b, fd, list.Get(i))
249 if err != nil {
250 return b, err
251 }
252 }
253 b = finishSpeculativeLength(b, pos)
254 return b, nil
255 }
256
257 kind := fd.Kind()
258 for i, llen := 0, list.Len(); i < llen; i++ {
259 var err error
260 b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
261 b, err = o.marshalSingular(b, fd, list.Get(i))
262 if err != nil {
263 return b, err
264 }
265 }
266 return b, nil
267}
268
269func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
270 keyf := fd.MapKey()
271 valf := fd.MapValue()
272 keyOrder := order.AnyKeyOrder
273 if o.Deterministic {
274 keyOrder = order.GenericKeyOrder
275 }
276 var err error
277 order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool {
278 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
279 var pos int
280 b, pos = appendSpeculativeLength(b)
281
282 b, err = o.marshalField(b, keyf, key.Value())
283 if err != nil {
284 return false
285 }
286 b, err = o.marshalField(b, valf, value)
287 if err != nil {
288 return false
289 }
290 b = finishSpeculativeLength(b, pos)
291 return true
292 })
293 return b, err
294}
295
296// When encoding length-prefixed fields, we speculatively set aside some number of bytes
297// for the length, encode the data, and then encode the length (shifting the data if necessary
298// to make room).
299const speculativeLength = 1
300
301func appendSpeculativeLength(b []byte) ([]byte, int) {
302 pos := len(b)
303 b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
304 return b, pos
305}
306
307func finishSpeculativeLength(b []byte, pos int) []byte {
308 mlen := len(b) - pos - speculativeLength
309 msiz := protowire.SizeVarint(uint64(mlen))
310 if msiz != speculativeLength {
311 for i := 0; i < msiz-speculativeLength; i++ {
312 b = append(b, 0)
313 }
314 copy(b[pos+msiz:], b[pos+speculativeLength:])
315 b = b[:pos+msiz+mlen]
316 }
317 protowire.AppendVarint(b[:pos], uint64(mlen))
318 return b
319}