Matteo Scandolo | a428586 | 2020-12-01 18:10:10 -0800 | [diff] [blame] | 1 | package yaml |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "reflect" |
| 9 | "strconv" |
| 10 | |
| 11 | "gopkg.in/yaml.v2" |
| 12 | ) |
| 13 | |
| 14 | // Marshal marshals the object into JSON then converts JSON to YAML and returns the |
| 15 | // YAML. |
| 16 | func Marshal(o interface{}) ([]byte, error) { |
| 17 | j, err := json.Marshal(o) |
| 18 | if err != nil { |
| 19 | return nil, fmt.Errorf("error marshaling into JSON: %v", err) |
| 20 | } |
| 21 | |
| 22 | y, err := JSONToYAML(j) |
| 23 | if err != nil { |
| 24 | return nil, fmt.Errorf("error converting JSON to YAML: %v", err) |
| 25 | } |
| 26 | |
| 27 | return y, nil |
| 28 | } |
| 29 | |
| 30 | // JSONOpt is a decoding option for decoding from JSON format. |
| 31 | type JSONOpt func(*json.Decoder) *json.Decoder |
| 32 | |
| 33 | // Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, |
| 34 | // optionally configuring the behavior of the JSON unmarshal. |
| 35 | func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error { |
| 36 | return yamlUnmarshal(y, o, false, opts...) |
| 37 | } |
| 38 | |
| 39 | // UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal |
| 40 | // into an object, optionally configuring the behavior of the JSON unmarshal. |
| 41 | func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error { |
| 42 | return yamlUnmarshal(y, o, true, append(opts, DisallowUnknownFields)...) |
| 43 | } |
| 44 | |
| 45 | // yamlUnmarshal unmarshals the given YAML byte stream into the given interface, |
| 46 | // optionally performing the unmarshalling strictly |
| 47 | func yamlUnmarshal(y []byte, o interface{}, strict bool, opts ...JSONOpt) error { |
| 48 | vo := reflect.ValueOf(o) |
| 49 | unmarshalFn := yaml.Unmarshal |
| 50 | if strict { |
| 51 | unmarshalFn = yaml.UnmarshalStrict |
| 52 | } |
| 53 | j, err := yamlToJSON(y, &vo, unmarshalFn) |
| 54 | if err != nil { |
| 55 | return fmt.Errorf("error converting YAML to JSON: %v", err) |
| 56 | } |
| 57 | |
| 58 | err = jsonUnmarshal(bytes.NewReader(j), o, opts...) |
| 59 | if err != nil { |
| 60 | return fmt.Errorf("error unmarshaling JSON: %v", err) |
| 61 | } |
| 62 | |
| 63 | return nil |
| 64 | } |
| 65 | |
| 66 | // jsonUnmarshal unmarshals the JSON byte stream from the given reader into the |
| 67 | // object, optionally applying decoder options prior to decoding. We are not |
| 68 | // using json.Unmarshal directly as we want the chance to pass in non-default |
| 69 | // options. |
| 70 | func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error { |
| 71 | d := json.NewDecoder(r) |
| 72 | for _, opt := range opts { |
| 73 | d = opt(d) |
| 74 | } |
| 75 | if err := d.Decode(&o); err != nil { |
| 76 | return fmt.Errorf("while decoding JSON: %v", err) |
| 77 | } |
| 78 | return nil |
| 79 | } |
| 80 | |
| 81 | // JSONToYAML Converts JSON to YAML. |
| 82 | func JSONToYAML(j []byte) ([]byte, error) { |
| 83 | // Convert the JSON to an object. |
| 84 | var jsonObj interface{} |
| 85 | // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the |
| 86 | // Go JSON library doesn't try to pick the right number type (int, float, |
| 87 | // etc.) when unmarshalling to interface{}, it just picks float64 |
| 88 | // universally. go-yaml does go through the effort of picking the right |
| 89 | // number type, so we can preserve number type throughout this process. |
| 90 | err := yaml.Unmarshal(j, &jsonObj) |
| 91 | if err != nil { |
| 92 | return nil, err |
| 93 | } |
| 94 | |
| 95 | // Marshal this object into YAML. |
| 96 | return yaml.Marshal(jsonObj) |
| 97 | } |
| 98 | |
| 99 | // YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, |
| 100 | // passing JSON through this method should be a no-op. |
| 101 | // |
| 102 | // Things YAML can do that are not supported by JSON: |
| 103 | // * In YAML you can have binary and null keys in your maps. These are invalid |
| 104 | // in JSON. (int and float keys are converted to strings.) |
| 105 | // * Binary data in YAML with the !!binary tag is not supported. If you want to |
| 106 | // use binary data with this library, encode the data as base64 as usual but do |
| 107 | // not use the !!binary tag in your YAML. This will ensure the original base64 |
| 108 | // encoded data makes it all the way through to the JSON. |
| 109 | // |
| 110 | // For strict decoding of YAML, use YAMLToJSONStrict. |
| 111 | func YAMLToJSON(y []byte) ([]byte, error) { |
| 112 | return yamlToJSON(y, nil, yaml.Unmarshal) |
| 113 | } |
| 114 | |
| 115 | // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, |
| 116 | // returning an error on any duplicate field names. |
| 117 | func YAMLToJSONStrict(y []byte) ([]byte, error) { |
| 118 | return yamlToJSON(y, nil, yaml.UnmarshalStrict) |
| 119 | } |
| 120 | |
| 121 | func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) { |
| 122 | // Convert the YAML to an object. |
| 123 | var yamlObj interface{} |
| 124 | err := yamlUnmarshal(y, &yamlObj) |
| 125 | if err != nil { |
| 126 | return nil, err |
| 127 | } |
| 128 | |
| 129 | // YAML objects are not completely compatible with JSON objects (e.g. you |
| 130 | // can have non-string keys in YAML). So, convert the YAML-compatible object |
| 131 | // to a JSON-compatible object, failing with an error if irrecoverable |
| 132 | // incompatibilties happen along the way. |
| 133 | jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget) |
| 134 | if err != nil { |
| 135 | return nil, err |
| 136 | } |
| 137 | |
| 138 | // Convert this object to JSON and return the data. |
| 139 | return json.Marshal(jsonObj) |
| 140 | } |
| 141 | |
| 142 | func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { |
| 143 | var err error |
| 144 | |
| 145 | // Resolve jsonTarget to a concrete value (i.e. not a pointer or an |
| 146 | // interface). We pass decodingNull as false because we're not actually |
| 147 | // decoding into the value, we're just checking if the ultimate target is a |
| 148 | // string. |
| 149 | if jsonTarget != nil { |
| 150 | ju, tu, pv := indirect(*jsonTarget, false) |
| 151 | // We have a JSON or Text Umarshaler at this level, so we can't be trying |
| 152 | // to decode into a string. |
| 153 | if ju != nil || tu != nil { |
| 154 | jsonTarget = nil |
| 155 | } else { |
| 156 | jsonTarget = &pv |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // If yamlObj is a number or a boolean, check if jsonTarget is a string - |
| 161 | // if so, coerce. Else return normal. |
| 162 | // If yamlObj is a map or array, find the field that each key is |
| 163 | // unmarshaling to, and when you recurse pass the reflect.Value for that |
| 164 | // field back into this function. |
| 165 | switch typedYAMLObj := yamlObj.(type) { |
| 166 | case map[interface{}]interface{}: |
| 167 | // JSON does not support arbitrary keys in a map, so we must convert |
| 168 | // these keys to strings. |
| 169 | // |
| 170 | // From my reading of go-yaml v2 (specifically the resolve function), |
| 171 | // keys can only have the types string, int, int64, float64, binary |
| 172 | // (unsupported), or null (unsupported). |
| 173 | strMap := make(map[string]interface{}) |
| 174 | for k, v := range typedYAMLObj { |
| 175 | // Resolve the key to a string first. |
| 176 | var keyString string |
| 177 | switch typedKey := k.(type) { |
| 178 | case string: |
| 179 | keyString = typedKey |
| 180 | case int: |
| 181 | keyString = strconv.Itoa(typedKey) |
| 182 | case int64: |
| 183 | // go-yaml will only return an int64 as a key if the system |
| 184 | // architecture is 32-bit and the key's value is between 32-bit |
| 185 | // and 64-bit. Otherwise the key type will simply be int. |
| 186 | keyString = strconv.FormatInt(typedKey, 10) |
| 187 | case float64: |
| 188 | // Stolen from go-yaml to use the same conversion to string as |
| 189 | // the go-yaml library uses to convert float to string when |
| 190 | // Marshaling. |
| 191 | s := strconv.FormatFloat(typedKey, 'g', -1, 32) |
| 192 | switch s { |
| 193 | case "+Inf": |
| 194 | s = ".inf" |
| 195 | case "-Inf": |
| 196 | s = "-.inf" |
| 197 | case "NaN": |
| 198 | s = ".nan" |
| 199 | } |
| 200 | keyString = s |
| 201 | case bool: |
| 202 | if typedKey { |
| 203 | keyString = "true" |
| 204 | } else { |
| 205 | keyString = "false" |
| 206 | } |
| 207 | default: |
| 208 | return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v", |
| 209 | reflect.TypeOf(k), k, v) |
| 210 | } |
| 211 | |
| 212 | // jsonTarget should be a struct or a map. If it's a struct, find |
| 213 | // the field it's going to map to and pass its reflect.Value. If |
| 214 | // it's a map, find the element type of the map and pass the |
| 215 | // reflect.Value created from that type. If it's neither, just pass |
| 216 | // nil - JSON conversion will error for us if it's a real issue. |
| 217 | if jsonTarget != nil { |
| 218 | t := *jsonTarget |
| 219 | if t.Kind() == reflect.Struct { |
| 220 | keyBytes := []byte(keyString) |
| 221 | // Find the field that the JSON library would use. |
| 222 | var f *field |
| 223 | fields := cachedTypeFields(t.Type()) |
| 224 | for i := range fields { |
| 225 | ff := &fields[i] |
| 226 | if bytes.Equal(ff.nameBytes, keyBytes) { |
| 227 | f = ff |
| 228 | break |
| 229 | } |
| 230 | // Do case-insensitive comparison. |
| 231 | if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { |
| 232 | f = ff |
| 233 | } |
| 234 | } |
| 235 | if f != nil { |
| 236 | // Find the reflect.Value of the most preferential |
| 237 | // struct field. |
| 238 | jtf := t.Field(f.index[0]) |
| 239 | strMap[keyString], err = convertToJSONableObject(v, &jtf) |
| 240 | if err != nil { |
| 241 | return nil, err |
| 242 | } |
| 243 | continue |
| 244 | } |
| 245 | } else if t.Kind() == reflect.Map { |
| 246 | // Create a zero value of the map's element type to use as |
| 247 | // the JSON target. |
| 248 | jtv := reflect.Zero(t.Type().Elem()) |
| 249 | strMap[keyString], err = convertToJSONableObject(v, &jtv) |
| 250 | if err != nil { |
| 251 | return nil, err |
| 252 | } |
| 253 | continue |
| 254 | } |
| 255 | } |
| 256 | strMap[keyString], err = convertToJSONableObject(v, nil) |
| 257 | if err != nil { |
| 258 | return nil, err |
| 259 | } |
| 260 | } |
| 261 | return strMap, nil |
| 262 | case []interface{}: |
| 263 | // We need to recurse into arrays in case there are any |
| 264 | // map[interface{}]interface{}'s inside and to convert any |
| 265 | // numbers to strings. |
| 266 | |
| 267 | // If jsonTarget is a slice (which it really should be), find the |
| 268 | // thing it's going to map to. If it's not a slice, just pass nil |
| 269 | // - JSON conversion will error for us if it's a real issue. |
| 270 | var jsonSliceElemValue *reflect.Value |
| 271 | if jsonTarget != nil { |
| 272 | t := *jsonTarget |
| 273 | if t.Kind() == reflect.Slice { |
| 274 | // By default slices point to nil, but we need a reflect.Value |
| 275 | // pointing to a value of the slice type, so we create one here. |
| 276 | ev := reflect.Indirect(reflect.New(t.Type().Elem())) |
| 277 | jsonSliceElemValue = &ev |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | // Make and use a new array. |
| 282 | arr := make([]interface{}, len(typedYAMLObj)) |
| 283 | for i, v := range typedYAMLObj { |
| 284 | arr[i], err = convertToJSONableObject(v, jsonSliceElemValue) |
| 285 | if err != nil { |
| 286 | return nil, err |
| 287 | } |
| 288 | } |
| 289 | return arr, nil |
| 290 | default: |
| 291 | // If the target type is a string and the YAML type is a number, |
| 292 | // convert the YAML type to a string. |
| 293 | if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String { |
| 294 | // Based on my reading of go-yaml, it may return int, int64, |
| 295 | // float64, or uint64. |
| 296 | var s string |
| 297 | switch typedVal := typedYAMLObj.(type) { |
| 298 | case int: |
| 299 | s = strconv.FormatInt(int64(typedVal), 10) |
| 300 | case int64: |
| 301 | s = strconv.FormatInt(typedVal, 10) |
| 302 | case float64: |
| 303 | s = strconv.FormatFloat(typedVal, 'g', -1, 32) |
| 304 | case uint64: |
| 305 | s = strconv.FormatUint(typedVal, 10) |
| 306 | case bool: |
| 307 | if typedVal { |
| 308 | s = "true" |
| 309 | } else { |
| 310 | s = "false" |
| 311 | } |
| 312 | } |
| 313 | if len(s) > 0 { |
| 314 | yamlObj = interface{}(s) |
| 315 | } |
| 316 | } |
| 317 | return yamlObj, nil |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | // JSONObjectToYAMLObject converts an in-memory JSON object into a YAML in-memory MapSlice, |
| 322 | // without going through a byte representation. A nil or empty map[string]interface{} input is |
| 323 | // converted to an empty map, i.e. yaml.MapSlice(nil). |
| 324 | // |
| 325 | // interface{} slices stay interface{} slices. map[string]interface{} becomes yaml.MapSlice. |
| 326 | // |
| 327 | // int64 and float64 are down casted following the logic of github.com/go-yaml/yaml: |
| 328 | // - float64s are down-casted as far as possible without data-loss to int, int64, uint64. |
| 329 | // - int64s are down-casted to int if possible without data-loss. |
| 330 | // |
| 331 | // Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. |
| 332 | // |
| 333 | // string, bool and any other types are unchanged. |
| 334 | func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { |
| 335 | if len(j) == 0 { |
| 336 | return nil |
| 337 | } |
| 338 | ret := make(yaml.MapSlice, 0, len(j)) |
| 339 | for k, v := range j { |
| 340 | ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) |
| 341 | } |
| 342 | return ret |
| 343 | } |
| 344 | |
| 345 | func jsonToYAMLValue(j interface{}) interface{} { |
| 346 | switch j := j.(type) { |
| 347 | case map[string]interface{}: |
| 348 | if j == nil { |
| 349 | return interface{}(nil) |
| 350 | } |
| 351 | return JSONObjectToYAMLObject(j) |
| 352 | case []interface{}: |
| 353 | if j == nil { |
| 354 | return interface{}(nil) |
| 355 | } |
| 356 | ret := make([]interface{}, len(j)) |
| 357 | for i := range j { |
| 358 | ret[i] = jsonToYAMLValue(j[i]) |
| 359 | } |
| 360 | return ret |
| 361 | case float64: |
| 362 | // replicate the logic in https://github.com/go-yaml/yaml/blob/51d6538a90f86fe93ac480b35f37b2be17fef232/resolve.go#L151 |
| 363 | if i64 := int64(j); j == float64(i64) { |
| 364 | if i := int(i64); i64 == int64(i) { |
| 365 | return i |
| 366 | } |
| 367 | return i64 |
| 368 | } |
| 369 | if ui64 := uint64(j); j == float64(ui64) { |
| 370 | return ui64 |
| 371 | } |
| 372 | return j |
| 373 | case int64: |
| 374 | if i := int(j); j == int64(i) { |
| 375 | return i |
| 376 | } |
| 377 | return j |
| 378 | } |
| 379 | return j |
| 380 | } |