blob: 7c7456374c132d7aba19230d96d209d283a99c7d [file] [log] [blame]
Matteo Scandoloa6a3aee2019-11-26 13:30:14 -07001// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
2
3// Copyright 2016 The Go Authors. All rights reserved.
4// Use of this source code is governed by a BSD-style
5// license that can be found in the LICENSE file.
6
David K. Bainbridgec415efe2021-08-19 13:05:21 +00007//go:build !go1.10
Matteo Scandoloa6a3aee2019-11-26 13:30:14 -07008// +build !go1.10
9
10// Package idna implements IDNA2008 using the compatibility processing
11// defined by UTS (Unicode Technical Standard) #46, which defines a standard to
12// deal with the transition from IDNA2003.
13//
14// IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC
15// 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894.
16// UTS #46 is defined in https://www.unicode.org/reports/tr46.
17// See https://unicode.org/cldr/utility/idna.jsp for a visualization of the
18// differences between these two standards.
19package idna // import "golang.org/x/net/idna"
20
21import (
22 "fmt"
23 "strings"
24 "unicode/utf8"
25
26 "golang.org/x/text/secure/bidirule"
27 "golang.org/x/text/unicode/norm"
28)
29
30// NOTE: Unlike common practice in Go APIs, the functions will return a
31// sanitized domain name in case of errors. Browsers sometimes use a partially
32// evaluated string as lookup.
33// TODO: the current error handling is, in my opinion, the least opinionated.
34// Other strategies are also viable, though:
35// Option 1) Return an empty string in case of error, but allow the user to
36// specify explicitly which errors to ignore.
37// Option 2) Return the partially evaluated string if it is itself a valid
38// string, otherwise return the empty string in case of error.
39// Option 3) Option 1 and 2.
40// Option 4) Always return an empty string for now and implement Option 1 as
41// needed, and document that the return string may not be empty in case of
42// error in the future.
43// I think Option 1 is best, but it is quite opinionated.
44
45// ToASCII is a wrapper for Punycode.ToASCII.
46func ToASCII(s string) (string, error) {
47 return Punycode.process(s, true)
48}
49
50// ToUnicode is a wrapper for Punycode.ToUnicode.
51func ToUnicode(s string) (string, error) {
52 return Punycode.process(s, false)
53}
54
55// An Option configures a Profile at creation time.
56type Option func(*options)
57
58// Transitional sets a Profile to use the Transitional mapping as defined in UTS
59// #46. This will cause, for example, "ß" to be mapped to "ss". Using the
60// transitional mapping provides a compromise between IDNA2003 and IDNA2008
61// compatibility. It is used by most browsers when resolving domain names. This
62// option is only meaningful if combined with MapForLookup.
63func Transitional(transitional bool) Option {
64 return func(o *options) { o.transitional = true }
65}
66
67// VerifyDNSLength sets whether a Profile should fail if any of the IDN parts
68// are longer than allowed by the RFC.
69func VerifyDNSLength(verify bool) Option {
70 return func(o *options) { o.verifyDNSLength = verify }
71}
72
73// RemoveLeadingDots removes leading label separators. Leading runes that map to
74// dots, such as U+3002 IDEOGRAPHIC FULL STOP, are removed as well.
75//
76// This is the behavior suggested by the UTS #46 and is adopted by some
77// browsers.
78func RemoveLeadingDots(remove bool) Option {
79 return func(o *options) { o.removeLeadingDots = remove }
80}
81
82// ValidateLabels sets whether to check the mandatory label validation criteria
83// as defined in Section 5.4 of RFC 5891. This includes testing for correct use
84// of hyphens ('-'), normalization, validity of runes, and the context rules.
85func ValidateLabels(enable bool) Option {
86 return func(o *options) {
87 // Don't override existing mappings, but set one that at least checks
88 // normalization if it is not set.
89 if o.mapping == nil && enable {
90 o.mapping = normalize
91 }
92 o.trie = trie
93 o.validateLabels = enable
94 o.fromPuny = validateFromPunycode
95 }
96}
97
98// StrictDomainName limits the set of permissable ASCII characters to those
99// allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the
100// hyphen). This is set by default for MapForLookup and ValidateForRegistration.
101//
102// This option is useful, for instance, for browsers that allow characters
103// outside this range, for example a '_' (U+005F LOW LINE). See
104// http://www.rfc-editor.org/std/std3.txt for more details This option
105// corresponds to the UseSTD3ASCIIRules option in UTS #46.
106func StrictDomainName(use bool) Option {
107 return func(o *options) {
108 o.trie = trie
109 o.useSTD3Rules = use
110 o.fromPuny = validateFromPunycode
111 }
112}
113
114// NOTE: the following options pull in tables. The tables should not be linked
115// in as long as the options are not used.
116
117// BidiRule enables the Bidi rule as defined in RFC 5893. Any application
118// that relies on proper validation of labels should include this rule.
119func BidiRule() Option {
120 return func(o *options) { o.bidirule = bidirule.ValidString }
121}
122
123// ValidateForRegistration sets validation options to verify that a given IDN is
124// properly formatted for registration as defined by Section 4 of RFC 5891.
125func ValidateForRegistration() Option {
126 return func(o *options) {
127 o.mapping = validateRegistration
128 StrictDomainName(true)(o)
129 ValidateLabels(true)(o)
130 VerifyDNSLength(true)(o)
131 BidiRule()(o)
132 }
133}
134
135// MapForLookup sets validation and mapping options such that a given IDN is
136// transformed for domain name lookup according to the requirements set out in
137// Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894,
138// RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option
139// to add this check.
140//
141// The mappings include normalization and mapping case, width and other
142// compatibility mappings.
143func MapForLookup() Option {
144 return func(o *options) {
145 o.mapping = validateAndMap
146 StrictDomainName(true)(o)
147 ValidateLabels(true)(o)
148 RemoveLeadingDots(true)(o)
149 }
150}
151
152type options struct {
153 transitional bool
154 useSTD3Rules bool
155 validateLabels bool
156 verifyDNSLength bool
157 removeLeadingDots bool
158
159 trie *idnaTrie
160
161 // fromPuny calls validation rules when converting A-labels to U-labels.
162 fromPuny func(p *Profile, s string) error
163
164 // mapping implements a validation and mapping step as defined in RFC 5895
165 // or UTS 46, tailored to, for example, domain registration or lookup.
166 mapping func(p *Profile, s string) (string, error)
167
168 // bidirule, if specified, checks whether s conforms to the Bidi Rule
169 // defined in RFC 5893.
170 bidirule func(s string) bool
171}
172
173// A Profile defines the configuration of a IDNA mapper.
174type Profile struct {
175 options
176}
177
178func apply(o *options, opts []Option) {
179 for _, f := range opts {
180 f(o)
181 }
182}
183
184// New creates a new Profile.
185//
186// With no options, the returned Profile is the most permissive and equals the
187// Punycode Profile. Options can be passed to further restrict the Profile. The
188// MapForLookup and ValidateForRegistration options set a collection of options,
189// for lookup and registration purposes respectively, which can be tailored by
190// adding more fine-grained options, where later options override earlier
191// options.
192func New(o ...Option) *Profile {
193 p := &Profile{}
194 apply(&p.options, o)
195 return p
196}
197
198// ToASCII converts a domain or domain label to its ASCII form. For example,
199// ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and
200// ToASCII("golang") is "golang". If an error is encountered it will return
201// an error and a (partially) processed result.
202func (p *Profile) ToASCII(s string) (string, error) {
203 return p.process(s, true)
204}
205
206// ToUnicode converts a domain or domain label to its Unicode form. For example,
207// ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and
208// ToUnicode("golang") is "golang". If an error is encountered it will return
209// an error and a (partially) processed result.
210func (p *Profile) ToUnicode(s string) (string, error) {
211 pp := *p
212 pp.transitional = false
213 return pp.process(s, false)
214}
215
216// String reports a string with a description of the profile for debugging
217// purposes. The string format may change with different versions.
218func (p *Profile) String() string {
219 s := ""
220 if p.transitional {
221 s = "Transitional"
222 } else {
223 s = "NonTransitional"
224 }
225 if p.useSTD3Rules {
226 s += ":UseSTD3Rules"
227 }
228 if p.validateLabels {
229 s += ":ValidateLabels"
230 }
231 if p.verifyDNSLength {
232 s += ":VerifyDNSLength"
233 }
234 return s
235}
236
237var (
238 // Punycode is a Profile that does raw punycode processing with a minimum
239 // of validation.
240 Punycode *Profile = punycode
241
242 // Lookup is the recommended profile for looking up domain names, according
243 // to Section 5 of RFC 5891. The exact configuration of this profile may
244 // change over time.
245 Lookup *Profile = lookup
246
247 // Display is the recommended profile for displaying domain names.
248 // The configuration of this profile may change over time.
249 Display *Profile = display
250
251 // Registration is the recommended profile for checking whether a given
252 // IDN is valid for registration, according to Section 4 of RFC 5891.
253 Registration *Profile = registration
254
255 punycode = &Profile{}
256 lookup = &Profile{options{
257 transitional: true,
258 useSTD3Rules: true,
259 validateLabels: true,
260 removeLeadingDots: true,
261 trie: trie,
262 fromPuny: validateFromPunycode,
263 mapping: validateAndMap,
264 bidirule: bidirule.ValidString,
265 }}
266 display = &Profile{options{
267 useSTD3Rules: true,
268 validateLabels: true,
269 removeLeadingDots: true,
270 trie: trie,
271 fromPuny: validateFromPunycode,
272 mapping: validateAndMap,
273 bidirule: bidirule.ValidString,
274 }}
275 registration = &Profile{options{
276 useSTD3Rules: true,
277 validateLabels: true,
278 verifyDNSLength: true,
279 trie: trie,
280 fromPuny: validateFromPunycode,
281 mapping: validateRegistration,
282 bidirule: bidirule.ValidString,
283 }}
284
285 // TODO: profiles
286 // Register: recommended for approving domain names: don't do any mappings
287 // but rather reject on invalid input. Bundle or block deviation characters.
288)
289
290type labelError struct{ label, code_ string }
291
292func (e labelError) code() string { return e.code_ }
293func (e labelError) Error() string {
294 return fmt.Sprintf("idna: invalid label %q", e.label)
295}
296
297type runeError rune
298
299func (e runeError) code() string { return "P1" }
300func (e runeError) Error() string {
301 return fmt.Sprintf("idna: disallowed rune %U", e)
302}
303
304// process implements the algorithm described in section 4 of UTS #46,
305// see https://www.unicode.org/reports/tr46.
306func (p *Profile) process(s string, toASCII bool) (string, error) {
307 var err error
308 if p.mapping != nil {
309 s, err = p.mapping(p, s)
310 }
311 // Remove leading empty labels.
312 if p.removeLeadingDots {
313 for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
314 }
315 }
316 // It seems like we should only create this error on ToASCII, but the
317 // UTS 46 conformance tests suggests we should always check this.
318 if err == nil && p.verifyDNSLength && s == "" {
319 err = &labelError{s, "A4"}
320 }
321 labels := labelIter{orig: s}
322 for ; !labels.done(); labels.next() {
323 label := labels.label()
324 if label == "" {
325 // Empty labels are not okay. The label iterator skips the last
326 // label if it is empty.
327 if err == nil && p.verifyDNSLength {
328 err = &labelError{s, "A4"}
329 }
330 continue
331 }
332 if strings.HasPrefix(label, acePrefix) {
333 u, err2 := decode(label[len(acePrefix):])
334 if err2 != nil {
335 if err == nil {
336 err = err2
337 }
338 // Spec says keep the old label.
339 continue
340 }
341 labels.set(u)
342 if err == nil && p.validateLabels {
343 err = p.fromPuny(p, u)
344 }
345 if err == nil {
346 // This should be called on NonTransitional, according to the
347 // spec, but that currently does not have any effect. Use the
348 // original profile to preserve options.
349 err = p.validateLabel(u)
350 }
351 } else if err == nil {
352 err = p.validateLabel(label)
353 }
354 }
355 if toASCII {
356 for labels.reset(); !labels.done(); labels.next() {
357 label := labels.label()
358 if !ascii(label) {
359 a, err2 := encode(acePrefix, label)
360 if err == nil {
361 err = err2
362 }
363 label = a
364 labels.set(a)
365 }
366 n := len(label)
367 if p.verifyDNSLength && err == nil && (n == 0 || n > 63) {
368 err = &labelError{label, "A4"}
369 }
370 }
371 }
372 s = labels.result()
373 if toASCII && p.verifyDNSLength && err == nil {
374 // Compute the length of the domain name minus the root label and its dot.
375 n := len(s)
376 if n > 0 && s[n-1] == '.' {
377 n--
378 }
379 if len(s) < 1 || n > 253 {
380 err = &labelError{s, "A4"}
381 }
382 }
383 return s, err
384}
385
386func normalize(p *Profile, s string) (string, error) {
387 return norm.NFC.String(s), nil
388}
389
390func validateRegistration(p *Profile, s string) (string, error) {
391 if !norm.NFC.IsNormalString(s) {
392 return s, &labelError{s, "V1"}
393 }
394 for i := 0; i < len(s); {
395 v, sz := trie.lookupString(s[i:])
396 // Copy bytes not copied so far.
397 switch p.simplify(info(v).category()) {
398 // TODO: handle the NV8 defined in the Unicode idna data set to allow
399 // for strict conformance to IDNA2008.
400 case valid, deviation:
401 case disallowed, mapped, unknown, ignored:
402 r, _ := utf8.DecodeRuneInString(s[i:])
403 return s, runeError(r)
404 }
405 i += sz
406 }
407 return s, nil
408}
409
410func validateAndMap(p *Profile, s string) (string, error) {
411 var (
412 err error
413 b []byte
414 k int
415 )
416 for i := 0; i < len(s); {
417 v, sz := trie.lookupString(s[i:])
418 start := i
419 i += sz
420 // Copy bytes not copied so far.
421 switch p.simplify(info(v).category()) {
422 case valid:
423 continue
424 case disallowed:
425 if err == nil {
426 r, _ := utf8.DecodeRuneInString(s[start:])
427 err = runeError(r)
428 }
429 continue
430 case mapped, deviation:
431 b = append(b, s[k:start]...)
432 b = info(v).appendMapping(b, s[start:i])
433 case ignored:
434 b = append(b, s[k:start]...)
435 // drop the rune
436 case unknown:
437 b = append(b, s[k:start]...)
438 b = append(b, "\ufffd"...)
439 }
440 k = i
441 }
442 if k == 0 {
443 // No changes so far.
444 s = norm.NFC.String(s)
445 } else {
446 b = append(b, s[k:]...)
447 if norm.NFC.QuickSpan(b) != len(b) {
448 b = norm.NFC.Bytes(b)
449 }
450 // TODO: the punycode converters require strings as input.
451 s = string(b)
452 }
453 return s, err
454}
455
456// A labelIter allows iterating over domain name labels.
457type labelIter struct {
458 orig string
459 slice []string
460 curStart int
461 curEnd int
462 i int
463}
464
465func (l *labelIter) reset() {
466 l.curStart = 0
467 l.curEnd = 0
468 l.i = 0
469}
470
471func (l *labelIter) done() bool {
472 return l.curStart >= len(l.orig)
473}
474
475func (l *labelIter) result() string {
476 if l.slice != nil {
477 return strings.Join(l.slice, ".")
478 }
479 return l.orig
480}
481
482func (l *labelIter) label() string {
483 if l.slice != nil {
484 return l.slice[l.i]
485 }
486 p := strings.IndexByte(l.orig[l.curStart:], '.')
487 l.curEnd = l.curStart + p
488 if p == -1 {
489 l.curEnd = len(l.orig)
490 }
491 return l.orig[l.curStart:l.curEnd]
492}
493
494// next sets the value to the next label. It skips the last label if it is empty.
495func (l *labelIter) next() {
496 l.i++
497 if l.slice != nil {
498 if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" {
499 l.curStart = len(l.orig)
500 }
501 } else {
502 l.curStart = l.curEnd + 1
503 if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' {
504 l.curStart = len(l.orig)
505 }
506 }
507}
508
509func (l *labelIter) set(s string) {
510 if l.slice == nil {
511 l.slice = strings.Split(l.orig, ".")
512 }
513 l.slice[l.i] = s
514}
515
516// acePrefix is the ASCII Compatible Encoding prefix.
517const acePrefix = "xn--"
518
519func (p *Profile) simplify(cat category) category {
520 switch cat {
521 case disallowedSTD3Mapped:
522 if p.useSTD3Rules {
523 cat = disallowed
524 } else {
525 cat = mapped
526 }
527 case disallowedSTD3Valid:
528 if p.useSTD3Rules {
529 cat = disallowed
530 } else {
531 cat = valid
532 }
533 case deviation:
534 if !p.transitional {
535 cat = valid
536 }
537 case validNV8, validXV8:
538 // TODO: handle V2008
539 cat = valid
540 }
541 return cat
542}
543
544func validateFromPunycode(p *Profile, s string) error {
545 if !norm.NFC.IsNormalString(s) {
546 return &labelError{s, "V1"}
547 }
548 for i := 0; i < len(s); {
549 v, sz := trie.lookupString(s[i:])
550 if c := p.simplify(info(v).category()); c != valid && c != deviation {
551 return &labelError{s, "V6"}
552 }
553 i += sz
554 }
555 return nil
556}
557
558const (
559 zwnj = "\u200c"
560 zwj = "\u200d"
561)
562
563type joinState int8
564
565const (
566 stateStart joinState = iota
567 stateVirama
568 stateBefore
569 stateBeforeVirama
570 stateAfter
571 stateFAIL
572)
573
574var joinStates = [][numJoinTypes]joinState{
575 stateStart: {
576 joiningL: stateBefore,
577 joiningD: stateBefore,
578 joinZWNJ: stateFAIL,
579 joinZWJ: stateFAIL,
580 joinVirama: stateVirama,
581 },
582 stateVirama: {
583 joiningL: stateBefore,
584 joiningD: stateBefore,
585 },
586 stateBefore: {
587 joiningL: stateBefore,
588 joiningD: stateBefore,
589 joiningT: stateBefore,
590 joinZWNJ: stateAfter,
591 joinZWJ: stateFAIL,
592 joinVirama: stateBeforeVirama,
593 },
594 stateBeforeVirama: {
595 joiningL: stateBefore,
596 joiningD: stateBefore,
597 joiningT: stateBefore,
598 },
599 stateAfter: {
600 joiningL: stateFAIL,
601 joiningD: stateBefore,
602 joiningT: stateAfter,
603 joiningR: stateStart,
604 joinZWNJ: stateFAIL,
605 joinZWJ: stateFAIL,
606 joinVirama: stateAfter, // no-op as we can't accept joiners here
607 },
608 stateFAIL: {
609 0: stateFAIL,
610 joiningL: stateFAIL,
611 joiningD: stateFAIL,
612 joiningT: stateFAIL,
613 joiningR: stateFAIL,
614 joinZWNJ: stateFAIL,
615 joinZWJ: stateFAIL,
616 joinVirama: stateFAIL,
617 },
618}
619
620// validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are
621// already implicitly satisfied by the overall implementation.
622func (p *Profile) validateLabel(s string) error {
623 if s == "" {
624 if p.verifyDNSLength {
625 return &labelError{s, "A4"}
626 }
627 return nil
628 }
629 if p.bidirule != nil && !p.bidirule(s) {
630 return &labelError{s, "B"}
631 }
632 if !p.validateLabels {
633 return nil
634 }
635 trie := p.trie // p.validateLabels is only set if trie is set.
636 if len(s) > 4 && s[2] == '-' && s[3] == '-' {
637 return &labelError{s, "V2"}
638 }
639 if s[0] == '-' || s[len(s)-1] == '-' {
640 return &labelError{s, "V3"}
641 }
642 // TODO: merge the use of this in the trie.
643 v, sz := trie.lookupString(s)
644 x := info(v)
645 if x.isModifier() {
646 return &labelError{s, "V5"}
647 }
648 // Quickly return in the absence of zero-width (non) joiners.
649 if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 {
650 return nil
651 }
652 st := stateStart
653 for i := 0; ; {
654 jt := x.joinType()
655 if s[i:i+sz] == zwj {
656 jt = joinZWJ
657 } else if s[i:i+sz] == zwnj {
658 jt = joinZWNJ
659 }
660 st = joinStates[st][jt]
661 if x.isViramaModifier() {
662 st = joinStates[st][joinVirama]
663 }
664 if i += sz; i == len(s) {
665 break
666 }
667 v, sz = trie.lookupString(s[i:])
668 x = info(v)
669 }
670 if st == stateFAIL || st == stateAfter {
671 return &labelError{s, "C"}
672 }
673 return nil
674}
675
676func ascii(s string) bool {
677 for i := 0; i < len(s); i++ {
678 if s[i] >= utf8.RuneSelf {
679 return false
680 }
681 }
682 return true
683}