blob: 7e69ee1b22edbb492b105fd6f6338e640ddafcf2 [file] [log] [blame]
Matteo Scandoloa6a3aee2019-11-26 13:30:14 -07001// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
2
3// Copyright 2016 The Go Authors. All rights reserved.
4// Use of this source code is governed by a BSD-style
5// license that can be found in the LICENSE file.
6
David K. Bainbridgec415efe2021-08-19 13:05:21 +00007//go:build go1.10
Matteo Scandoloa6a3aee2019-11-26 13:30:14 -07008// +build go1.10
9
10// Package idna implements IDNA2008 using the compatibility processing
11// defined by UTS (Unicode Technical Standard) #46, which defines a standard to
12// deal with the transition from IDNA2003.
13//
14// IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC
15// 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894.
16// UTS #46 is defined in https://www.unicode.org/reports/tr46.
17// See https://unicode.org/cldr/utility/idna.jsp for a visualization of the
18// differences between these two standards.
19package idna // import "golang.org/x/net/idna"
20
21import (
22 "fmt"
23 "strings"
24 "unicode/utf8"
25
26 "golang.org/x/text/secure/bidirule"
27 "golang.org/x/text/unicode/bidi"
28 "golang.org/x/text/unicode/norm"
29)
30
31// NOTE: Unlike common practice in Go APIs, the functions will return a
32// sanitized domain name in case of errors. Browsers sometimes use a partially
33// evaluated string as lookup.
34// TODO: the current error handling is, in my opinion, the least opinionated.
35// Other strategies are also viable, though:
36// Option 1) Return an empty string in case of error, but allow the user to
37// specify explicitly which errors to ignore.
38// Option 2) Return the partially evaluated string if it is itself a valid
39// string, otherwise return the empty string in case of error.
40// Option 3) Option 1 and 2.
41// Option 4) Always return an empty string for now and implement Option 1 as
42// needed, and document that the return string may not be empty in case of
43// error in the future.
44// I think Option 1 is best, but it is quite opinionated.
45
46// ToASCII is a wrapper for Punycode.ToASCII.
47func ToASCII(s string) (string, error) {
48 return Punycode.process(s, true)
49}
50
51// ToUnicode is a wrapper for Punycode.ToUnicode.
52func ToUnicode(s string) (string, error) {
53 return Punycode.process(s, false)
54}
55
56// An Option configures a Profile at creation time.
57type Option func(*options)
58
59// Transitional sets a Profile to use the Transitional mapping as defined in UTS
60// #46. This will cause, for example, "ß" to be mapped to "ss". Using the
61// transitional mapping provides a compromise between IDNA2003 and IDNA2008
62// compatibility. It is used by most browsers when resolving domain names. This
63// option is only meaningful if combined with MapForLookup.
64func Transitional(transitional bool) Option {
65 return func(o *options) { o.transitional = true }
66}
67
68// VerifyDNSLength sets whether a Profile should fail if any of the IDN parts
69// are longer than allowed by the RFC.
70func VerifyDNSLength(verify bool) Option {
71 return func(o *options) { o.verifyDNSLength = verify }
72}
73
74// RemoveLeadingDots removes leading label separators. Leading runes that map to
75// dots, such as U+3002 IDEOGRAPHIC FULL STOP, are removed as well.
76//
77// This is the behavior suggested by the UTS #46 and is adopted by some
78// browsers.
79func RemoveLeadingDots(remove bool) Option {
80 return func(o *options) { o.removeLeadingDots = remove }
81}
82
83// ValidateLabels sets whether to check the mandatory label validation criteria
84// as defined in Section 5.4 of RFC 5891. This includes testing for correct use
85// of hyphens ('-'), normalization, validity of runes, and the context rules.
86func ValidateLabels(enable bool) Option {
87 return func(o *options) {
88 // Don't override existing mappings, but set one that at least checks
89 // normalization if it is not set.
90 if o.mapping == nil && enable {
91 o.mapping = normalize
92 }
93 o.trie = trie
94 o.validateLabels = enable
95 o.fromPuny = validateFromPunycode
96 }
97}
98
99// StrictDomainName limits the set of permissible ASCII characters to those
100// allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the
101// hyphen). This is set by default for MapForLookup and ValidateForRegistration.
102//
103// This option is useful, for instance, for browsers that allow characters
104// outside this range, for example a '_' (U+005F LOW LINE). See
105// http://www.rfc-editor.org/std/std3.txt for more details This option
106// corresponds to the UseSTD3ASCIIRules option in UTS #46.
107func StrictDomainName(use bool) Option {
108 return func(o *options) {
109 o.trie = trie
110 o.useSTD3Rules = use
111 o.fromPuny = validateFromPunycode
112 }
113}
114
115// NOTE: the following options pull in tables. The tables should not be linked
116// in as long as the options are not used.
117
118// BidiRule enables the Bidi rule as defined in RFC 5893. Any application
119// that relies on proper validation of labels should include this rule.
120func BidiRule() Option {
121 return func(o *options) { o.bidirule = bidirule.ValidString }
122}
123
124// ValidateForRegistration sets validation options to verify that a given IDN is
125// properly formatted for registration as defined by Section 4 of RFC 5891.
126func ValidateForRegistration() Option {
127 return func(o *options) {
128 o.mapping = validateRegistration
129 StrictDomainName(true)(o)
130 ValidateLabels(true)(o)
131 VerifyDNSLength(true)(o)
132 BidiRule()(o)
133 }
134}
135
136// MapForLookup sets validation and mapping options such that a given IDN is
137// transformed for domain name lookup according to the requirements set out in
138// Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894,
139// RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option
140// to add this check.
141//
142// The mappings include normalization and mapping case, width and other
143// compatibility mappings.
144func MapForLookup() Option {
145 return func(o *options) {
146 o.mapping = validateAndMap
147 StrictDomainName(true)(o)
148 ValidateLabels(true)(o)
149 }
150}
151
152type options struct {
153 transitional bool
154 useSTD3Rules bool
155 validateLabels bool
156 verifyDNSLength bool
157 removeLeadingDots bool
158
159 trie *idnaTrie
160
161 // fromPuny calls validation rules when converting A-labels to U-labels.
162 fromPuny func(p *Profile, s string) error
163
164 // mapping implements a validation and mapping step as defined in RFC 5895
165 // or UTS 46, tailored to, for example, domain registration or lookup.
166 mapping func(p *Profile, s string) (mapped string, isBidi bool, err error)
167
168 // bidirule, if specified, checks whether s conforms to the Bidi Rule
169 // defined in RFC 5893.
170 bidirule func(s string) bool
171}
172
173// A Profile defines the configuration of an IDNA mapper.
174type Profile struct {
175 options
176}
177
178func apply(o *options, opts []Option) {
179 for _, f := range opts {
180 f(o)
181 }
182}
183
184// New creates a new Profile.
185//
186// With no options, the returned Profile is the most permissive and equals the
187// Punycode Profile. Options can be passed to further restrict the Profile. The
188// MapForLookup and ValidateForRegistration options set a collection of options,
189// for lookup and registration purposes respectively, which can be tailored by
190// adding more fine-grained options, where later options override earlier
191// options.
192func New(o ...Option) *Profile {
193 p := &Profile{}
194 apply(&p.options, o)
195 return p
196}
197
198// ToASCII converts a domain or domain label to its ASCII form. For example,
199// ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and
200// ToASCII("golang") is "golang". If an error is encountered it will return
201// an error and a (partially) processed result.
202func (p *Profile) ToASCII(s string) (string, error) {
203 return p.process(s, true)
204}
205
206// ToUnicode converts a domain or domain label to its Unicode form. For example,
207// ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and
208// ToUnicode("golang") is "golang". If an error is encountered it will return
209// an error and a (partially) processed result.
210func (p *Profile) ToUnicode(s string) (string, error) {
211 pp := *p
212 pp.transitional = false
213 return pp.process(s, false)
214}
215
216// String reports a string with a description of the profile for debugging
217// purposes. The string format may change with different versions.
218func (p *Profile) String() string {
219 s := ""
220 if p.transitional {
221 s = "Transitional"
222 } else {
223 s = "NonTransitional"
224 }
225 if p.useSTD3Rules {
226 s += ":UseSTD3Rules"
227 }
228 if p.validateLabels {
229 s += ":ValidateLabels"
230 }
231 if p.verifyDNSLength {
232 s += ":VerifyDNSLength"
233 }
234 return s
235}
236
237var (
238 // Punycode is a Profile that does raw punycode processing with a minimum
239 // of validation.
240 Punycode *Profile = punycode
241
242 // Lookup is the recommended profile for looking up domain names, according
243 // to Section 5 of RFC 5891. The exact configuration of this profile may
244 // change over time.
245 Lookup *Profile = lookup
246
247 // Display is the recommended profile for displaying domain names.
248 // The configuration of this profile may change over time.
249 Display *Profile = display
250
251 // Registration is the recommended profile for checking whether a given
252 // IDN is valid for registration, according to Section 4 of RFC 5891.
253 Registration *Profile = registration
254
255 punycode = &Profile{}
256 lookup = &Profile{options{
257 transitional: true,
258 useSTD3Rules: true,
259 validateLabels: true,
260 trie: trie,
261 fromPuny: validateFromPunycode,
262 mapping: validateAndMap,
263 bidirule: bidirule.ValidString,
264 }}
265 display = &Profile{options{
266 useSTD3Rules: true,
267 validateLabels: true,
268 trie: trie,
269 fromPuny: validateFromPunycode,
270 mapping: validateAndMap,
271 bidirule: bidirule.ValidString,
272 }}
273 registration = &Profile{options{
274 useSTD3Rules: true,
275 validateLabels: true,
276 verifyDNSLength: true,
277 trie: trie,
278 fromPuny: validateFromPunycode,
279 mapping: validateRegistration,
280 bidirule: bidirule.ValidString,
281 }}
282
283 // TODO: profiles
284 // Register: recommended for approving domain names: don't do any mappings
285 // but rather reject on invalid input. Bundle or block deviation characters.
286)
287
288type labelError struct{ label, code_ string }
289
290func (e labelError) code() string { return e.code_ }
291func (e labelError) Error() string {
292 return fmt.Sprintf("idna: invalid label %q", e.label)
293}
294
295type runeError rune
296
297func (e runeError) code() string { return "P1" }
298func (e runeError) Error() string {
299 return fmt.Sprintf("idna: disallowed rune %U", e)
300}
301
302// process implements the algorithm described in section 4 of UTS #46,
303// see https://www.unicode.org/reports/tr46.
304func (p *Profile) process(s string, toASCII bool) (string, error) {
305 var err error
306 var isBidi bool
307 if p.mapping != nil {
308 s, isBidi, err = p.mapping(p, s)
309 }
310 // Remove leading empty labels.
311 if p.removeLeadingDots {
312 for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
313 }
314 }
315 // TODO: allow for a quick check of the tables data.
316 // It seems like we should only create this error on ToASCII, but the
317 // UTS 46 conformance tests suggests we should always check this.
318 if err == nil && p.verifyDNSLength && s == "" {
319 err = &labelError{s, "A4"}
320 }
321 labels := labelIter{orig: s}
322 for ; !labels.done(); labels.next() {
323 label := labels.label()
324 if label == "" {
325 // Empty labels are not okay. The label iterator skips the last
326 // label if it is empty.
327 if err == nil && p.verifyDNSLength {
328 err = &labelError{s, "A4"}
329 }
330 continue
331 }
332 if strings.HasPrefix(label, acePrefix) {
333 u, err2 := decode(label[len(acePrefix):])
334 if err2 != nil {
335 if err == nil {
336 err = err2
337 }
338 // Spec says keep the old label.
339 continue
340 }
341 isBidi = isBidi || bidirule.DirectionString(u) != bidi.LeftToRight
342 labels.set(u)
343 if err == nil && p.validateLabels {
344 err = p.fromPuny(p, u)
345 }
346 if err == nil {
347 // This should be called on NonTransitional, according to the
348 // spec, but that currently does not have any effect. Use the
349 // original profile to preserve options.
350 err = p.validateLabel(u)
351 }
352 } else if err == nil {
353 err = p.validateLabel(label)
354 }
355 }
356 if isBidi && p.bidirule != nil && err == nil {
357 for labels.reset(); !labels.done(); labels.next() {
358 if !p.bidirule(labels.label()) {
359 err = &labelError{s, "B"}
360 break
361 }
362 }
363 }
364 if toASCII {
365 for labels.reset(); !labels.done(); labels.next() {
366 label := labels.label()
367 if !ascii(label) {
368 a, err2 := encode(acePrefix, label)
369 if err == nil {
370 err = err2
371 }
372 label = a
373 labels.set(a)
374 }
375 n := len(label)
376 if p.verifyDNSLength && err == nil && (n == 0 || n > 63) {
377 err = &labelError{label, "A4"}
378 }
379 }
380 }
381 s = labels.result()
382 if toASCII && p.verifyDNSLength && err == nil {
383 // Compute the length of the domain name minus the root label and its dot.
384 n := len(s)
385 if n > 0 && s[n-1] == '.' {
386 n--
387 }
388 if len(s) < 1 || n > 253 {
389 err = &labelError{s, "A4"}
390 }
391 }
392 return s, err
393}
394
395func normalize(p *Profile, s string) (mapped string, isBidi bool, err error) {
396 // TODO: consider first doing a quick check to see if any of these checks
397 // need to be done. This will make it slower in the general case, but
398 // faster in the common case.
399 mapped = norm.NFC.String(s)
400 isBidi = bidirule.DirectionString(mapped) == bidi.RightToLeft
401 return mapped, isBidi, nil
402}
403
404func validateRegistration(p *Profile, s string) (idem string, bidi bool, err error) {
405 // TODO: filter need for normalization in loop below.
406 if !norm.NFC.IsNormalString(s) {
407 return s, false, &labelError{s, "V1"}
408 }
409 for i := 0; i < len(s); {
410 v, sz := trie.lookupString(s[i:])
411 if sz == 0 {
412 return s, bidi, runeError(utf8.RuneError)
413 }
414 bidi = bidi || info(v).isBidi(s[i:])
415 // Copy bytes not copied so far.
416 switch p.simplify(info(v).category()) {
417 // TODO: handle the NV8 defined in the Unicode idna data set to allow
418 // for strict conformance to IDNA2008.
419 case valid, deviation:
420 case disallowed, mapped, unknown, ignored:
421 r, _ := utf8.DecodeRuneInString(s[i:])
422 return s, bidi, runeError(r)
423 }
424 i += sz
425 }
426 return s, bidi, nil
427}
428
429func (c info) isBidi(s string) bool {
430 if !c.isMapped() {
431 return c&attributesMask == rtl
432 }
433 // TODO: also store bidi info for mapped data. This is possible, but a bit
434 // cumbersome and not for the common case.
435 p, _ := bidi.LookupString(s)
436 switch p.Class() {
437 case bidi.R, bidi.AL, bidi.AN:
438 return true
439 }
440 return false
441}
442
443func validateAndMap(p *Profile, s string) (vm string, bidi bool, err error) {
444 var (
445 b []byte
446 k int
447 )
448 // combinedInfoBits contains the or-ed bits of all runes. We use this
449 // to derive the mayNeedNorm bit later. This may trigger normalization
450 // overeagerly, but it will not do so in the common case. The end result
451 // is another 10% saving on BenchmarkProfile for the common case.
452 var combinedInfoBits info
453 for i := 0; i < len(s); {
454 v, sz := trie.lookupString(s[i:])
455 if sz == 0 {
456 b = append(b, s[k:i]...)
457 b = append(b, "\ufffd"...)
458 k = len(s)
459 if err == nil {
460 err = runeError(utf8.RuneError)
461 }
462 break
463 }
464 combinedInfoBits |= info(v)
465 bidi = bidi || info(v).isBidi(s[i:])
466 start := i
467 i += sz
468 // Copy bytes not copied so far.
469 switch p.simplify(info(v).category()) {
470 case valid:
471 continue
472 case disallowed:
473 if err == nil {
474 r, _ := utf8.DecodeRuneInString(s[start:])
475 err = runeError(r)
476 }
477 continue
478 case mapped, deviation:
479 b = append(b, s[k:start]...)
480 b = info(v).appendMapping(b, s[start:i])
481 case ignored:
482 b = append(b, s[k:start]...)
483 // drop the rune
484 case unknown:
485 b = append(b, s[k:start]...)
486 b = append(b, "\ufffd"...)
487 }
488 k = i
489 }
490 if k == 0 {
491 // No changes so far.
492 if combinedInfoBits&mayNeedNorm != 0 {
493 s = norm.NFC.String(s)
494 }
495 } else {
496 b = append(b, s[k:]...)
497 if norm.NFC.QuickSpan(b) != len(b) {
498 b = norm.NFC.Bytes(b)
499 }
500 // TODO: the punycode converters require strings as input.
501 s = string(b)
502 }
503 return s, bidi, err
504}
505
506// A labelIter allows iterating over domain name labels.
507type labelIter struct {
508 orig string
509 slice []string
510 curStart int
511 curEnd int
512 i int
513}
514
515func (l *labelIter) reset() {
516 l.curStart = 0
517 l.curEnd = 0
518 l.i = 0
519}
520
521func (l *labelIter) done() bool {
522 return l.curStart >= len(l.orig)
523}
524
525func (l *labelIter) result() string {
526 if l.slice != nil {
527 return strings.Join(l.slice, ".")
528 }
529 return l.orig
530}
531
532func (l *labelIter) label() string {
533 if l.slice != nil {
534 return l.slice[l.i]
535 }
536 p := strings.IndexByte(l.orig[l.curStart:], '.')
537 l.curEnd = l.curStart + p
538 if p == -1 {
539 l.curEnd = len(l.orig)
540 }
541 return l.orig[l.curStart:l.curEnd]
542}
543
544// next sets the value to the next label. It skips the last label if it is empty.
545func (l *labelIter) next() {
546 l.i++
547 if l.slice != nil {
548 if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" {
549 l.curStart = len(l.orig)
550 }
551 } else {
552 l.curStart = l.curEnd + 1
553 if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' {
554 l.curStart = len(l.orig)
555 }
556 }
557}
558
559func (l *labelIter) set(s string) {
560 if l.slice == nil {
561 l.slice = strings.Split(l.orig, ".")
562 }
563 l.slice[l.i] = s
564}
565
566// acePrefix is the ASCII Compatible Encoding prefix.
567const acePrefix = "xn--"
568
569func (p *Profile) simplify(cat category) category {
570 switch cat {
571 case disallowedSTD3Mapped:
572 if p.useSTD3Rules {
573 cat = disallowed
574 } else {
575 cat = mapped
576 }
577 case disallowedSTD3Valid:
578 if p.useSTD3Rules {
579 cat = disallowed
580 } else {
581 cat = valid
582 }
583 case deviation:
584 if !p.transitional {
585 cat = valid
586 }
587 case validNV8, validXV8:
588 // TODO: handle V2008
589 cat = valid
590 }
591 return cat
592}
593
594func validateFromPunycode(p *Profile, s string) error {
595 if !norm.NFC.IsNormalString(s) {
596 return &labelError{s, "V1"}
597 }
598 // TODO: detect whether string may have to be normalized in the following
599 // loop.
600 for i := 0; i < len(s); {
601 v, sz := trie.lookupString(s[i:])
602 if sz == 0 {
603 return runeError(utf8.RuneError)
604 }
605 if c := p.simplify(info(v).category()); c != valid && c != deviation {
606 return &labelError{s, "V6"}
607 }
608 i += sz
609 }
610 return nil
611}
612
613const (
614 zwnj = "\u200c"
615 zwj = "\u200d"
616)
617
618type joinState int8
619
620const (
621 stateStart joinState = iota
622 stateVirama
623 stateBefore
624 stateBeforeVirama
625 stateAfter
626 stateFAIL
627)
628
629var joinStates = [][numJoinTypes]joinState{
630 stateStart: {
631 joiningL: stateBefore,
632 joiningD: stateBefore,
633 joinZWNJ: stateFAIL,
634 joinZWJ: stateFAIL,
635 joinVirama: stateVirama,
636 },
637 stateVirama: {
638 joiningL: stateBefore,
639 joiningD: stateBefore,
640 },
641 stateBefore: {
642 joiningL: stateBefore,
643 joiningD: stateBefore,
644 joiningT: stateBefore,
645 joinZWNJ: stateAfter,
646 joinZWJ: stateFAIL,
647 joinVirama: stateBeforeVirama,
648 },
649 stateBeforeVirama: {
650 joiningL: stateBefore,
651 joiningD: stateBefore,
652 joiningT: stateBefore,
653 },
654 stateAfter: {
655 joiningL: stateFAIL,
656 joiningD: stateBefore,
657 joiningT: stateAfter,
658 joiningR: stateStart,
659 joinZWNJ: stateFAIL,
660 joinZWJ: stateFAIL,
661 joinVirama: stateAfter, // no-op as we can't accept joiners here
662 },
663 stateFAIL: {
664 0: stateFAIL,
665 joiningL: stateFAIL,
666 joiningD: stateFAIL,
667 joiningT: stateFAIL,
668 joiningR: stateFAIL,
669 joinZWNJ: stateFAIL,
670 joinZWJ: stateFAIL,
671 joinVirama: stateFAIL,
672 },
673}
674
675// validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are
676// already implicitly satisfied by the overall implementation.
677func (p *Profile) validateLabel(s string) (err error) {
678 if s == "" {
679 if p.verifyDNSLength {
680 return &labelError{s, "A4"}
681 }
682 return nil
683 }
684 if !p.validateLabels {
685 return nil
686 }
687 trie := p.trie // p.validateLabels is only set if trie is set.
688 if len(s) > 4 && s[2] == '-' && s[3] == '-' {
689 return &labelError{s, "V2"}
690 }
691 if s[0] == '-' || s[len(s)-1] == '-' {
692 return &labelError{s, "V3"}
693 }
694 // TODO: merge the use of this in the trie.
695 v, sz := trie.lookupString(s)
696 x := info(v)
697 if x.isModifier() {
698 return &labelError{s, "V5"}
699 }
700 // Quickly return in the absence of zero-width (non) joiners.
701 if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 {
702 return nil
703 }
704 st := stateStart
705 for i := 0; ; {
706 jt := x.joinType()
707 if s[i:i+sz] == zwj {
708 jt = joinZWJ
709 } else if s[i:i+sz] == zwnj {
710 jt = joinZWNJ
711 }
712 st = joinStates[st][jt]
713 if x.isViramaModifier() {
714 st = joinStates[st][joinVirama]
715 }
716 if i += sz; i == len(s) {
717 break
718 }
719 v, sz = trie.lookupString(s[i:])
720 x = info(v)
721 }
722 if st == stateFAIL || st == stateAfter {
723 return &labelError{s, "C"}
724 }
725 return nil
726}
727
728func ascii(s string) bool {
729 for i := 0; i < len(s); i++ {
730 if s[i] >= utf8.RuneSelf {
731 return false
732 }
733 }
734 return true
735}