vendor/golang.org/x/text/collate/maketables.go - voltha-go - Gitiles

 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // +build ignore

 // Collation table generator.
 // Data read from the web.

 package main

 import (
 	"archive/zip"
 	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"unicode/utf8"

 	"golang.org/x/text/collate"
 	"golang.org/x/text/collate/build"
 	"golang.org/x/text/internal/colltab"
 	"golang.org/x/text/internal/gen"
 	"golang.org/x/text/language"
 	"golang.org/x/text/unicode/cldr"
 )

 var (
 	test = flag.Bool("test", false,
 		"test existing tables; can be used to compare web data with package data.")
 	short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
 	draft = flag.Bool("draft", false, `Use draft versions, when available.`)
 	tags  = flag.String("tags", "", "build tags to be included after +build directive")
 	pkg   = flag.String("package", "collate",
 		"the name of the package in which the generated file is to be included")

 	tables = flagStringSetAllowAll("tables", "collate", "collate,chars",
 		"comma-spearated list of tables to generate.")
 	exclude = flagStringSet("exclude", "zh2", "",
 		"comma-separated list of languages to exclude.")
 	include = flagStringSet("include", "", "",
 		"comma-separated list of languages to include. Include trumps exclude.")
 	// TODO: Not included: unihan gb2312han zhuyin big5han (for size reasons)
 	// TODO: Not included: traditional (buggy for Bengali)
 	types = flagStringSetAllowAll("types", "standard,phonebook,phonetic,reformed,pinyin,stroke", "",
 		"comma-separated list of types that should be included.")
 )

 // stringSet implements an ordered set based on a list.  It implements flag.Value
 // to allow a set to be specified as a comma-separated list.
 type stringSet struct {
 	s        []string
 	allowed  *stringSet
 	dirty    bool // needs compaction if true
 	all      bool
 	allowAll bool
 }

 func flagStringSet(name, def, allowed, usage string) *stringSet {
 	ss := &stringSet{}
 	if allowed != "" {
 		usage += fmt.Sprintf(" (allowed values: any of %s)", allowed)
 		ss.allowed = &stringSet{}
 		failOnError(ss.allowed.Set(allowed))
 	}
 	ss.Set(def)
 	flag.Var(ss, name, usage)
 	return ss
 }

 func flagStringSetAllowAll(name, def, allowed, usage string) *stringSet {
 	ss := &stringSet{allowAll: true}
 	if allowed == "" {
 		flag.Var(ss, name, usage+fmt.Sprintf(` Use "all" to select all.`))
 	} else {
 		ss.allowed = &stringSet{}
 		failOnError(ss.allowed.Set(allowed))
 		flag.Var(ss, name, usage+fmt.Sprintf(` (allowed values: "all" or any of %s)`, allowed))
 	}
 	ss.Set(def)
 	return ss
 }

 func (ss stringSet) Len() int {
 	return len(ss.s)
 }

 func (ss stringSet) String() string {
 	return strings.Join(ss.s, ",")
 }

 func (ss *stringSet) Set(s string) error {
 	if ss.allowAll && s == "all" {
 		ss.s = nil
 		ss.all = true
 		return nil
 	}
 	ss.s = ss.s[:0]
 	for _, s := range strings.Split(s, ",") {
 		if s := strings.TrimSpace(s); s != "" {
 			if ss.allowed != nil && !ss.allowed.contains(s) {
 				return fmt.Errorf("unsupported value %q; must be one of %s", s, ss.allowed)
 			}
 			ss.add(s)
 		}
 	}
 	ss.compact()
 	return nil
 }

 func (ss *stringSet) add(s string) {
 	ss.s = append(ss.s, s)
 	ss.dirty = true
 }

 func (ss *stringSet) values() []string {
 	ss.compact()
 	return ss.s
 }

 func (ss *stringSet) contains(s string) bool {
 	if ss.all {
 		return true
 	}
 	for _, v := range ss.s {
 		if v == s {
 			return true
 		}
 	}
 	return false
 }

 func (ss *stringSet) compact() {
 	if !ss.dirty {
 		return
 	}
 	a := ss.s
 	sort.Strings(a)
 	k := 0
 	for i := 1; i < len(a); i++ {
 		if a[k] != a[i] {
 			a[k+1] = a[i]
 			k++
 		}
 	}
 	ss.s = a[:k+1]
 	ss.dirty = false
 }

 func skipLang(l string) bool {
 	if include.Len() > 0 {
 		return !include.contains(l)
 	}
 	return exclude.contains(l)
 }

 // altInclude returns a list of alternatives (for the LDML alt attribute)
 // in order of preference.  An empty string in this list indicates the
 // default entry.
 func altInclude() []string {
 	l := []string{}
 	if *short {
 		l = append(l, "short")
 	}
 	l = append(l, "")
 	// TODO: handle draft using cldr.SetDraftLevel
 	if *draft {
 		l = append(l, "proposed")
 	}
 	return l
 }

 func failOnError(e error) {
 	if e != nil {
 		log.Panic(e)
 	}
 }

 func openArchive() *zip.Reader {
 	f := gen.OpenCLDRCoreZip()
 	buffer, err := ioutil.ReadAll(f)
 	f.Close()
 	failOnError(err)
 	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
 	failOnError(err)
 	return archive
 }

 // parseUCA parses a Default Unicode Collation Element Table of the format
 // specified in http://www.unicode.org/reports/tr10/#File_Format.
 // It returns the variable top.
 func parseUCA(builder *build.Builder) {
 	var r io.ReadCloser
 	var err error
 	for _, f := range openArchive().File {
 		if strings.HasSuffix(f.Name, "allkeys_CLDR.txt") {
 			r, err = f.Open()
 		}
 	}
 	if r == nil {
 		log.Fatal("File allkeys_CLDR.txt not found in archive.")
 	}
 	failOnError(err)
 	defer r.Close()
 	scanner := bufio.NewScanner(r)
 	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
 	for i := 1; scanner.Scan(); i++ {
 		line := scanner.Text()
 		if len(line) == 0 || line[0] == '#' {
 			continue
 		}
 		if line[0] == '@' {
 			// parse properties
 			switch {
 			case strings.HasPrefix(line[1:], "version "):
 				a := strings.Split(line[1:], " ")
 				if a[1] != gen.UnicodeVersion() {
 					log.Fatalf("incompatible version %s; want %s", a[1], gen.UnicodeVersion())
 				}
 			case strings.HasPrefix(line[1:], "backwards "):
 				log.Fatalf("%d: unsupported option backwards", i)
 			default:
 				log.Printf("%d: unknown option %s", i, line[1:])
 			}
 		} else {
 			// parse entries
 			part := strings.Split(line, " ; ")
 			if len(part) != 2 {
 				log.Fatalf("%d: production rule without ';': %v", i, line)
 			}
 			lhs := []rune{}
 			for _, v := range strings.Split(part[0], " ") {
 				if v == "" {
 					continue
 				}
 				lhs = append(lhs, rune(convHex(i, v)))
 			}
 			var n int
 			var vars []int
 			rhs := [][]int{}
 			for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
 				n += len(m[0])
 				elem := []int{}
 				for _, h := range strings.Split(m[2], ".") {
 					elem = append(elem, convHex(i, h))
 				}
 				if m[1] == "*" {
 					vars = append(vars, i)
 				}
 				rhs = append(rhs, elem)
 			}
 			if len(part[1]) < n+3 || part[1][n+1] != '#' {
 				log.Fatalf("%d: expected comment; found %s", i, part[1][n:])
 			}
 			if *test {
 				testInput.add(string(lhs))
 			}
 			failOnError(builder.Add(lhs, rhs, vars))
 		}
 	}
 	if scanner.Err() != nil {
 		log.Fatal(scanner.Err())
 	}
 }

 func convHex(line int, s string) int {
 	r, e := strconv.ParseInt(s, 16, 32)
 	if e != nil {
 		log.Fatalf("%d: %v", line, e)
 	}
 	return int(r)
 }

 var testInput = stringSet{}

 var charRe = regexp.MustCompile(`&#x([0-9A-F]*);`)
 var tagRe = regexp.MustCompile(`<([a-z_]*)  */>`)

 var mainLocales = []string{}

 // charsets holds a list of exemplar characters per category.
 type charSets map[string][]string

 func (p charSets) fprint(w io.Writer) {
 	fmt.Fprintln(w, "[exN]string{")
 	for i, k := range []string{"", "contractions", "punctuation", "auxiliary", "currencySymbol", "index"} {
 		if set := p[k]; len(set) != 0 {
 			fmt.Fprintf(w, "\t\t%d: %q,\n", i, strings.Join(set, " "))
 		}
 	}
 	fmt.Fprintln(w, "\t},")
 }

 var localeChars = make(map[string]charSets)

 const exemplarHeader = `
 type exemplarType int
 const (
 	exCharacters exemplarType = iota
 	exContractions
 	exPunctuation
 	exAuxiliary
 	exCurrency
 	exIndex
 	exN
 )
 `

 func printExemplarCharacters(w io.Writer) {
 	fmt.Fprintln(w, exemplarHeader)
 	fmt.Fprintln(w, "var exemplarCharacters = map[string][exN]string{")
 	for _, loc := range mainLocales {
 		fmt.Fprintf(w, "\t%q: ", loc)
 		localeChars[loc].fprint(w)
 	}
 	fmt.Fprintln(w, "}")
 }

 func decodeCLDR(d *cldr.Decoder) *cldr.CLDR {
 	r := gen.OpenCLDRCoreZip()
 	data, err := d.DecodeZip(r)
 	failOnError(err)
 	return data
 }

 // parseMain parses XML files in the main directory of the CLDR core.zip file.
 func parseMain() {
 	d := &cldr.Decoder{}
 	d.SetDirFilter("main")
 	d.SetSectionFilter("characters")
 	data := decodeCLDR(d)
 	for _, loc := range data.Locales() {
 		x := data.RawLDML(loc)
 		if skipLang(x.Identity.Language.Type) {
 			continue
 		}
 		if x.Characters != nil {
 			x, _ = data.LDML(loc)
 			loc = language.Make(loc).String()
 			for _, ec := range x.Characters.ExemplarCharacters {
 				if ec.Draft != "" {
 					continue
 				}
 				if _, ok := localeChars[loc]; !ok {
 					mainLocales = append(mainLocales, loc)
 					localeChars[loc] = make(charSets)
 				}
 				localeChars[loc][ec.Type] = parseCharacters(ec.Data())
 			}
 		}
 	}
 }

 func parseCharacters(chars string) []string {
 	parseSingle := func(s string) (r rune, tail string, escaped bool) {
 		if s[0] == '\\' {
 			return rune(s[1]), s[2:], true
 		}
 		r, sz := utf8.DecodeRuneInString(s)
 		return r, s[sz:], false
 	}
 	chars = strings.TrimSpace(chars)
 	if n := len(chars) - 1; chars[n] == ']' && chars[0] == '[' {
 		chars = chars[1:n]
 	}
 	list := []string{}
 	var r, last, end rune
 	for len(chars) > 0 {
 		if chars[0] == '{' { // character sequence
 			buf := []rune{}
 			for chars = chars[1:]; len(chars) > 0; {
 				r, chars, _ = parseSingle(chars)
 				if r == '}' {
 					break
 				}
 				if r == ' ' {
 					log.Fatalf("space not supported in sequence %q", chars)
 				}
 				buf = append(buf, r)
 			}
 			list = append(list, string(buf))
 			last = 0
 		} else { // single character
 			escaped := false
 			r, chars, escaped = parseSingle(chars)
 			if r != ' ' {
 				if r == '-' && !escaped {
 					if last == 0 {
 						log.Fatal("'-' should be preceded by a character")
 					}
 					end, chars, _ = parseSingle(chars)
 					for ; last <= end; last++ {
 						list = append(list, string(last))
 					}
 					last = 0
 				} else {
 					list = append(list, string(r))
 					last = r
 				}
 			}
 		}
 	}
 	return list
 }

 var fileRe = regexp.MustCompile(`.*/collation/(.*)\.xml`)

 // typeMap translates legacy type keys to their BCP47 equivalent.
 var typeMap = map[string]string{
 	"phonebook":   "phonebk",
 	"traditional": "trad",
 }

 // parseCollation parses XML files in the collation directory of the CLDR core.zip file.
 func parseCollation(b *build.Builder) {
 	d := &cldr.Decoder{}
 	d.SetDirFilter("collation")
 	data := decodeCLDR(d)
 	for _, loc := range data.Locales() {
 		x, err := data.LDML(loc)
 		failOnError(err)
 		if skipLang(x.Identity.Language.Type) {
 			continue
 		}
 		cs := x.Collations.Collation
 		sl := cldr.MakeSlice(&cs)
 		if len(types.s) == 0 {
 			sl.SelectAnyOf("type", x.Collations.Default())
 		} else if !types.all {
 			sl.SelectAnyOf("type", types.s...)
 		}
 		sl.SelectOnePerGroup("alt", altInclude())

 		for _, c := range cs {
 			id, err := language.Parse(loc)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "invalid locale: %q", err)
 				continue
 			}
 			// Support both old- and new-style defaults.
 			d := c.Type
 			if x.Collations.DefaultCollation == nil {
 				d = x.Collations.Default()
 			} else {
 				d = x.Collations.DefaultCollation.Data()
 			}
 			// We assume tables are being built either for search or collation,
 			// but not both. For search the default is always "search".
 			if d != c.Type && c.Type != "search" {
 				typ := c.Type
 				if len(c.Type) > 8 {
 					typ = typeMap[c.Type]
 				}
 				id, err = id.SetTypeForKey("co", typ)
 				failOnError(err)
 			}
 			t := b.Tailoring(id)
 			c.Process(processor{t})
 		}
 	}
 }

 type processor struct {
 	t *build.Tailoring
 }

 func (p processor) Reset(anchor string, before int) (err error) {
 	if before != 0 {
 		err = p.t.SetAnchorBefore(anchor)
 	} else {
 		err = p.t.SetAnchor(anchor)
 	}
 	failOnError(err)
 	return nil
 }

 func (p processor) Insert(level int, str, context, extend string) error {
 	str = context + str
 	if *test {
 		testInput.add(str)
 	}
 	// TODO: mimic bug in old maketables: remove.
 	err := p.t.Insert(colltab.Level(level-1), str, context+extend)
 	failOnError(err)
 	return nil
 }

 func (p processor) Index(id string) {
 }

 func testCollator(c *collate.Collator) {
 	c0 := collate.New(language.Und)

 	// iterator over all characters for all locales and check
 	// whether Key is equal.
 	buf := collate.Buffer{}

 	// Add all common and not too uncommon runes to the test set.
 	for i := rune(0); i < 0x30000; i++ {
 		testInput.add(string(i))
 	}
 	for i := rune(0xE0000); i < 0xF0000; i++ {
 		testInput.add(string(i))
 	}
 	for _, str := range testInput.values() {
 		k0 := c0.KeyFromString(&buf, str)
 		k := c.KeyFromString(&buf, str)
 		if !bytes.Equal(k0, k) {
 			failOnError(fmt.Errorf("test:%U: keys differ (%x vs %x)", []rune(str), k0, k))
 		}
 		buf.Reset()
 	}
 	fmt.Println("PASS")
 }

 func main() {
 	gen.Init()
 	b := build.NewBuilder()
 	parseUCA(b)
 	if tables.contains("chars") {
 		parseMain()
 	}
 	parseCollation(b)

 	c, err := b.Build()
 	failOnError(err)

 	if *test {
 		testCollator(collate.NewFromTable(c))
 	} else {
 		w := &bytes.Buffer{}

 		gen.WriteUnicodeVersion(w)
 		gen.WriteCLDRVersion(w)

 		if tables.contains("collate") {
 			_, err = b.Print(w)
 			failOnError(err)
 		}
 		if tables.contains("chars") {
 			printExemplarCharacters(w)
 		}
 		gen.WriteGoFile("tables.go", *pkg, w.Bytes())
 	}
 }
	// Copyright 2012 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// +build ignore

	// Collation table generator.
	// Data read from the web.

	package main

	import (
	"archive/zip"
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode/utf8"

	"golang.org/x/text/collate"
	"golang.org/x/text/collate/build"
	"golang.org/x/text/internal/colltab"
	"golang.org/x/text/internal/gen"
	"golang.org/x/text/language"
	"golang.org/x/text/unicode/cldr"
	)

	var (
	test = flag.Bool("test", false,
	"test existing tables; can be used to compare web data with package data.")
	short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
	draft = flag.Bool("draft", false, `Use draft versions, when available.`)
	tags = flag.String("tags", "", "build tags to be included after +build directive")
	pkg = flag.String("package", "collate",
	"the name of the package in which the generated file is to be included")

	tables = flagStringSetAllowAll("tables", "collate", "collate,chars",
	"comma-spearated list of tables to generate.")
	exclude = flagStringSet("exclude", "zh2", "",
	"comma-separated list of languages to exclude.")
	include = flagStringSet("include", "", "",
	"comma-separated list of languages to include. Include trumps exclude.")
	// TODO: Not included: unihan gb2312han zhuyin big5han (for size reasons)
	// TODO: Not included: traditional (buggy for Bengali)
	types = flagStringSetAllowAll("types", "standard,phonebook,phonetic,reformed,pinyin,stroke", "",
	"comma-separated list of types that should be included.")
	)

	// stringSet implements an ordered set based on a list. It implements flag.Value
	// to allow a set to be specified as a comma-separated list.
	type stringSet struct {
	s []string
	allowed *stringSet
	dirty bool // needs compaction if true
	all bool
	allowAll bool
	}

	func flagStringSet(name, def, allowed, usage string) *stringSet {
	ss := &stringSet{}
	if allowed != "" {
	usage += fmt.Sprintf(" (allowed values: any of %s)", allowed)
	ss.allowed = &stringSet{}
	failOnError(ss.allowed.Set(allowed))
	}
	ss.Set(def)
	flag.Var(ss, name, usage)
	return ss
	}

	func flagStringSetAllowAll(name, def, allowed, usage string) *stringSet {
	ss := &stringSet{allowAll: true}
	if allowed == "" {
	flag.Var(ss, name, usage+fmt.Sprintf(` Use "all" to select all.`))
	} else {
	ss.allowed = &stringSet{}
	failOnError(ss.allowed.Set(allowed))
	flag.Var(ss, name, usage+fmt.Sprintf(` (allowed values: "all" or any of %s)`, allowed))
	}
	ss.Set(def)
	return ss
	}

	func (ss stringSet) Len() int {
	return len(ss.s)
	}

	func (ss stringSet) String() string {
	return strings.Join(ss.s, ",")
	}

	func (ss *stringSet) Set(s string) error {
	if ss.allowAll && s == "all" {
	ss.s = nil
	ss.all = true
	return nil
	}
	ss.s = ss.s[:0]
	for _, s := range strings.Split(s, ",") {
	if s := strings.TrimSpace(s); s != "" {
	if ss.allowed != nil && !ss.allowed.contains(s) {
	return fmt.Errorf("unsupported value %q; must be one of %s", s, ss.allowed)
	}
	ss.add(s)
	}
	}
	ss.compact()
	return nil
	}

	func (ss *stringSet) add(s string) {
	ss.s = append(ss.s, s)
	ss.dirty = true
	}

	func (ss *stringSet) values() []string {
	ss.compact()
	return ss.s
	}

	func (ss *stringSet) contains(s string) bool {
	if ss.all {
	return true
	}
	for _, v := range ss.s {
	if v == s {
	return true
	}
	}
	return false
	}

	func (ss *stringSet) compact() {
	if !ss.dirty {
	return
	}
	a := ss.s
	sort.Strings(a)
	k := 0
	for i := 1; i < len(a); i++ {
	if a[k] != a[i] {
	a[k+1] = a[i]
	k++
	}
	}
	ss.s = a[:k+1]
	ss.dirty = false
	}

	func skipLang(l string) bool {
	if include.Len() > 0 {
	return !include.contains(l)
	}
	return exclude.contains(l)
	}

	// altInclude returns a list of alternatives (for the LDML alt attribute)
	// in order of preference. An empty string in this list indicates the
	// default entry.
	func altInclude() []string {
	l := []string{}
	if *short {
	l = append(l, "short")
	}
	l = append(l, "")
	// TODO: handle draft using cldr.SetDraftLevel
	if *draft {
	l = append(l, "proposed")
	}
	return l
	}

	func failOnError(e error) {
	if e != nil {
	log.Panic(e)
	}
	}

	func openArchive() *zip.Reader {
	f := gen.OpenCLDRCoreZip()
	buffer, err := ioutil.ReadAll(f)
	f.Close()
	failOnError(err)
	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
	failOnError(err)
	return archive
	}

	// parseUCA parses a Default Unicode Collation Element Table of the format
	// specified in http://www.unicode.org/reports/tr10/#File_Format.
	// It returns the variable top.
	func parseUCA(builder *build.Builder) {
	var r io.ReadCloser
	var err error
	for _, f := range openArchive().File {
	if strings.HasSuffix(f.Name, "allkeys_CLDR.txt") {
	r, err = f.Open()
	}
	}
	if r == nil {
	log.Fatal("File allkeys_CLDR.txt not found in archive.")
	}
	failOnError(err)
	defer r.Close()
	scanner := bufio.NewScanner(r)
	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
	for i := 1; scanner.Scan(); i++ {
	line := scanner.Text()
	if len(line) == 0 \|\| line[0] == '#' {
	continue
	}
	if line[0] == '@' {
	// parse properties
	switch {
	case strings.HasPrefix(line[1:], "version "):
	a := strings.Split(line[1:], " ")
	if a[1] != gen.UnicodeVersion() {
	log.Fatalf("incompatible version %s; want %s", a[1], gen.UnicodeVersion())
	}
	case strings.HasPrefix(line[1:], "backwards "):
	log.Fatalf("%d: unsupported option backwards", i)
	default:
	log.Printf("%d: unknown option %s", i, line[1:])
	}
	} else {
	// parse entries
	part := strings.Split(line, " ; ")
	if len(part) != 2 {
	log.Fatalf("%d: production rule without ';': %v", i, line)
	}
	lhs := []rune{}
	for _, v := range strings.Split(part[0], " ") {
	if v == "" {
	continue
	}
	lhs = append(lhs, rune(convHex(i, v)))
	}
	var n int
	var vars []int
	rhs := [][]int{}
	for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
	n += len(m[0])
	elem := []int{}
	for _, h := range strings.Split(m[2], ".") {
	elem = append(elem, convHex(i, h))
	}
	if m[1] == "*" {
	vars = append(vars, i)
	}
	rhs = append(rhs, elem)
	}
	if len(part[1]) < n+3 \|\| part[1][n+1] != '#' {
	log.Fatalf("%d: expected comment; found %s", i, part[1][n:])
	}
	if *test {
	testInput.add(string(lhs))
	}
	failOnError(builder.Add(lhs, rhs, vars))
	}
	}
	if scanner.Err() != nil {
	log.Fatal(scanner.Err())
	}
	}

	func convHex(line int, s string) int {
	r, e := strconv.ParseInt(s, 16, 32)
	if e != nil {
	log.Fatalf("%d: %v", line, e)
	}
	return int(r)
	}

	var testInput = stringSet{}

	var charRe = regexp.MustCompile(`&#x([0-9A-F]*);`)
	var tagRe = regexp.MustCompile(`<([a-z_]) />`)

	var mainLocales = []string{}

	// charsets holds a list of exemplar characters per category.
	type charSets map[string][]string

	func (p charSets) fprint(w io.Writer) {
	fmt.Fprintln(w, "[exN]string{")
	for i, k := range []string{"", "contractions", "punctuation", "auxiliary", "currencySymbol", "index"} {
	if set := p[k]; len(set) != 0 {
	fmt.Fprintf(w, "\t\t%d: %q,\n", i, strings.Join(set, " "))
	}
	}
	fmt.Fprintln(w, "\t},")
	}

	var localeChars = make(map[string]charSets)

	const exemplarHeader = `
	type exemplarType int
	const (
	exCharacters exemplarType = iota
	exContractions
	exPunctuation
	exAuxiliary
	exCurrency
	exIndex
	exN
	)
	`

	func printExemplarCharacters(w io.Writer) {
	fmt.Fprintln(w, exemplarHeader)
	fmt.Fprintln(w, "var exemplarCharacters = map[string][exN]string{")
	for _, loc := range mainLocales {
	fmt.Fprintf(w, "\t%q: ", loc)
	localeChars[loc].fprint(w)
	}
	fmt.Fprintln(w, "}")
	}

	func decodeCLDR(d cldr.Decoder) cldr.CLDR {
	r := gen.OpenCLDRCoreZip()
	data, err := d.DecodeZip(r)
	failOnError(err)
	return data
	}

	// parseMain parses XML files in the main directory of the CLDR core.zip file.
	func parseMain() {
	d := &cldr.Decoder{}
	d.SetDirFilter("main")
	d.SetSectionFilter("characters")
	data := decodeCLDR(d)
	for _, loc := range data.Locales() {
	x := data.RawLDML(loc)
	if skipLang(x.Identity.Language.Type) {
	continue
	}
	if x.Characters != nil {
	x, _ = data.LDML(loc)
	loc = language.Make(loc).String()
	for _, ec := range x.Characters.ExemplarCharacters {
	if ec.Draft != "" {
	continue
	}
	if _, ok := localeChars[loc]; !ok {
	mainLocales = append(mainLocales, loc)
	localeChars[loc] = make(charSets)
	}
	localeChars[loc][ec.Type] = parseCharacters(ec.Data())
	}
	}
	}
	}

	func parseCharacters(chars string) []string {
	parseSingle := func(s string) (r rune, tail string, escaped bool) {
	if s[0] == '\\' {
	return rune(s[1]), s[2:], true
	}
	r, sz := utf8.DecodeRuneInString(s)
	return r, s[sz:], false
	}
	chars = strings.TrimSpace(chars)
	if n := len(chars) - 1; chars[n] == ']' && chars[0] == '[' {
	chars = chars[1:n]
	}
	list := []string{}
	var r, last, end rune
	for len(chars) > 0 {
	if chars[0] == '{' { // character sequence
	buf := []rune{}
	for chars = chars[1:]; len(chars) > 0; {
	r, chars, _ = parseSingle(chars)
	if r == '}' {
	break
	}
	if r == ' ' {
	log.Fatalf("space not supported in sequence %q", chars)
	}
	buf = append(buf, r)
	}
	list = append(list, string(buf))
	last = 0
	} else { // single character
	escaped := false
	r, chars, escaped = parseSingle(chars)
	if r != ' ' {
	if r == '-' && !escaped {
	if last == 0 {
	log.Fatal("'-' should be preceded by a character")
	}
	end, chars, _ = parseSingle(chars)
	for ; last <= end; last++ {
	list = append(list, string(last))
	}
	last = 0
	} else {
	list = append(list, string(r))
	last = r
	}
	}
	}
	}
	return list
	}

	var fileRe = regexp.MustCompile(`./collation/(.)\.xml`)

	// typeMap translates legacy type keys to their BCP47 equivalent.
	var typeMap = map[string]string{
	"phonebook": "phonebk",
	"traditional": "trad",
	}

	// parseCollation parses XML files in the collation directory of the CLDR core.zip file.
	func parseCollation(b *build.Builder) {
	d := &cldr.Decoder{}
	d.SetDirFilter("collation")
	data := decodeCLDR(d)
	for _, loc := range data.Locales() {
	x, err := data.LDML(loc)
	failOnError(err)
	if skipLang(x.Identity.Language.Type) {
	continue
	}
	cs := x.Collations.Collation
	sl := cldr.MakeSlice(&cs)
	if len(types.s) == 0 {
	sl.SelectAnyOf("type", x.Collations.Default())
	} else if !types.all {
	sl.SelectAnyOf("type", types.s...)
	}
	sl.SelectOnePerGroup("alt", altInclude())

	for _, c := range cs {
	id, err := language.Parse(loc)
	if err != nil {
	fmt.Fprintf(os.Stderr, "invalid locale: %q", err)
	continue
	}
	// Support both old- and new-style defaults.
	d := c.Type
	if x.Collations.DefaultCollation == nil {
	d = x.Collations.Default()
	} else {
	d = x.Collations.DefaultCollation.Data()
	}
	// We assume tables are being built either for search or collation,
	// but not both. For search the default is always "search".
	if d != c.Type && c.Type != "search" {
	typ := c.Type
	if len(c.Type) > 8 {
	typ = typeMap[c.Type]
	}
	id, err = id.SetTypeForKey("co", typ)
	failOnError(err)
	}
	t := b.Tailoring(id)
	c.Process(processor{t})
	}
	}
	}

	type processor struct {
	t *build.Tailoring
	}

	func (p processor) Reset(anchor string, before int) (err error) {
	if before != 0 {
	err = p.t.SetAnchorBefore(anchor)
	} else {
	err = p.t.SetAnchor(anchor)
	}
	failOnError(err)
	return nil
	}

	func (p processor) Insert(level int, str, context, extend string) error {
	str = context + str
	if *test {
	testInput.add(str)
	}
	// TODO: mimic bug in old maketables: remove.
	err := p.t.Insert(colltab.Level(level-1), str, context+extend)
	failOnError(err)
	return nil
	}

	func (p processor) Index(id string) {
	}

	func testCollator(c *collate.Collator) {
	c0 := collate.New(language.Und)

	// iterator over all characters for all locales and check
	// whether Key is equal.
	buf := collate.Buffer{}

	// Add all common and not too uncommon runes to the test set.
	for i := rune(0); i < 0x30000; i++ {
	testInput.add(string(i))
	}
	for i := rune(0xE0000); i < 0xF0000; i++ {
	testInput.add(string(i))
	}
	for _, str := range testInput.values() {
	k0 := c0.KeyFromString(&buf, str)
	k := c.KeyFromString(&buf, str)
	if !bytes.Equal(k0, k) {
	failOnError(fmt.Errorf("test:%U: keys differ (%x vs %x)", []rune(str), k0, k))
	}
	buf.Reset()
	}
	fmt.Println("PASS")
	}

	func main() {
	gen.Init()
	b := build.NewBuilder()
	parseUCA(b)
	if tables.contains("chars") {
	parseMain()
	}
	parseCollation(b)

	c, err := b.Build()
	failOnError(err)

	if *test {
	testCollator(collate.NewFromTable(c))
	} else {
	w := &bytes.Buffer{}

	gen.WriteUnicodeVersion(w)
	gen.WriteCLDRVersion(w)

	if tables.contains("collate") {
	_, err = b.Print(w)
	failOnError(err)
	}
	if tables.contains("chars") {
	printExemplarCharacters(w)
	}
	gen.WriteGoFile("tables.go", *pkg, w.Bytes())
	}
	}