William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 1 | // Copyright 2015 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | // Package gen contains common code for the various code generation tools in the |
| 6 | // text repository. Its usage ensures consistency between tools. |
| 7 | // |
| 8 | // This package defines command line flags that are common to most generation |
| 9 | // tools. The flags allow for specifying specific Unicode and CLDR versions |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 10 | // in the public Unicode data repository (https://www.unicode.org/Public). |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 11 | // |
| 12 | // A local Unicode data mirror can be set through the flag -local or the |
| 13 | // environment variable UNICODE_DIR. The former takes precedence. The local |
| 14 | // directory should follow the same structure as the public repository. |
| 15 | // |
| 16 | // IANA data can also optionally be mirrored by putting it in the iana directory |
| 17 | // rooted at the top of the local mirror. Beware, though, that IANA data is not |
| 18 | // versioned. So it is up to the developer to use the right version. |
| 19 | package gen // import "golang.org/x/text/internal/gen" |
| 20 | |
| 21 | import ( |
| 22 | "bytes" |
| 23 | "flag" |
| 24 | "fmt" |
| 25 | "go/build" |
| 26 | "go/format" |
| 27 | "io" |
| 28 | "io/ioutil" |
| 29 | "log" |
| 30 | "net/http" |
| 31 | "os" |
| 32 | "path" |
| 33 | "path/filepath" |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 34 | "regexp" |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 35 | "strings" |
| 36 | "sync" |
| 37 | "unicode" |
| 38 | |
| 39 | "golang.org/x/text/unicode/cldr" |
| 40 | ) |
| 41 | |
| 42 | var ( |
| 43 | url = flag.String("url", |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 44 | "https://www.unicode.org/Public", |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 45 | "URL of Unicode database directory") |
| 46 | iana = flag.String("iana", |
| 47 | "http://www.iana.org", |
| 48 | "URL of the IANA repository") |
| 49 | unicodeVersion = flag.String("unicode", |
| 50 | getEnv("UNICODE_VERSION", unicode.Version), |
| 51 | "unicode version to use") |
| 52 | cldrVersion = flag.String("cldr", |
| 53 | getEnv("CLDR_VERSION", cldr.Version), |
| 54 | "cldr version to use") |
| 55 | ) |
| 56 | |
| 57 | func getEnv(name, def string) string { |
| 58 | if v := os.Getenv(name); v != "" { |
| 59 | return v |
| 60 | } |
| 61 | return def |
| 62 | } |
| 63 | |
| 64 | // Init performs common initialization for a gen command. It parses the flags |
| 65 | // and sets up the standard logging parameters. |
| 66 | func Init() { |
| 67 | log.SetPrefix("") |
| 68 | log.SetFlags(log.Lshortfile) |
| 69 | flag.Parse() |
| 70 | } |
| 71 | |
| 72 | const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. |
| 73 | |
| 74 | ` |
| 75 | |
| 76 | // UnicodeVersion reports the requested Unicode version. |
| 77 | func UnicodeVersion() string { |
| 78 | return *unicodeVersion |
| 79 | } |
| 80 | |
| 81 | // CLDRVersion reports the requested CLDR version. |
| 82 | func CLDRVersion() string { |
| 83 | return *cldrVersion |
| 84 | } |
| 85 | |
| 86 | var tags = []struct{ version, buildTags string }{ |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 87 | {"9.0.0", "!go1.10"}, |
| 88 | {"10.0.0", "go1.10,!go1.13"}, |
| 89 | {"11.0.0", "go1.13"}, |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 90 | } |
| 91 | |
| 92 | // buildTags reports the build tags used for the current Unicode version. |
| 93 | func buildTags() string { |
| 94 | v := UnicodeVersion() |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 95 | for _, e := range tags { |
| 96 | if e.version == v { |
| 97 | return e.buildTags |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 98 | } |
| 99 | } |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 100 | log.Fatalf("Unknown build tags for Unicode version %q.", v) |
| 101 | return "" |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 102 | } |
| 103 | |
| 104 | // IsLocal reports whether data files are available locally. |
| 105 | func IsLocal() bool { |
| 106 | dir, err := localReadmeFile() |
| 107 | if err != nil { |
| 108 | return false |
| 109 | } |
| 110 | if _, err = os.Stat(dir); err != nil { |
| 111 | return false |
| 112 | } |
| 113 | return true |
| 114 | } |
| 115 | |
| 116 | // OpenUCDFile opens the requested UCD file. The file is specified relative to |
| 117 | // the public Unicode root directory. It will call log.Fatal if there are any |
| 118 | // errors. |
| 119 | func OpenUCDFile(file string) io.ReadCloser { |
| 120 | return openUnicode(path.Join(*unicodeVersion, "ucd", file)) |
| 121 | } |
| 122 | |
| 123 | // OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there |
| 124 | // are any errors. |
| 125 | func OpenCLDRCoreZip() io.ReadCloser { |
| 126 | return OpenUnicodeFile("cldr", *cldrVersion, "core.zip") |
| 127 | } |
| 128 | |
| 129 | // OpenUnicodeFile opens the requested file of the requested category from the |
| 130 | // root of the Unicode data archive. The file is specified relative to the |
| 131 | // public Unicode root directory. If version is "", it will use the default |
| 132 | // Unicode version. It will call log.Fatal if there are any errors. |
| 133 | func OpenUnicodeFile(category, version, file string) io.ReadCloser { |
| 134 | if version == "" { |
| 135 | version = UnicodeVersion() |
| 136 | } |
| 137 | return openUnicode(path.Join(category, version, file)) |
| 138 | } |
| 139 | |
| 140 | // OpenIANAFile opens the requested IANA file. The file is specified relative |
| 141 | // to the IANA root, which is typically either http://www.iana.org or the |
| 142 | // iana directory in the local mirror. It will call log.Fatal if there are any |
| 143 | // errors. |
| 144 | func OpenIANAFile(path string) io.ReadCloser { |
| 145 | return Open(*iana, "iana", path) |
| 146 | } |
| 147 | |
| 148 | var ( |
| 149 | dirMutex sync.Mutex |
| 150 | localDir string |
| 151 | ) |
| 152 | |
| 153 | const permissions = 0755 |
| 154 | |
| 155 | func localReadmeFile() (string, error) { |
| 156 | p, err := build.Import("golang.org/x/text", "", build.FindOnly) |
| 157 | if err != nil { |
| 158 | return "", fmt.Errorf("Could not locate package: %v", err) |
| 159 | } |
| 160 | return filepath.Join(p.Dir, "DATA", "README"), nil |
| 161 | } |
| 162 | |
| 163 | func getLocalDir() string { |
| 164 | dirMutex.Lock() |
| 165 | defer dirMutex.Unlock() |
| 166 | |
| 167 | readme, err := localReadmeFile() |
| 168 | if err != nil { |
| 169 | log.Fatal(err) |
| 170 | } |
| 171 | dir := filepath.Dir(readme) |
| 172 | if _, err := os.Stat(readme); err != nil { |
| 173 | if err := os.MkdirAll(dir, permissions); err != nil { |
| 174 | log.Fatalf("Could not create directory: %v", err) |
| 175 | } |
| 176 | ioutil.WriteFile(readme, []byte(readmeTxt), permissions) |
| 177 | } |
| 178 | return dir |
| 179 | } |
| 180 | |
| 181 | const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT. |
| 182 | |
| 183 | This directory contains downloaded files used to generate the various tables |
| 184 | in the golang.org/x/text subrepo. |
| 185 | |
| 186 | Note that the language subtag repo (iana/assignments/language-subtag-registry) |
| 187 | and all other times in the iana subdirectory are not versioned and will need |
| 188 | to be periodically manually updated. The easiest way to do this is to remove |
| 189 | the entire iana directory. This is mostly of concern when updating the language |
| 190 | package. |
| 191 | ` |
| 192 | |
| 193 | // Open opens subdir/path if a local directory is specified and the file exists, |
| 194 | // where subdir is a directory relative to the local root, or fetches it from |
| 195 | // urlRoot/path otherwise. It will call log.Fatal if there are any errors. |
| 196 | func Open(urlRoot, subdir, path string) io.ReadCloser { |
| 197 | file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path)) |
| 198 | return open(file, urlRoot, path) |
| 199 | } |
| 200 | |
| 201 | func openUnicode(path string) io.ReadCloser { |
| 202 | file := filepath.Join(getLocalDir(), filepath.FromSlash(path)) |
| 203 | return open(file, *url, path) |
| 204 | } |
| 205 | |
| 206 | // TODO: automatically periodically update non-versioned files. |
| 207 | |
| 208 | func open(file, urlRoot, path string) io.ReadCloser { |
| 209 | if f, err := os.Open(file); err == nil { |
| 210 | return f |
| 211 | } |
| 212 | r := get(urlRoot, path) |
| 213 | defer r.Close() |
| 214 | b, err := ioutil.ReadAll(r) |
| 215 | if err != nil { |
| 216 | log.Fatalf("Could not download file: %v", err) |
| 217 | } |
| 218 | os.MkdirAll(filepath.Dir(file), permissions) |
| 219 | if err := ioutil.WriteFile(file, b, permissions); err != nil { |
| 220 | log.Fatalf("Could not create file: %v", err) |
| 221 | } |
| 222 | return ioutil.NopCloser(bytes.NewReader(b)) |
| 223 | } |
| 224 | |
| 225 | func get(root, path string) io.ReadCloser { |
| 226 | url := root + "/" + path |
| 227 | fmt.Printf("Fetching %s...", url) |
| 228 | defer fmt.Println(" done.") |
| 229 | resp, err := http.Get(url) |
| 230 | if err != nil { |
| 231 | log.Fatalf("HTTP GET: %v", err) |
| 232 | } |
| 233 | if resp.StatusCode != 200 { |
| 234 | log.Fatalf("Bad GET status for %q: %q", url, resp.Status) |
| 235 | } |
| 236 | return resp.Body |
| 237 | } |
| 238 | |
| 239 | // TODO: use Write*Version in all applicable packages. |
| 240 | |
| 241 | // WriteUnicodeVersion writes a constant for the Unicode version from which the |
| 242 | // tables are generated. |
| 243 | func WriteUnicodeVersion(w io.Writer) { |
| 244 | fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n") |
| 245 | fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion()) |
| 246 | } |
| 247 | |
| 248 | // WriteCLDRVersion writes a constant for the CLDR version from which the |
| 249 | // tables are generated. |
| 250 | func WriteCLDRVersion(w io.Writer) { |
| 251 | fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n") |
| 252 | fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion()) |
| 253 | } |
| 254 | |
| 255 | // WriteGoFile prepends a standard file comment and package statement to the |
| 256 | // given bytes, applies gofmt, and writes them to a file with the given name. |
| 257 | // It will call log.Fatal if there are any errors. |
| 258 | func WriteGoFile(filename, pkg string, b []byte) { |
| 259 | w, err := os.Create(filename) |
| 260 | if err != nil { |
| 261 | log.Fatalf("Could not create file %s: %v", filename, err) |
| 262 | } |
| 263 | defer w.Close() |
| 264 | if _, err = WriteGo(w, pkg, "", b); err != nil { |
| 265 | log.Fatalf("Error writing file %s: %v", filename, err) |
| 266 | } |
| 267 | } |
| 268 | |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 269 | func fileToPattern(filename string) string { |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 270 | suffix := ".go" |
| 271 | if strings.HasSuffix(filename, "_test.go") { |
| 272 | suffix = "_test.go" |
| 273 | } |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 274 | prefix := filename[:len(filename)-len(suffix)] |
| 275 | return fmt.Sprint(prefix, "%s", suffix) |
| 276 | } |
| 277 | |
| 278 | func updateBuildTags(pattern string) { |
| 279 | for _, t := range tags { |
| 280 | oldFile := fmt.Sprintf(pattern, t.version) |
| 281 | b, err := ioutil.ReadFile(oldFile) |
| 282 | if err != nil { |
| 283 | continue |
| 284 | } |
| 285 | build := fmt.Sprintf("// +build %s", t.buildTags) |
| 286 | b = regexp.MustCompile(`// \+build .*`).ReplaceAll(b, []byte(build)) |
| 287 | err = ioutil.WriteFile(oldFile, b, 0644) |
| 288 | if err != nil { |
| 289 | log.Fatal(err) |
| 290 | } |
| 291 | } |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 292 | } |
| 293 | |
| 294 | // WriteVersionedGoFile prepends a standard file comment, adds build tags to |
| 295 | // version the file for the current Unicode version, and package statement to |
| 296 | // the given bytes, applies gofmt, and writes them to a file with the given |
| 297 | // name. It will call log.Fatal if there are any errors. |
| 298 | func WriteVersionedGoFile(filename, pkg string, b []byte) { |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 299 | pattern := fileToPattern(filename) |
| 300 | updateBuildTags(pattern) |
| 301 | filename = fmt.Sprintf(pattern, UnicodeVersion()) |
| 302 | |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 303 | w, err := os.Create(filename) |
| 304 | if err != nil { |
| 305 | log.Fatalf("Could not create file %s: %v", filename, err) |
| 306 | } |
| 307 | defer w.Close() |
Abhilash S.L | 3b49463 | 2019-07-16 15:51:09 +0530 | [diff] [blame] | 308 | if _, err = WriteGo(w, pkg, buildTags(), b); err != nil { |
William Kurkian | ea86948 | 2019-04-09 15:16:11 -0400 | [diff] [blame] | 309 | log.Fatalf("Error writing file %s: %v", filename, err) |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | // WriteGo prepends a standard file comment and package statement to the given |
| 314 | // bytes, applies gofmt, and writes them to w. |
| 315 | func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) { |
| 316 | src := []byte(header) |
| 317 | if tags != "" { |
| 318 | src = append(src, fmt.Sprintf("// +build %s\n\n", tags)...) |
| 319 | } |
| 320 | src = append(src, fmt.Sprintf("package %s\n\n", pkg)...) |
| 321 | src = append(src, b...) |
| 322 | formatted, err := format.Source(src) |
| 323 | if err != nil { |
| 324 | // Print the generated code even in case of an error so that the |
| 325 | // returned error can be meaningfully interpreted. |
| 326 | n, _ = w.Write(src) |
| 327 | return n, err |
| 328 | } |
| 329 | return w.Write(formatted) |
| 330 | } |
| 331 | |
| 332 | // Repackage rewrites a Go file from belonging to package main to belonging to |
| 333 | // the given package. |
| 334 | func Repackage(inFile, outFile, pkg string) { |
| 335 | src, err := ioutil.ReadFile(inFile) |
| 336 | if err != nil { |
| 337 | log.Fatalf("reading %s: %v", inFile, err) |
| 338 | } |
| 339 | const toDelete = "package main\n\n" |
| 340 | i := bytes.Index(src, []byte(toDelete)) |
| 341 | if i < 0 { |
| 342 | log.Fatalf("Could not find %q in %s.", toDelete, inFile) |
| 343 | } |
| 344 | w := &bytes.Buffer{} |
| 345 | w.Write(src[i+len(toDelete):]) |
| 346 | WriteGoFile(outFile, pkg, w.Bytes()) |
| 347 | } |