khenaidoo | ac63710 | 2019-01-14 15:44:34 -0500 | [diff] [blame] | 1 | // Copyright 2015 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | // Package gen contains common code for the various code generation tools in the |
| 6 | // text repository. Its usage ensures consistency between tools. |
| 7 | // |
| 8 | // This package defines command line flags that are common to most generation |
| 9 | // tools. The flags allow for specifying specific Unicode and CLDR versions |
| 10 | // in the public Unicode data repository (http://www.unicode.org/Public). |
| 11 | // |
| 12 | // A local Unicode data mirror can be set through the flag -local or the |
| 13 | // environment variable UNICODE_DIR. The former takes precedence. The local |
| 14 | // directory should follow the same structure as the public repository. |
| 15 | // |
| 16 | // IANA data can also optionally be mirrored by putting it in the iana directory |
| 17 | // rooted at the top of the local mirror. Beware, though, that IANA data is not |
| 18 | // versioned. So it is up to the developer to use the right version. |
| 19 | package gen // import "golang.org/x/text/internal/gen" |
| 20 | |
| 21 | import ( |
| 22 | "bytes" |
| 23 | "flag" |
| 24 | "fmt" |
| 25 | "go/build" |
| 26 | "go/format" |
| 27 | "io" |
| 28 | "io/ioutil" |
| 29 | "log" |
| 30 | "net/http" |
| 31 | "os" |
| 32 | "path" |
| 33 | "path/filepath" |
| 34 | "strings" |
| 35 | "sync" |
| 36 | "unicode" |
| 37 | |
| 38 | "golang.org/x/text/unicode/cldr" |
| 39 | ) |
| 40 | |
| 41 | var ( |
| 42 | url = flag.String("url", |
| 43 | "http://www.unicode.org/Public", |
| 44 | "URL of Unicode database directory") |
| 45 | iana = flag.String("iana", |
| 46 | "http://www.iana.org", |
| 47 | "URL of the IANA repository") |
| 48 | unicodeVersion = flag.String("unicode", |
| 49 | getEnv("UNICODE_VERSION", unicode.Version), |
| 50 | "unicode version to use") |
| 51 | cldrVersion = flag.String("cldr", |
| 52 | getEnv("CLDR_VERSION", cldr.Version), |
| 53 | "cldr version to use") |
| 54 | ) |
| 55 | |
| 56 | func getEnv(name, def string) string { |
| 57 | if v := os.Getenv(name); v != "" { |
| 58 | return v |
| 59 | } |
| 60 | return def |
| 61 | } |
| 62 | |
| 63 | // Init performs common initialization for a gen command. It parses the flags |
| 64 | // and sets up the standard logging parameters. |
| 65 | func Init() { |
| 66 | log.SetPrefix("") |
| 67 | log.SetFlags(log.Lshortfile) |
| 68 | flag.Parse() |
| 69 | } |
| 70 | |
| 71 | const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. |
| 72 | |
| 73 | ` |
| 74 | |
| 75 | // UnicodeVersion reports the requested Unicode version. |
| 76 | func UnicodeVersion() string { |
| 77 | return *unicodeVersion |
| 78 | } |
| 79 | |
| 80 | // CLDRVersion reports the requested CLDR version. |
| 81 | func CLDRVersion() string { |
| 82 | return *cldrVersion |
| 83 | } |
| 84 | |
| 85 | var tags = []struct{ version, buildTags string }{ |
| 86 | {"10.0.0", "go1.10"}, |
| 87 | {"", "!go1.10"}, |
| 88 | } |
| 89 | |
| 90 | // buildTags reports the build tags used for the current Unicode version. |
| 91 | func buildTags() string { |
| 92 | v := UnicodeVersion() |
| 93 | for _, x := range tags { |
| 94 | // We should do a numeric comparison, but including the collate package |
| 95 | // would create an import cycle. We approximate it by assuming that |
| 96 | // longer version strings are later. |
| 97 | if len(x.version) <= len(v) { |
| 98 | return x.buildTags |
| 99 | } |
| 100 | if len(x.version) == len(v) && x.version <= v { |
| 101 | return x.buildTags |
| 102 | } |
| 103 | } |
| 104 | return tags[0].buildTags |
| 105 | } |
| 106 | |
| 107 | // IsLocal reports whether data files are available locally. |
| 108 | func IsLocal() bool { |
| 109 | dir, err := localReadmeFile() |
| 110 | if err != nil { |
| 111 | return false |
| 112 | } |
| 113 | if _, err = os.Stat(dir); err != nil { |
| 114 | return false |
| 115 | } |
| 116 | return true |
| 117 | } |
| 118 | |
| 119 | // OpenUCDFile opens the requested UCD file. The file is specified relative to |
| 120 | // the public Unicode root directory. It will call log.Fatal if there are any |
| 121 | // errors. |
| 122 | func OpenUCDFile(file string) io.ReadCloser { |
| 123 | return openUnicode(path.Join(*unicodeVersion, "ucd", file)) |
| 124 | } |
| 125 | |
| 126 | // OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there |
| 127 | // are any errors. |
| 128 | func OpenCLDRCoreZip() io.ReadCloser { |
| 129 | return OpenUnicodeFile("cldr", *cldrVersion, "core.zip") |
| 130 | } |
| 131 | |
| 132 | // OpenUnicodeFile opens the requested file of the requested category from the |
| 133 | // root of the Unicode data archive. The file is specified relative to the |
| 134 | // public Unicode root directory. If version is "", it will use the default |
| 135 | // Unicode version. It will call log.Fatal if there are any errors. |
| 136 | func OpenUnicodeFile(category, version, file string) io.ReadCloser { |
| 137 | if version == "" { |
| 138 | version = UnicodeVersion() |
| 139 | } |
| 140 | return openUnicode(path.Join(category, version, file)) |
| 141 | } |
| 142 | |
| 143 | // OpenIANAFile opens the requested IANA file. The file is specified relative |
| 144 | // to the IANA root, which is typically either http://www.iana.org or the |
| 145 | // iana directory in the local mirror. It will call log.Fatal if there are any |
| 146 | // errors. |
| 147 | func OpenIANAFile(path string) io.ReadCloser { |
| 148 | return Open(*iana, "iana", path) |
| 149 | } |
| 150 | |
| 151 | var ( |
| 152 | dirMutex sync.Mutex |
| 153 | localDir string |
| 154 | ) |
| 155 | |
| 156 | const permissions = 0755 |
| 157 | |
| 158 | func localReadmeFile() (string, error) { |
| 159 | p, err := build.Import("golang.org/x/text", "", build.FindOnly) |
| 160 | if err != nil { |
| 161 | return "", fmt.Errorf("Could not locate package: %v", err) |
| 162 | } |
| 163 | return filepath.Join(p.Dir, "DATA", "README"), nil |
| 164 | } |
| 165 | |
| 166 | func getLocalDir() string { |
| 167 | dirMutex.Lock() |
| 168 | defer dirMutex.Unlock() |
| 169 | |
| 170 | readme, err := localReadmeFile() |
| 171 | if err != nil { |
| 172 | log.Fatal(err) |
| 173 | } |
| 174 | dir := filepath.Dir(readme) |
| 175 | if _, err := os.Stat(readme); err != nil { |
| 176 | if err := os.MkdirAll(dir, permissions); err != nil { |
| 177 | log.Fatalf("Could not create directory: %v", err) |
| 178 | } |
| 179 | ioutil.WriteFile(readme, []byte(readmeTxt), permissions) |
| 180 | } |
| 181 | return dir |
| 182 | } |
| 183 | |
| 184 | const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT. |
| 185 | |
| 186 | This directory contains downloaded files used to generate the various tables |
| 187 | in the golang.org/x/text subrepo. |
| 188 | |
| 189 | Note that the language subtag repo (iana/assignments/language-subtag-registry) |
| 190 | and all other times in the iana subdirectory are not versioned and will need |
| 191 | to be periodically manually updated. The easiest way to do this is to remove |
| 192 | the entire iana directory. This is mostly of concern when updating the language |
| 193 | package. |
| 194 | ` |
| 195 | |
| 196 | // Open opens subdir/path if a local directory is specified and the file exists, |
| 197 | // where subdir is a directory relative to the local root, or fetches it from |
| 198 | // urlRoot/path otherwise. It will call log.Fatal if there are any errors. |
| 199 | func Open(urlRoot, subdir, path string) io.ReadCloser { |
| 200 | file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path)) |
| 201 | return open(file, urlRoot, path) |
| 202 | } |
| 203 | |
| 204 | func openUnicode(path string) io.ReadCloser { |
| 205 | file := filepath.Join(getLocalDir(), filepath.FromSlash(path)) |
| 206 | return open(file, *url, path) |
| 207 | } |
| 208 | |
| 209 | // TODO: automatically periodically update non-versioned files. |
| 210 | |
| 211 | func open(file, urlRoot, path string) io.ReadCloser { |
| 212 | if f, err := os.Open(file); err == nil { |
| 213 | return f |
| 214 | } |
| 215 | r := get(urlRoot, path) |
| 216 | defer r.Close() |
| 217 | b, err := ioutil.ReadAll(r) |
| 218 | if err != nil { |
| 219 | log.Fatalf("Could not download file: %v", err) |
| 220 | } |
| 221 | os.MkdirAll(filepath.Dir(file), permissions) |
| 222 | if err := ioutil.WriteFile(file, b, permissions); err != nil { |
| 223 | log.Fatalf("Could not create file: %v", err) |
| 224 | } |
| 225 | return ioutil.NopCloser(bytes.NewReader(b)) |
| 226 | } |
| 227 | |
| 228 | func get(root, path string) io.ReadCloser { |
| 229 | url := root + "/" + path |
| 230 | fmt.Printf("Fetching %s...", url) |
| 231 | defer fmt.Println(" done.") |
| 232 | resp, err := http.Get(url) |
| 233 | if err != nil { |
| 234 | log.Fatalf("HTTP GET: %v", err) |
| 235 | } |
| 236 | if resp.StatusCode != 200 { |
| 237 | log.Fatalf("Bad GET status for %q: %q", url, resp.Status) |
| 238 | } |
| 239 | return resp.Body |
| 240 | } |
| 241 | |
| 242 | // TODO: use Write*Version in all applicable packages. |
| 243 | |
| 244 | // WriteUnicodeVersion writes a constant for the Unicode version from which the |
| 245 | // tables are generated. |
| 246 | func WriteUnicodeVersion(w io.Writer) { |
| 247 | fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n") |
| 248 | fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion()) |
| 249 | } |
| 250 | |
| 251 | // WriteCLDRVersion writes a constant for the CLDR version from which the |
| 252 | // tables are generated. |
| 253 | func WriteCLDRVersion(w io.Writer) { |
| 254 | fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n") |
| 255 | fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion()) |
| 256 | } |
| 257 | |
| 258 | // WriteGoFile prepends a standard file comment and package statement to the |
| 259 | // given bytes, applies gofmt, and writes them to a file with the given name. |
| 260 | // It will call log.Fatal if there are any errors. |
| 261 | func WriteGoFile(filename, pkg string, b []byte) { |
| 262 | w, err := os.Create(filename) |
| 263 | if err != nil { |
| 264 | log.Fatalf("Could not create file %s: %v", filename, err) |
| 265 | } |
| 266 | defer w.Close() |
| 267 | if _, err = WriteGo(w, pkg, "", b); err != nil { |
| 268 | log.Fatalf("Error writing file %s: %v", filename, err) |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | func insertVersion(filename, version string) string { |
| 273 | suffix := ".go" |
| 274 | if strings.HasSuffix(filename, "_test.go") { |
| 275 | suffix = "_test.go" |
| 276 | } |
| 277 | return fmt.Sprint(filename[:len(filename)-len(suffix)], version, suffix) |
| 278 | } |
| 279 | |
| 280 | // WriteVersionedGoFile prepends a standard file comment, adds build tags to |
| 281 | // version the file for the current Unicode version, and package statement to |
| 282 | // the given bytes, applies gofmt, and writes them to a file with the given |
| 283 | // name. It will call log.Fatal if there are any errors. |
| 284 | func WriteVersionedGoFile(filename, pkg string, b []byte) { |
| 285 | tags := buildTags() |
| 286 | if tags != "" { |
| 287 | filename = insertVersion(filename, UnicodeVersion()) |
| 288 | } |
| 289 | w, err := os.Create(filename) |
| 290 | if err != nil { |
| 291 | log.Fatalf("Could not create file %s: %v", filename, err) |
| 292 | } |
| 293 | defer w.Close() |
| 294 | if _, err = WriteGo(w, pkg, tags, b); err != nil { |
| 295 | log.Fatalf("Error writing file %s: %v", filename, err) |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | // WriteGo prepends a standard file comment and package statement to the given |
| 300 | // bytes, applies gofmt, and writes them to w. |
| 301 | func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) { |
| 302 | src := []byte(header) |
| 303 | if tags != "" { |
| 304 | src = append(src, fmt.Sprintf("// +build %s\n\n", tags)...) |
| 305 | } |
| 306 | src = append(src, fmt.Sprintf("package %s\n\n", pkg)...) |
| 307 | src = append(src, b...) |
| 308 | formatted, err := format.Source(src) |
| 309 | if err != nil { |
| 310 | // Print the generated code even in case of an error so that the |
| 311 | // returned error can be meaningfully interpreted. |
| 312 | n, _ = w.Write(src) |
| 313 | return n, err |
| 314 | } |
| 315 | return w.Write(formatted) |
| 316 | } |
| 317 | |
| 318 | // Repackage rewrites a Go file from belonging to package main to belonging to |
| 319 | // the given package. |
| 320 | func Repackage(inFile, outFile, pkg string) { |
| 321 | src, err := ioutil.ReadFile(inFile) |
| 322 | if err != nil { |
| 323 | log.Fatalf("reading %s: %v", inFile, err) |
| 324 | } |
| 325 | const toDelete = "package main\n\n" |
| 326 | i := bytes.Index(src, []byte(toDelete)) |
| 327 | if i < 0 { |
| 328 | log.Fatalf("Could not find %q in %s.", toDelete, inFile) |
| 329 | } |
| 330 | w := &bytes.Buffer{} |
| 331 | w.Write(src[i+len(toDelete):]) |
| 332 | WriteGoFile(outFile, pkg, w.Bytes()) |
| 333 | } |