1 /* 2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package build.tools.cldrconverter; 27 28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames; 29 import build.tools.cldrconverter.BundleGenerator.BundleType; 30 import java.io.File; 31 import java.io.IOException; 32 import java.io.UncheckedIOException; 33 import java.nio.file.*; 34 import java.text.MessageFormat; 35 import java.time.*; 36 import java.util.*; 37 import java.util.ResourceBundle.Control; 38 import java.util.logging.Level; 39 import java.util.logging.Logger; 40 import java.util.stream.Collectors; 41 import java.util.stream.IntStream; 42 import java.util.stream.Stream; 43 import javax.xml.parsers.SAXParser; 44 import javax.xml.parsers.SAXParserFactory; 45 import org.xml.sax.SAXNotRecognizedException; 46 import org.xml.sax.SAXNotSupportedException; 47 48 49 /** 50 * Converts locale data from "Locale Data Markup Language" format to 51 * JRE resource bundle format. LDML is the format used by the Common 52 * Locale Data Repository maintained by the Unicode Consortium. 53 */ 54 public class CLDRConverter { 55 56 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd"; 57 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd"; 58 static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd"; 59 60 61 private static String CLDR_BASE; 62 static String LOCAL_LDML_DTD; 63 static String LOCAL_SPPL_LDML_DTD; 64 static String LOCAL_BCP47_LDML_DTD; 65 private static String SOURCE_FILE_DIR; 66 private static String SPPL_SOURCE_FILE; 67 private static String SPPL_META_SOURCE_FILE; 68 private static String NUMBERING_SOURCE_FILE; 69 private static String METAZONES_SOURCE_FILE; 70 private static String LIKELYSUBTAGS_SOURCE_FILE; 71 private static String TIMEZONE_SOURCE_FILE; 72 private static String WINZONES_SOURCE_FILE; 73 static String DESTINATION_DIR = "build/gensrc"; 74 75 static final String LOCALE_NAME_PREFIX = "locale.displayname."; 76 static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator"; 77 static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype"; 78 static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key."; 79 static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type."; 80 static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca."; 81 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol."; 82 static final String CURRENCY_NAME_PREFIX = "currency.displayname."; 83 static final String CALENDAR_NAME_PREFIX = "calendarname."; 84 static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay."; 85 static final String CALENDAR_MINDAYS_PREFIX = "minDays."; 86 static final String TIMEZONE_ID_PREFIX = "timezone.id."; 87 static final String EXEMPLAR_CITY_PREFIX = "timezone.excity."; 88 static final String ZONE_NAME_PREFIX = "timezone.displayname."; 89 static final String METAZONE_ID_PREFIX = "metazone.id."; 90 static final String PARENT_LOCALE_PREFIX = "parentLocale."; 91 static final String[] EMPTY_ZONE = {"", "", "", "", "", ""}; 92 93 private static SupplementDataParseHandler handlerSuppl; 94 private static LikelySubtagsParseHandler handlerLikelySubtags; 95 private static WinZonesParseHandler handlerWinZones; 96 static SupplementalMetadataParseHandler handlerSupplMeta; 97 static NumberingSystemsParseHandler handlerNumbering; 98 static MetaZonesParseHandler handlerMetaZones; 99 static TimeZoneParseHandler handlerTimeZone; 100 private static BundleGenerator bundleGenerator; 101 102 // java.base module related 103 static boolean isBaseModule = false; 104 static final Set<Locale> BASE_LOCALES = new HashSet<>(); 105 106 // "parentLocales" map 107 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>(); 108 private static final ResourceBundle.Control defCon = 109 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT); 110 111 private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs(); 112 private static String zoneNameTempFile; 113 private static String tzDataDir; 114 private static final Map<String, String> canonicalTZMap = new HashMap<>(); 115 116 static enum DraftType { 117 UNCONFIRMED, 118 PROVISIONAL, 119 CONTRIBUTED, 120 APPROVED; 121 122 private static final Map<String, DraftType> map = new HashMap<>(); 123 static { 124 for (DraftType dt : values()) { 125 map.put(dt.getKeyword(), dt); 126 } 127 } 128 static private DraftType defaultType = CONTRIBUTED; 129 130 private final String keyword; 131 132 private DraftType() { 133 keyword = this.name().toLowerCase(Locale.ROOT); 134 135 } 136 137 static DraftType forKeyword(String keyword) { 138 return map.get(keyword); 139 } 140 141 static DraftType getDefault() { 142 return defaultType; 143 } 144 145 static void setDefault(String keyword) { 146 defaultType = Objects.requireNonNull(forKeyword(keyword)); 147 } 148 149 String getKeyword() { 150 return keyword; 151 } 152 } 153 154 static boolean USE_UTF8 = false; 155 private static boolean verbose; 156 157 private CLDRConverter() { 158 // no instantiation 159 } 160 161 @SuppressWarnings("AssignmentToForLoopParameter") 162 public static void main(String[] args) throws Exception { 163 if (args.length != 0) { 164 String currentArg = null; 165 try { 166 for (int i = 0; i < args.length; i++) { 167 currentArg = args[i]; 168 switch (currentArg) { 169 case "-draft": 170 String draftDataType = args[++i]; 171 try { 172 DraftType.setDefault(draftDataType); 173 } catch (NullPointerException e) { 174 severe("Error: incorrect draft value: %s%n", draftDataType); 175 System.exit(1); 176 } 177 info("Using the specified data type: %s%n", draftDataType); 178 break; 179 180 case "-base": 181 // base directory for input files 182 CLDR_BASE = args[++i]; 183 if (!CLDR_BASE.endsWith("/")) { 184 CLDR_BASE += "/"; 185 } 186 break; 187 188 case "-baselocales": 189 // base locales 190 setupBaseLocales(args[++i]); 191 break; 192 193 case "-basemodule": 194 // indicates java.base module resource generation 195 isBaseModule = true; 196 break; 197 198 case "-o": 199 // output directory 200 DESTINATION_DIR = args[++i]; 201 break; 202 203 case "-utf8": 204 USE_UTF8 = true; 205 break; 206 207 case "-verbose": 208 verbose = true; 209 break; 210 211 case "-zntempfile": 212 zoneNameTempFile = args[++i]; 213 break; 214 215 case "-tzdatadir": 216 tzDataDir = args[++i]; 217 break; 218 219 case "-help": 220 usage(); 221 System.exit(0); 222 break; 223 224 default: 225 throw new RuntimeException(); 226 } 227 } 228 } catch (RuntimeException e) { 229 severe("unknown or imcomplete arg(s): " + currentArg); 230 usage(); 231 System.exit(1); 232 } 233 } 234 235 // Set up path names 236 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd"; 237 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd"; 238 LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd"; 239 SOURCE_FILE_DIR = CLDR_BASE + "/main"; 240 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml"; 241 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml"; 242 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml"; 243 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml"; 244 TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml"; 245 SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml"; 246 WINZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/windowsZones.xml"; 247 248 if (BASE_LOCALES.isEmpty()) { 249 setupBaseLocales("en-US"); 250 } 251 252 bundleGenerator = new ResourceBundleGenerator(); 253 254 // Parse data independent of locales 255 parseSupplemental(); 256 parseBCP47(); 257 258 List<Bundle> bundles = readBundleList(); 259 convertBundles(bundles); 260 261 if (isBaseModule) { 262 // Generate java.time.format.ZoneName.java 263 generateZoneName(); 264 265 // Generate Windows tzmappings 266 generateWindowsTZMappings(); 267 } 268 } 269 270 private static void usage() { 271 errout("Usage: java CLDRConverter [options]%n" 272 + "\t-help output this usage message and exit%n" 273 + "\t-verbose output information%n" 274 + "\t-draft [contributed | approved | provisional | unconfirmed]%n" 275 + "\t\t draft level for using data (default: contributed)%n" 276 + "\t-base dir base directory for CLDR input files%n" 277 + "\t-basemodule generates bundles that go into java.base module%n" 278 + "\t-baselocales loc(,loc)* locales that go into the base module%n" 279 + "\t-o dir output directory (default: ./build/gensrc)%n" 280 + "\t-zntempfile template file for java.time.format.ZoneName.java%n" 281 + "\t-tzdatadir tzdata directory for java.time.format.ZoneName.java%n" 282 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n"); 283 } 284 285 static void info(String fmt, Object... args) { 286 if (verbose) { 287 System.out.printf(fmt, args); 288 } 289 } 290 291 static void info(String msg) { 292 if (verbose) { 293 System.out.println(msg); 294 } 295 } 296 297 static void warning(String fmt, Object... args) { 298 System.err.print("Warning: "); 299 System.err.printf(fmt, args); 300 } 301 302 static void warning(String msg) { 303 System.err.print("Warning: "); 304 errout(msg); 305 } 306 307 static void severe(String fmt, Object... args) { 308 System.err.print("Error: "); 309 System.err.printf(fmt, args); 310 } 311 312 static void severe(String msg) { 313 System.err.print("Error: "); 314 errout(msg); 315 } 316 317 private static void errout(String msg) { 318 if (msg.contains("%n")) { 319 System.err.printf(msg); 320 } else { 321 System.err.println(msg); 322 } 323 } 324 325 /** 326 * Configure the parser to allow access to DTDs on the file system. 327 */ 328 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException { 329 try { 330 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file"); 331 } catch (SAXNotRecognizedException ignore) { 332 // property requires >= JAXP 1.5 333 } 334 } 335 336 private static List<Bundle> readBundleList() throws Exception { 337 List<Bundle> retList = new ArrayList<>(); 338 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR); 339 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) { 340 for (Path entry : dirStr) { 341 String fileName = entry.getFileName().toString(); 342 if (fileName.endsWith(".xml")) { 343 String id = fileName.substring(0, fileName.indexOf('.')); 344 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id)); 345 StringBuilder sb = getCandLocales(cldrLoc); 346 if (sb.indexOf("root") == -1) { 347 sb.append("root"); 348 } 349 Bundle b = new Bundle(id, sb.toString(), null, null); 350 // Insert the bundle for root at the top so that it will get 351 // processed first. 352 if ("root".equals(id)) { 353 retList.add(0, b); 354 } else { 355 retList.add(b); 356 } 357 } 358 } 359 } 360 return retList; 361 } 362 363 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>(); 364 365 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>(); 366 367 static { 368 // For generating information on supported locales. 369 metaInfo.put("AvailableLocales", new TreeSet<>()); 370 } 371 372 static Map<String, Object> getCLDRBundle(String id) throws Exception { 373 Map<String, Object> bundle = cldrBundles.get(id); 374 if (bundle != null) { 375 return bundle; 376 } 377 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml"); 378 if (!file.exists()) { 379 // Skip if the file doesn't exist. 380 return Collections.emptyMap(); 381 } 382 383 info("..... main directory ....."); 384 LDMLParseHandler handler = new LDMLParseHandler(id); 385 parseLDMLFile(file, handler); 386 387 bundle = handler.getData(); 388 cldrBundles.put(id, bundle); 389 390 if (id.equals("root")) { 391 // Calendar data (firstDayOfWeek & minDaysInFirstWeek) 392 bundle = handlerSuppl.getData("root"); 393 if (bundle != null) { 394 //merge two maps into one map 395 Map<String, Object> temp = cldrBundles.remove(id); 396 bundle.putAll(temp); 397 cldrBundles.put(id, bundle); 398 } 399 } 400 return bundle; 401 } 402 403 // Parsers for data in "supplemental" directory 404 // 405 private static void parseSupplemental() throws Exception { 406 // Parse SupplementalData file and store the information in the HashMap 407 // Calendar information such as firstDay and minDay are stored in 408 // supplementalData.xml as of CLDR1.4. Individual territory is listed 409 // with its ISO 3166 country code while default is listed using UNM49 410 // region and composition numerical code (001 for World.) 411 // 412 // SupplementalData file also provides the "parent" locales which 413 // are othrwise not to be fallen back. Process them here as well. 414 // 415 handlerSuppl = new SupplementDataParseHandler(); 416 parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl); 417 Map<String, Object> parentData = handlerSuppl.getData("root"); 418 parentData.keySet().stream() 419 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX)) 420 .forEach(key -> { 421 parentLocalesMap.put(key, new TreeSet( 422 Arrays.asList(((String)parentData.get(key)).split(" ")))); 423 }); 424 425 // Parse numberingSystems to get digit zero character information. 426 handlerNumbering = new NumberingSystemsParseHandler(); 427 parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering); 428 429 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names 430 handlerMetaZones = new MetaZonesParseHandler(); 431 parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones); 432 433 // Parse likelySubtags 434 handlerLikelySubtags = new LikelySubtagsParseHandler(); 435 parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags); 436 437 // Parse supplementalMetadata 438 // Currently interested in deprecated time zone ids and language aliases. 439 handlerSupplMeta = new SupplementalMetadataParseHandler(); 440 parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta); 441 442 // Parse windowsZones 443 handlerWinZones = new WinZonesParseHandler(); 444 parseLDMLFile(new File(WINZONES_SOURCE_FILE), handlerWinZones); 445 } 446 447 // Parsers for data in "bcp47" directory 448 // 449 private static void parseBCP47() throws Exception { 450 // Parse timezone 451 handlerTimeZone = new TimeZoneParseHandler(); 452 parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone); 453 454 // canonical tz name map 455 // alias -> primary 456 handlerTimeZone.getData().forEach((k, v) -> { 457 String[] ids = ((String)v).split("\\s"); 458 for (int i = 1; i < ids.length; i++) { 459 canonicalTZMap.put(ids[i], ids[0]); 460 } 461 }); 462 } 463 464 private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception { 465 info("..... Parsing " + srcfile.getName() + " ....."); 466 SAXParserFactory pf = SAXParserFactory.newInstance(); 467 pf.setValidating(true); 468 SAXParser parser = pf.newSAXParser(); 469 enableFileAccess(parser); 470 parser.parse(srcfile, handler); 471 } 472 473 private static StringBuilder getCandLocales(Locale cldrLoc) { 474 List<Locale> candList = getCandidateLocales(cldrLoc); 475 StringBuilder sb = new StringBuilder(); 476 for (Locale loc : candList) { 477 if (!loc.equals(Locale.ROOT)) { 478 sb.append(toLocaleName(loc.toLanguageTag())); 479 sb.append(","); 480 } 481 } 482 return sb; 483 } 484 485 private static List<Locale> getCandidateLocales(Locale cldrLoc) { 486 List<Locale> candList = new ArrayList<>(); 487 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc)); 488 return candList; 489 } 490 491 private static void convertBundles(List<Bundle> bundles) throws Exception { 492 // parent locales map. The mappings are put in base metaInfo file 493 // for now. 494 if (isBaseModule) { 495 metaInfo.putAll(parentLocalesMap); 496 } 497 498 for (Bundle bundle : bundles) { 499 // Get the target map, which contains all the data that should be 500 // visible for the bundle's locale 501 502 Map<String, Object> targetMap = bundle.getTargetMap(); 503 504 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes(); 505 506 if (bundle.isRoot()) { 507 // Add DateTimePatternChars because CLDR no longer supports localized patterns. 508 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ"); 509 } 510 511 // Now the map contains just the entries that need to be in the resources bundles. 512 // Go ahead and generate them. 513 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) { 514 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID()); 515 if (!localeNamesMap.isEmpty() || bundle.isRoot()) { 516 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN); 517 } 518 } 519 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) { 520 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies()); 521 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) { 522 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN); 523 } 524 } 525 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) { 526 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID()); 527 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) { 528 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE); 529 } 530 } 531 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) { 532 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID()); 533 if (!calendarDataMap.isEmpty() || bundle.isRoot()) { 534 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN); 535 } 536 } 537 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) { 538 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID()); 539 if (!formatDataMap.isEmpty() || bundle.isRoot()) { 540 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN); 541 } 542 } 543 544 // For AvailableLocales 545 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID())); 546 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID()); 547 } 548 bundleGenerator.generateMetaInfo(metaInfo); 549 } 550 551 static final Map<String, String> aliases = new HashMap<>(); 552 553 /** 554 * Translate the aliases into the real entries in the bundle map. 555 */ 556 static void handleAliases(Map<String, Object> bundleMap) { 557 Set bundleKeys = bundleMap.keySet(); 558 try { 559 for (String key : aliases.keySet()) { 560 String targetKey = aliases.get(key); 561 if (bundleKeys.contains(targetKey)) { 562 bundleMap.putIfAbsent(key, bundleMap.get(targetKey)); 563 } 564 } 565 } catch (Exception ex) { 566 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex); 567 } 568 } 569 570 /* 571 * Returns the language portion of the given id. 572 * If id is "root", "" is returned. 573 */ 574 static String getLanguageCode(String id) { 575 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage(); 576 } 577 578 /** 579 * Examine if the id includes the country (territory) code. If it does, it returns 580 * the country code. 581 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG". 582 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot 583 * be translated into package names. 584 */ 585 static String getCountryCode(String id) { 586 String rgn = getRegionCode(id); 587 return rgn.length() == 2 ? rgn: null; 588 } 589 590 /** 591 * Examine if the id includes the region code. If it does, it returns 592 * the region code. 593 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG". 594 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes. 595 */ 596 static String getRegionCode(String id) { 597 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry(); 598 } 599 600 private static class KeyComparator implements Comparator<String> { 601 static KeyComparator INSTANCE = new KeyComparator(); 602 603 private KeyComparator() { 604 } 605 606 @Override 607 public int compare(String o1, String o2) { 608 int len1 = o1.length(); 609 int len2 = o2.length(); 610 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) { 611 // Shorter string comes first unless either starts with a digit. 612 if (len1 < len2) { 613 return -1; 614 } 615 if (len1 > len2) { 616 return 1; 617 } 618 } 619 return o1.compareTo(o2); 620 } 621 622 private boolean isDigit(char c) { 623 return c >= '0' && c <= '9'; 624 } 625 } 626 627 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) { 628 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE); 629 for (String key : map.keySet()) { 630 if (key.startsWith(LOCALE_NAME_PREFIX)) { 631 switch (key) { 632 case LOCALE_SEPARATOR: 633 localeNames.put("ListCompositionPattern", map.get(key)); 634 break; 635 case LOCALE_KEYTYPE: 636 localeNames.put("ListKeyTypePattern", map.get(key)); 637 break; 638 default: 639 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key)); 640 break; 641 } 642 } 643 } 644 645 if (id.equals("root")) { 646 // Add display name pattern, which is not in CLDR 647 localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}"); 648 } 649 650 return localeNames; 651 } 652 653 @SuppressWarnings("AssignmentToForLoopParameter") 654 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names) 655 throws Exception { 656 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE); 657 for (String key : map.keySet()) { 658 if (key.startsWith(CURRENCY_NAME_PREFIX)) { 659 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key)); 660 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) { 661 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key)); 662 } 663 } 664 return currencyNames; 665 } 666 667 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) { 668 Map<String, Object> names = new HashMap<>(); 669 670 // Copy over missing time zone ids from JRE for English locale 671 if (id.equals("en")) { 672 Map<String[], String> jreMetaMap = new HashMap<>(); 673 jreTimeZoneNames.stream().forEach(e -> { 674 String tzid = (String)e[0]; 675 String[] data = (String[])e[1]; 676 677 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null && 678 handlerMetaZones.get(tzid) == null || 679 handlerMetaZones.get(tzid) != null && 680 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) { 681 682 // First, check the alias 683 String canonID = canonicalTZMap.get(tzid); 684 if (canonID != null && !tzid.equals(canonID)) { 685 Object value = map.get(TIMEZONE_ID_PREFIX + canonID); 686 if (value != null) { 687 names.put(tzid, value); 688 return; 689 } else { 690 String meta = handlerMetaZones.get(canonID); 691 if (meta != null) { 692 value = map.get(METAZONE_ID_PREFIX + meta); 693 if (value != null) { 694 names.put(tzid, meta); 695 return; 696 } 697 } 698 } 699 } 700 701 // Check the CLDR meta key 702 Optional<Map.Entry<String, String>> cldrMeta = 703 handlerMetaZones.getData().entrySet().stream() 704 .filter(me -> 705 Arrays.deepEquals(data, 706 (String[])map.get(METAZONE_ID_PREFIX + me.getValue()))) 707 .findAny(); 708 cldrMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> { 709 // Check the JRE meta key, add if there is not. 710 Optional<Map.Entry<String[], String>> jreMeta = 711 jreMetaMap.entrySet().stream() 712 .filter(jm -> Arrays.deepEquals(data, jm.getKey())) 713 .findAny(); 714 jreMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> { 715 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_"); 716 names.put(METAZONE_ID_PREFIX + metaName, data); 717 names.put(tzid, metaName); 718 }); 719 }); 720 } 721 }); 722 } 723 724 Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> { 725 // If the tzid is deprecated, get the data for the replacement id 726 String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid)) 727 .orElse(tzid); 728 Object data = map.get(TIMEZONE_ID_PREFIX + tzKey); 729 730 if (data instanceof String[]) { 731 names.put(tzid, data); 732 } else { 733 String meta = handlerMetaZones.get(tzKey); 734 if (meta != null) { 735 String metaKey = METAZONE_ID_PREFIX + meta; 736 data = map.get(metaKey); 737 if (data instanceof String[]) { 738 // Keep the metazone prefix here. 739 names.put(metaKey, data); 740 names.put(tzid, meta); 741 } 742 } 743 } 744 }); 745 746 // exemplar cities. 747 Map<String, Object> exCities = map.entrySet().stream() 748 .filter(e -> e.getKey().startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX)) 749 .collect(Collectors 750 .toMap(Map.Entry::getKey, Map.Entry::getValue)); 751 names.putAll(exCities); 752 753 if (!id.equals("en") && 754 !names.isEmpty()) { 755 // CLDR does not have UTC entry, so add it here. 756 names.put("UTC", EMPTY_ZONE); 757 758 // no metazone zones 759 Arrays.asList(handlerMetaZones.get(MetaZonesParseHandler.NO_METAZONE_KEY) 760 .split("\\s")).stream() 761 .forEach(tz -> { 762 names.put(tz, EMPTY_ZONE); 763 }); 764 } 765 766 return names; 767 } 768 769 /** 770 * Extracts the language independent calendar data. Each of the two keys, 771 * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of 772 * one or multiple occurrences of: 773 * i: rg1 rg2 ... rgn; 774 * where "i" is the data for the following regions (delimited by a space) after 775 * ":", and ends with a ";". 776 */ 777 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) { 778 Map<String, Object> calendarData = new LinkedHashMap<>(); 779 if (id.equals("root")) { 780 calendarData.put("firstDayOfWeek", 781 IntStream.range(1, 8) 782 .mapToObj(String::valueOf) 783 .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d)) 784 .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d)) 785 .collect(Collectors.joining(";"))); 786 calendarData.put("minimalDaysInFirstWeek", 787 IntStream.range(0, 7) 788 .mapToObj(String::valueOf) 789 .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d)) 790 .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d)) 791 .collect(Collectors.joining(";"))); 792 } 793 return calendarData; 794 } 795 796 static final String[] FORMAT_DATA_ELEMENTS = { 797 "MonthNames", 798 "standalone.MonthNames", 799 "MonthAbbreviations", 800 "standalone.MonthAbbreviations", 801 "MonthNarrows", 802 "standalone.MonthNarrows", 803 "DayNames", 804 "standalone.DayNames", 805 "DayAbbreviations", 806 "standalone.DayAbbreviations", 807 "DayNarrows", 808 "standalone.DayNarrows", 809 "QuarterNames", 810 "standalone.QuarterNames", 811 "QuarterAbbreviations", 812 "standalone.QuarterAbbreviations", 813 "QuarterNarrows", 814 "standalone.QuarterNarrows", 815 "AmPmMarkers", 816 "narrow.AmPmMarkers", 817 "abbreviated.AmPmMarkers", 818 "long.Eras", 819 "Eras", 820 "narrow.Eras", 821 "field.era", 822 "field.year", 823 "field.month", 824 "field.week", 825 "field.weekday", 826 "field.dayperiod", 827 "field.hour", 828 "timezone.hourFormat", 829 "timezone.gmtFormat", 830 "timezone.gmtZeroFormat", 831 "timezone.regionFormat", 832 "timezone.regionFormat.daylight", 833 "timezone.regionFormat.standard", 834 "field.minute", 835 "field.second", 836 "field.zone", 837 "TimePatterns", 838 "DatePatterns", 839 "DateTimePatterns", 840 "DateTimePatternChars" 841 }; 842 843 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) { 844 Map<String, Object> formatData = new LinkedHashMap<>(); 845 for (CalendarType calendarType : CalendarType.values()) { 846 if (calendarType == CalendarType.GENERIC) { 847 continue; 848 } 849 String prefix = calendarType.keyElementName(); 850 for (String element : FORMAT_DATA_ELEMENTS) { 851 String key = prefix + element; 852 copyIfPresent(map, "java.time." + key, formatData); 853 copyIfPresent(map, key, formatData); 854 } 855 } 856 857 for (String key : map.keySet()) { 858 // Copy available calendar names 859 if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) { 860 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length()); 861 for (CalendarType calendarType : CalendarType.values()) { 862 if (calendarType == CalendarType.GENERIC) { 863 continue; 864 } 865 if (type.equals(calendarType.lname())) { 866 Object value = map.get(key); 867 String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA, 868 CALENDAR_NAME_PREFIX); 869 formatData.put(dataKey, value); 870 String ukey = CALENDAR_NAME_PREFIX + calendarType.uname(); 871 if (!dataKey.equals(ukey)) { 872 formatData.put(ukey, value); 873 } 874 } 875 } 876 } 877 } 878 879 copyIfPresent(map, "DefaultNumberingSystem", formatData); 880 881 @SuppressWarnings("unchecked") 882 List<String> numberingScripts = (List<String>) map.remove("numberingScripts"); 883 if (numberingScripts != null) { 884 for (String script : numberingScripts) { 885 copyIfPresent(map, script + "." + "NumberElements", formatData); 886 } 887 } else { 888 copyIfPresent(map, "NumberElements", formatData); 889 } 890 copyIfPresent(map, "NumberPatterns", formatData); 891 copyIfPresent(map, "short.CompactNumberPatterns", formatData); 892 copyIfPresent(map, "long.CompactNumberPatterns", formatData); 893 894 // put extra number elements for available scripts into formatData, if it is "root" 895 if (id.equals("root")) { 896 handlerNumbering.keySet().stream() 897 .filter(k -> !numberingScripts.contains(k)) 898 .forEach(k -> { 899 String[] ne = (String[])map.get("latn.NumberElements"); 900 String[] neNew = Arrays.copyOf(ne, ne.length); 901 neNew[4] = handlerNumbering.get(k).substring(0, 1); 902 formatData.put(k + ".NumberElements", neNew); 903 }); 904 } 905 return formatData; 906 } 907 908 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) { 909 Object value = src.get(key); 910 if (value != null) { 911 dest.put(key, value); 912 } 913 } 914 915 // --- code below here is adapted from java.util.Properties --- 916 private static final String specialSaveCharsJava = "\""; 917 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!"; 918 919 /* 920 * Converts unicodes to encoded \uxxxx 921 * and writes out any of the characters in specialSaveChars 922 * with a preceding slash 923 */ 924 static String saveConvert(String theString, boolean useJava) { 925 if (theString == null) { 926 return ""; 927 } 928 929 String specialSaveChars; 930 if (useJava) { 931 specialSaveChars = specialSaveCharsJava; 932 } else { 933 specialSaveChars = specialSaveCharsProperties; 934 } 935 boolean escapeSpace = false; 936 937 int len = theString.length(); 938 StringBuilder outBuffer = new StringBuilder(len * 2); 939 Formatter formatter = new Formatter(outBuffer, Locale.ROOT); 940 941 for (int x = 0; x < len; x++) { 942 char aChar = theString.charAt(x); 943 switch (aChar) { 944 case ' ': 945 if (x == 0 || escapeSpace) { 946 outBuffer.append('\\'); 947 } 948 outBuffer.append(' '); 949 break; 950 case '\\': 951 outBuffer.append('\\'); 952 outBuffer.append('\\'); 953 break; 954 case '\t': 955 outBuffer.append('\\'); 956 outBuffer.append('t'); 957 break; 958 case '\n': 959 outBuffer.append('\\'); 960 outBuffer.append('n'); 961 break; 962 case '\r': 963 outBuffer.append('\\'); 964 outBuffer.append('r'); 965 break; 966 case '\f': 967 outBuffer.append('\\'); 968 outBuffer.append('f'); 969 break; 970 default: 971 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) { 972 formatter.format("\\u%04x", (int)aChar); 973 } else { 974 if (specialSaveChars.indexOf(aChar) != -1) { 975 outBuffer.append('\\'); 976 } 977 outBuffer.append(aChar); 978 } 979 } 980 } 981 return outBuffer.toString(); 982 } 983 984 private static String toLanguageTag(String locName) { 985 if (locName.indexOf('_') == -1) { 986 return locName; 987 } 988 String tag = locName.replaceAll("_", "-"); 989 Locale loc = Locale.forLanguageTag(tag); 990 return loc.toLanguageTag(); 991 } 992 993 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) { 994 String likelySubtag = handlerLikelySubtags.get(id); 995 if (likelySubtag != null) { 996 // Remove Script for now 997 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", "")); 998 } 999 } 1000 1001 private static String toLocaleName(String tag) { 1002 if (tag.indexOf('-') == -1) { 1003 return tag; 1004 } 1005 return tag.replaceAll("-", "_"); 1006 } 1007 1008 private static void setupBaseLocales(String localeList) { 1009 Arrays.stream(localeList.split(",")) 1010 .map(Locale::forLanguageTag) 1011 .map(l -> Control.getControl(Control.FORMAT_DEFAULT) 1012 .getCandidateLocales("", l)) 1013 .forEach(BASE_LOCALES::addAll); 1014 } 1015 1016 // applying parent locale rules to the passed candidates list 1017 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter 1018 private static Map<Locale, Locale> childToParentLocaleMap = null; 1019 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) { 1020 if (Objects.isNull(childToParentLocaleMap)) { 1021 childToParentLocaleMap = new HashMap<>(); 1022 parentLocalesMap.keySet().forEach(key -> { 1023 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-"); 1024 parentLocalesMap.get(key).stream().forEach(child -> { 1025 childToParentLocaleMap.put(Locale.forLanguageTag(child), 1026 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent)); 1027 }); 1028 }); 1029 } 1030 1031 // check irregular parents 1032 for (int i = 0; i < candidates.size(); i++) { 1033 Locale l = candidates.get(i); 1034 Locale p = childToParentLocaleMap.get(l); 1035 if (!l.equals(Locale.ROOT) && 1036 Objects.nonNull(p) && 1037 !candidates.get(i+1).equals(p)) { 1038 List<Locale> applied = candidates.subList(0, i+1); 1039 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p))); 1040 return applied; 1041 } 1042 } 1043 1044 return candidates; 1045 } 1046 1047 private static void generateZoneName() throws Exception { 1048 Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format")); 1049 Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"), 1050 Files.lines(Paths.get(zoneNameTempFile)) 1051 .flatMap(l -> { 1052 if (l.equals("%%%%ZIDMAP%%%%")) { 1053 return zidMapEntry(); 1054 } else if (l.equals("%%%%MZONEMAP%%%%")) { 1055 return handlerMetaZones.mzoneMapEntry(); 1056 } else if (l.equals("%%%%DEPRECATED%%%%")) { 1057 return handlerSupplMeta.deprecatedMap(); 1058 } else if (l.equals("%%%%TZDATALINK%%%%")) { 1059 return tzDataLinkEntry(); 1060 } else { 1061 return Stream.of(l); 1062 } 1063 }) 1064 .collect(Collectors.toList()), 1065 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); 1066 } 1067 1068 private static Stream<String> zidMapEntry() { 1069 return ZoneId.getAvailableZoneIds().stream() 1070 .map(id -> { 1071 String canonId = canonicalTZMap.getOrDefault(id, id); 1072 String meta = handlerMetaZones.get(canonId); 1073 String zone001 = handlerMetaZones.zidMap().get(meta); 1074 return zone001 == null ? "" : 1075 String.format(" \"%s\", \"%s\", \"%s\",", 1076 id, meta, zone001); 1077 }) 1078 .filter(s -> !s.isEmpty()) 1079 .sorted(); 1080 } 1081 1082 private static Stream<String> tzDataLinkEntry() { 1083 try { 1084 return Files.walk(Paths.get(tzDataDir), 1) 1085 .filter(p -> !Files.isDirectory(p)) 1086 .flatMap(CLDRConverter::extractLinks) 1087 .sorted(); 1088 } catch (IOException e) { 1089 throw new UncheckedIOException(e); 1090 } 1091 } 1092 1093 private static Stream<String> extractLinks(Path tzFile) { 1094 try { 1095 return Files.lines(tzFile) 1096 .filter(l -> l.startsWith("Link")) 1097 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*", 1098 " \"$2\", \"$1\",")); 1099 } catch (IOException e) { 1100 throw new UncheckedIOException(e); 1101 } 1102 } 1103 1104 // Generate tzmappings for Windows. The format is: 1105 // 1106 // (Windows Zone Name):(REGION):(Java TZID) 1107 // 1108 // where: 1109 // Windows Zone Name: arbitrary time zone name string used in Windows 1110 // REGION: ISO3166 or UN M.49 code 1111 // Java TZID: Java's time zone ID 1112 // 1113 // Note: the entries are alphabetically sorted, *except* the "world" region 1114 // code, i.e., "001". It should be the last entry for the same windows time 1115 // zone name entries. (cf. TimeZone_md.c) 1116 private static void generateWindowsTZMappings() throws Exception { 1117 Files.createDirectories(Paths.get(DESTINATION_DIR, "windows", "conf")); 1118 Files.write(Paths.get(DESTINATION_DIR, "windows", "conf", "tzmappings"), 1119 handlerWinZones.keySet().stream() 1120 .map(k -> k + ":" + handlerWinZones.get(k) + ":") 1121 .sorted(new Comparator<String>() { 1122 public int compare(String t1, String t2) { 1123 String[] s1 = t1.split(":"); 1124 String[] s2 = t2.split(":"); 1125 if (s1[0].equals(s2[0])) { 1126 if (s1[1].equals("001")) { 1127 return 1; 1128 } else if (s2[1].equals("001")) { 1129 return -1; 1130 } else { 1131 return s1[1].compareTo(s2[1]); 1132 } 1133 } else { 1134 return s1[0].compareTo(s2[0]); 1135 } 1136 } 1137 }) 1138 .collect(Collectors.toList()), 1139 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); 1140 } 1141 }