Update cldr-import to use CLDR v21. Review by: jlabanca@google.com git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@10925 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/eclipse/tools/cldr-import/.classpath b/eclipse/tools/cldr-import/.classpath index 927e3d4..14514f2 100644 --- a/eclipse/tools/cldr-import/.classpath +++ b/eclipse/tools/cldr-import/.classpath
@@ -8,7 +8,8 @@ <classpathentry combineaccessrules="false" exported="true" kind="src" path="/gwt-user"/> <classpathentry combineaccessrules="false" kind="src" path="/cldr-tools"/> <classpathentry combineaccessrules="false" kind="src" path="/gwt-dev"/> - <classpathentry kind="var" path="GWT_TOOLS/lib/icu4j/4.4.2/utilities.jar" sourcepath="/GWT_TOOLS/lib/icu4j/4.4.2/icu4jsrc.jar"/> - <classpathentry kind="var" path="GWT_TOOLS/lib/cldr/1.8.1/cldr.jar" /> + <classpathentry kind="var" path="GWT_TOOLS/lib/cldr/21/cldr.jar" /> + <classpathentry kind="var" path="GWT_TOOLS/lib/cldr/21/icu4j.jar" /> + <classpathentry kind="var" path="GWT_TOOLS/lib/cldr/21/utilities.jar" /> <classpathentry kind="output" path="bin"/> </classpath>
diff --git a/tools/cldr-import/build.xml b/tools/cldr-import/build.xml index 958a95e..8de567e 100644 --- a/tools/cldr-import/build.xml +++ b/tools/cldr-import/build.xml
@@ -8,8 +8,8 @@ <property.ensure name="gwt.user.jar" location="${gwt.build.lib}/gwt-user.jar" /> <property.ensure name="gwt.servlet.jar" location="${gwt.build.lib}/gwt-servlet.jar" /> - <property name="ICU" location="${gwt.tools}/lib/icu4j/4.4.2" /> - <property name="CLDR_TOOLS" location="${gwt.tools}/lib/cldr/1.8.1" /> + <property name="CLDR_TOOLS" location="${gwt.tools}/lib/cldr/21" /> + <property name="ICU" location="${CLDR_TOOLS}" /> <property.ensure name="ICU_CORE_PATH" location="${ICU}/icu4j.jar" /> @@ -105,10 +105,12 @@ <target name="gen.temp" depends="build, cldrdata" description="Generate GWT classes/property files in a temporary directory"> + <delete dir="${tmpdir}" quiet="true"/> <mkdir dir="${tmpdir}"/> <java classname="com.google.gwt.tools.cldr.GenerateGwtCldrData" fork="true" maxmemory="2048m" failonerror="true"> <arg line="--sourcedir ${cldr.data} --outdir ${tmpdir}"/> + <sysproperty key="CLDR_DIR" value="${cldr.root}"/> <classpath> <pathelement location="${javac.out}"/> <pathelement location="${gwt.user.jar}"/> @@ -125,6 +127,7 @@ <java classname="com.google.gwt.tools.cldr.GenerateGwtCldrData" fork="true" maxmemory="2048m" failonerror="true"> <arg line="--sourcedir ${cldr.data} --outdir ${gwt.root}"/> + <sysproperty key="CLDR_DIR" value="${cldr.root}"/> <classpath> <pathelement location="${javac.out}"/> <pathelement location="${gwt.user.jar}"/>
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/CurrencyDataProcessor.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/CurrencyDataProcessor.java index 7896b89..a3ea5fd 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/CurrencyDataProcessor.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/CurrencyDataProcessor.java
@@ -18,9 +18,8 @@ import com.google.gwt.i18n.shared.GwtLocale; import org.unicode.cldr.util.CLDRFile; +import org.unicode.cldr.util.Factory; import org.unicode.cldr.util.XPathParts; -import org.unicode.cldr.util.CLDRFile.DraftStatus; -import org.unicode.cldr.util.CLDRFile.Factory; import java.io.File; import java.io.IOException; @@ -39,6 +38,7 @@ private Map<String, Integer> currencyFractions = new HashMap<String, Integer>(); private int defaultCurrencyFraction; + private Map<String, Integer> rounding = new HashMap<String, Integer>(); private Set<String> stillInUse = new HashSet<String>(); @@ -54,20 +54,24 @@ @Override protected void loadData() throws IOException { System.out.println("Loading data for currencies"); + localeData.addVersions(cldrFactory); loadLocaleIndependentCurrencyData(); localeData.addCurrencyEntries("currency", cldrFactory, currencyFractions, - defaultCurrencyFraction, stillInUse); + defaultCurrencyFraction, stillInUse, rounding); } @Override protected void printHeader(PrintWriter pw) { - pw.println("# Do not edit - generated from Unicode CLDR data"); + printPropertiesHeader(pw); + pw.println(); pw.println("#"); pw.println("# The key is an ISO4217 currency code, and the value is of the " + "form:"); - pw.println("# display name|symbol|decimal digits|not-used-flag"); + pw.println("# display name|symbol|decimal digits|not-used-flag|rounding"); pw.println("# If a symbol is not supplied, the currency code will be used"); pw.println("# If # of decimal digits is omitted, 2 is used"); pw.println("# If a currency is not generally used, not-used-flag=1"); + pw.println("# If a currency should be rounded to a multiple of of the least significant"); + pw.println("# digit, rounding will be present"); pw.println("# Trailing empty fields can be omitted"); pw.println(); } @@ -78,6 +82,7 @@ String path = "client/impl/cldr/CurrencyData"; PrintWriter pw = createOutputFile(path + Processor.localeSuffix(locale) + ".properties"); printHeader(pw); + printVersion(pw, locale, "# "); Map<String, String> map = localeData.getEntries("currency", locale); String[] keys = new String[map.size()]; map.keySet().toArray(keys); @@ -93,10 +98,9 @@ } private void loadLocaleIndependentCurrencyData() { - CLDRFile supp = cldrFactory.make("supplementalData", true, DraftStatus.approved); + CLDRFile supp = cldrFactory.getSupplementalData(); - // load the table of default # of decimal places for each currency - currencyFractions = new HashMap<String, Integer>(); + // load the table of default # of decimal places and rounding for each currency defaultCurrencyFraction = 0; XPathParts parts = new XPathParts(); Iterator<String> iterator = supp.iterator("//supplementalData/currencyData/fractions/info"); @@ -109,12 +113,15 @@ } String curCode = attr.get("iso4217"); int digits = Integer.valueOf(attr.get("digits")); - // TODO(jat): make use of the "rounding" attribute, currently only on CHF if ("DEFAULT".equalsIgnoreCase(curCode)) { defaultCurrencyFraction = digits; } else { currencyFractions.put(curCode, digits); } + int roundingDigits = Integer.valueOf(attr.get("rounding")); + if (roundingDigits != 0) { + rounding.put(curCode, roundingDigits); + } } // find which currencies are still in use in some region, everything else
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/DateTimeFormatInfoProcessor.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/DateTimeFormatInfoProcessor.java index c15ac79..92c050f 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/DateTimeFormatInfoProcessor.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/DateTimeFormatInfoProcessor.java
@@ -16,20 +16,21 @@ package com.google.gwt.tools.cldr; import com.google.gwt.codegen.server.StringGenerator; -import com.google.gwt.core.ext.UnableToCompleteException; -import com.google.gwt.i18n.client.DateTimeFormatInfo; -import com.google.gwt.i18n.client.impl.cldr.DateTimeFormatInfoImpl; import com.google.gwt.i18n.rebind.DateTimePatternGenerator; -import com.google.gwt.i18n.rebind.MessageFormatParser; -import com.google.gwt.i18n.rebind.MessageFormatParser.ArgumentChunk; -import com.google.gwt.i18n.rebind.MessageFormatParser.DefaultTemplateChunkVisitor; -import com.google.gwt.i18n.rebind.MessageFormatParser.StringChunk; -import com.google.gwt.i18n.rebind.MessageFormatParser.TemplateChunk; +import com.google.gwt.i18n.server.MessageFormatUtils.ArgumentChunk; +import com.google.gwt.i18n.server.MessageFormatUtils.DefaultTemplateChunkVisitor; +import com.google.gwt.i18n.server.MessageFormatUtils.MessageStyle; +import com.google.gwt.i18n.server.MessageFormatUtils.StringChunk; +import com.google.gwt.i18n.server.MessageFormatUtils.TemplateChunk; +import com.google.gwt.i18n.server.MessageFormatUtils.VisitorAbortException; +import com.google.gwt.i18n.shared.DateTimeFormatInfo; import com.google.gwt.i18n.shared.GwtLocale; +import com.google.gwt.i18n.shared.impl.cldr.DateTimeFormatInfoImpl; -import org.unicode.cldr.util.CLDRFile.Factory; +import org.unicode.cldr.util.Factory; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.text.ParseException; @@ -38,8 +39,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; +import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -204,16 +205,16 @@ final StringBuilder buf = new StringBuilder(); final StringGenerator gen = StringGenerator.create(buf, false); try { - List<TemplateChunk> chunks = MessageFormatParser.parse(value); + List<TemplateChunk> chunks = MessageStyle.MESSAGE_FORMAT.parse(value); for (TemplateChunk chunk : chunks) { chunk.accept(new DefaultTemplateChunkVisitor() { @Override - public void visit(ArgumentChunk argChunk) throws UnableToCompleteException { + public void visit(ArgumentChunk argChunk) { gen.appendStringValuedExpression(args[argChunk.getArgumentNumber()]); } @Override - public void visit(StringChunk stringChunk) throws UnableToCompleteException { + public void visit(StringChunk stringChunk) { gen.appendStringLiteral(stringChunk.getString()); } }); @@ -221,7 +222,7 @@ } catch (ParseException e) { throw new RuntimeException("Unable to parse pattern '" + value + "' for locale " + locale + " key " + category + "/" + key, e); - } catch (UnableToCompleteException e) { + } catch (VisitorAbortException e) { throw new RuntimeException("Unable to parse pattern '" + value + "' for locale " + locale + " key " + category + "/" + key, e); } @@ -382,6 +383,7 @@ @Override protected void loadData() throws IOException { System.out.println("Loading data for date/time formats"); + localeData.addVersions(cldrFactory); localeData.addEntries("predef", cldrFactory, "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/" + "availableFormats", "dateFormatItem", "id"); @@ -411,121 +413,144 @@ loadFormatPatterns(); } + /** + * Write an output file. + * + * @param locale + * @param clientShared "client" or "shared", determines the package names + * being written to + * @throws IOException + * @throws FileNotFoundException + */ + protected void writeOneOutputFile(GwtLocale locale, String clientShared) throws IOException, + FileNotFoundException { + // TODO(jat): make uz_UZ inherit from uz_Cyrl rather than uz, for example + String myClass; + String pathSuffix; + if (locale.isDefault()) { + if ("client".equals(clientShared)) { + // The client default is hand-written code that extends the shared one. + return; + } + myClass = "DefaultDateTimeFormatInfo"; + pathSuffix = "/"; + } else { + myClass = "DateTimeFormatInfoImpl" + localeSuffix(locale); + pathSuffix = "/impl/cldr/"; + } + GwtLocale parent = localeData.inheritsFrom(locale); + PrintWriter pw = createOutputFile(clientShared + pathSuffix + myClass + ".java"); + printHeader(pw); + pw.print("package com.google.gwt.i18n." + clientShared); + // GWT now requires JDK 1.6, so we always generate @Overrides + setOverrides(true); + if (!locale.isDefault()) { + pw.print(".impl.cldr"); + } + pw.println(";"); + pw.println(); + pw.println("// DO NOT EDIT - GENERATED FROM CLDR AND ICU DATA"); + Map<String, String> map = localeData.getEntries("version", locale); + for (Map.Entry<String, String> entry : map.entrySet()) { + pw.println("// " + entry.getKey() + "=" + entry.getValue()); + } + pw.println(); + if (locale.isDefault()) { + pw.println("/**"); + pw.println(" * Default implementation of DateTimeFormatInfo interface, " + + "using values from"); + pw.println(" * the CLDR root locale."); + pw.println(" * <p>"); + pw.println(" * Users who need to create their own DateTimeFormatInfo " + + "implementation are"); + pw.println(" * encouraged to extend this class so their implementation " + + "won't break when "); + pw.println(" * new methods are added."); + pw.println(" */"); + } else { + pw.println("/**"); + pw.println(" * Implementation of DateTimeFormatInfo for the \"" + locale + "\" locale."); + pw.println(" */"); + } + pw.print("public class " + myClass); + if (locale.isDefault()) { + pw.print(" implements " + DateTimeFormatInfo.class.getSimpleName()); + } else { + pw.print(" extends "); + pw.print(DateTimeFormatInfoImpl.class.getSimpleName()); + if (!parent.isDefault()) { + pw.print('_'); + pw.print(parent.getAsString()); + } + } + pw.println(" {"); + + // write AM/PM names + generateStringList(pw, "dayPeriod-abbrev", null, locale, "ampms", "am", "pm"); + + // write standard date formats + generateArgMethod(pw, "date", locale, "dateFormat"); + generateStringMethod(pw, "date", locale, "full", "dateFormatFull"); + generateStringMethod(pw, "date", locale, "long", "dateFormatLong"); + generateStringMethod(pw, "date", locale, "medium", "dateFormatMedium"); + generateStringMethod(pw, "date", locale, "short", "dateFormatShort"); + + // write methods for assembling date/time formats + generateArgMethod(pw, "dateTime", locale, "dateTime", "timePattern", "datePattern"); + generateArgMethodRedirect(pw, "dateTime", locale, "full", "dateTimeFull", "timePattern", + "datePattern"); + generateArgMethodRedirect(pw, "dateTime", locale, "long", "dateTimeLong", "timePattern", + "datePattern"); + generateArgMethodRedirect(pw, "dateTime", locale, "medium", "dateTimeMedium", "timePattern", + "datePattern"); + generateArgMethodRedirect(pw, "dateTime", locale, "short", "dateTimeShort", "timePattern", + "datePattern"); + + // write era names + generateStringList(pw, "era-wide", null, locale, "erasFull", "0", "1"); + generateStringList(pw, "era-abbrev", null, locale, "erasShort", "0", "1"); + + // write firstDayOfTheWeek + generateDayNumber(pw, locale, "firstDay", "firstDayOfTheWeek"); + + // write predefined date/time formats + for (Map.Entry<String, String> entry : FORMAT_BY_METHOD.entrySet()) { + generateFormat(locale, pw, entry.getValue(), entry.getKey()); + } + + // write month names + generateFullStringList(pw, "month", locale, "months", "1", "2", "3", "4", "5", "6", "7", "8", + "9", "10", "11", "12"); + + // write quarter names + generateStringList(pw, "quarter-wide", null, locale, "quartersFull", "1", "2", "3", "4"); + generateStringList(pw, "quarter-abbrev", null, locale, "quartersShort", "1", "2", "3", "4"); + + // write standard time formats + generateArgMethod(pw, "time", locale, "timeFormat"); + generateStringMethod(pw, "time", locale, "full", "timeFormatFull"); + generateStringMethod(pw, "time", locale, "long", "timeFormatLong"); + generateStringMethod(pw, "time", locale, "medium", "timeFormatMedium"); + generateStringMethod(pw, "time", locale, "short", "timeFormatShort"); + + // write weekday names + generateFullStringList(pw, "day", locale, "weekdays", DAYS); + + // write weekend boundaries + generateDayNumber(pw, locale, "weekendEnd", "weekendEnd"); + generateDayNumber(pw, locale, "weekendStart", "weekendStart"); + + pw.println("}"); + pw.close(); + } + @Override protected void writeOutputFiles() throws IOException { - // TODO(jat): make uz_UZ inherit from uz_Cyrl rather than uz, for example System.out.println("Writing output for date/time formats"); for (GwtLocale locale : localeData.getNonEmptyLocales()) { - String myClass; - String path = "client/"; - String pathSuffix = ""; - if (locale.isDefault()) { - myClass = "DefaultDateTimeFormatInfo"; - } else { - myClass = "DateTimeFormatInfoImpl" + localeSuffix(locale); - pathSuffix = "impl/cldr/"; - } - GwtLocale parent = localeData.inheritsFrom(locale); - PrintWriter pw = createOutputFile(path + pathSuffix + myClass + ".java"); - printHeader(pw); - pw.print("package com.google.gwt.i18n.client"); - if (!locale.isDefault()) { - pw.print(".impl.cldr"); - setOverrides(true); - } else { - setOverrides(false); - } - pw.println(";"); - pw.println(); - pw.println("// DO NOT EDIT - GENERATED FROM CLDR AND ICU DATA"); - pw.println(); - if (locale.isDefault()) { - pw.println("/**"); - pw.println(" * Default implementation of DateTimeFormatInfo interface, " - + "using values from"); - pw.println(" * the CLDR root locale."); - pw.println(" * <p>"); - pw.println(" * Users who need to create their own DateTimeFormatInfo " - + "implementation are"); - pw.println(" * encouraged to extend this class so their implementation " - + "won't break when "); - pw.println(" * new methods are added."); - pw.println(" */"); - } else { - pw.println("/**"); - pw.println(" * Implementation of DateTimeFormatInfo for the \"" + locale + "\" locale."); - pw.println(" */"); - } - pw.print("public class " + myClass); - if (locale.isDefault()) { - pw.print(" implements " + DateTimeFormatInfo.class.getSimpleName()); - } else { - pw.print(" extends "); - pw.print(DateTimeFormatInfoImpl.class.getSimpleName()); - if (!parent.isDefault()) { - pw.print('_'); - pw.print(parent.getAsString()); - } - } - pw.println(" {"); - - // write AM/PM names - generateStringList(pw, "dayPeriod-abbrev", null, locale, "ampms", "am", "pm"); - - // write standard date formats - generateArgMethod(pw, "date", locale, "dateFormat"); - generateStringMethod(pw, "date", locale, "full", "dateFormatFull"); - generateStringMethod(pw, "date", locale, "long", "dateFormatLong"); - generateStringMethod(pw, "date", locale, "medium", "dateFormatMedium"); - generateStringMethod(pw, "date", locale, "short", "dateFormatShort"); - - // write methods for assembling date/time formats - generateArgMethod(pw, "dateTime", locale, "dateTime", "timePattern", "datePattern"); - generateArgMethodRedirect(pw, "dateTime", locale, "full", "dateTimeFull", "timePattern", - "datePattern"); - generateArgMethodRedirect(pw, "dateTime", locale, "long", "dateTimeLong", "timePattern", - "datePattern"); - generateArgMethodRedirect(pw, "dateTime", locale, "medium", "dateTimeMedium", "timePattern", - "datePattern"); - generateArgMethodRedirect(pw, "dateTime", locale, "short", "dateTimeShort", "timePattern", - "datePattern"); - - // write era names - generateStringList(pw, "era-wide", null, locale, "erasFull", "0", "1"); - generateStringList(pw, "era-abbrev", null, locale, "erasShort", "0", "1"); - - // write firstDayOfTheWeek - generateDayNumber(pw, locale, "firstDay", "firstDayOfTheWeek"); - - // write predefined date/time formats - for (Map.Entry<String, String> entry : FORMAT_BY_METHOD.entrySet()) { - generateFormat(locale, pw, entry.getValue(), entry.getKey()); - } - - // write month names - generateFullStringList(pw, "month", locale, "months", "1", "2", "3", "4", "5", "6", "7", "8", - "9", "10", "11", "12"); - - // write quarter names - generateStringList(pw, "quarter-wide", null, locale, "quartersFull", "1", "2", "3", "4"); - generateStringList(pw, "quarter-abbrev", null, locale, "quartersShort", "1", "2", "3", "4"); - - // write standard time formats - generateArgMethod(pw, "time", locale, "timeFormat"); - generateStringMethod(pw, "time", locale, "full", "timeFormatFull"); - generateStringMethod(pw, "time", locale, "long", "timeFormatLong"); - generateStringMethod(pw, "time", locale, "medium", "timeFormatMedium"); - generateStringMethod(pw, "time", locale, "short", "timeFormatShort"); - - // write weekday names - generateFullStringList(pw, "day", locale, "weekdays", DAYS); - - // write weekend boundaries - generateDayNumber(pw, locale, "weekendEnd", "weekendEnd"); - generateDayNumber(pw, locale, "weekendStart", "weekendStart"); - - pw.println("}"); - pw.close(); + // TODO(jat): remove client when we no longer need it + writeOneOutputFile(locale, "client"); + writeOneOutputFile(locale, "shared"); } }
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/GenerateGwtCldrData.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/GenerateGwtCldrData.java index 217286b..8ebeb8a 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/GenerateGwtCldrData.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/GenerateGwtCldrData.java
@@ -21,14 +21,15 @@ import com.ibm.icu.dev.tool.UOption; import org.unicode.cldr.util.CldrUtility; -import org.unicode.cldr.util.CLDRFile.Factory; +import org.unicode.cldr.util.Factory; import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -40,42 +41,63 @@ private static final GwtLocaleFactory factory = new GwtLocaleFactoryImpl(); - private static boolean DEBUG = false; - - private static final Class<? extends Processor>[] PROCESSORS; - - static { - PROCESSORS = - new Class[] { - CurrencyDataProcessor.class, DateTimeFormatInfoProcessor.class, - ListFormattingProcessor.class, LocalizedNamesProcessor.class,}; - } + private static final String DEFAULT_PROCESSORS = "CurrencyDataProcessor," + + "DateTimeFormatInfoProcessor,ListFormattingProcessor,LocalizedNamesProcessor"; public static void main(String[] args) throws IOException, SecurityException, NoSuchMethodException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException { System.out.println("Starting to generate from CLDR data (ignore -D lines " + "produced by cldr-tools)"); - UOption[] options = - { - UOption.HELP_H(), UOption.HELP_QUESTION_MARK(), - UOption.SOURCEDIR().setDefault(CldrUtility.MAIN_DIRECTORY), // C:\cvsdata/unicode\cldr\diff\summary - outputDir().setDefault("./"), restrictLocales(),}; + UOption[] options = { + UOption.HELP_H(), UOption.HELP_QUESTION_MARK(), + UOption.SOURCEDIR().setDefault(CldrUtility.MAIN_DIRECTORY), + outputDir().setDefault("./"), + restrictLocales(), + processors().setDefault(DEFAULT_PROCESSORS), + }; UOption.parseArgs(args, options); String sourceDir = options[2].value; // SOURCEDIR String targetDir = options[3].value; // outputDir - DEBUG = options[4].doesOccur; // --restrictLocales + String restrictLocales = options[4].value; // --restrictLocales + String procNames = options[5].value; // processors + + List<Class<? extends Processor>> processorClasses = new ArrayList<Class<? extends Processor>>(); + for (String procName : procNames.split(",")) { + if (!procName.contains(".")) { + procName = Processor.class.getPackage().getName() + "." + procName; + } + Throwable thrown = null; + try { + Class<?> clazz = Class.forName(procName); + processorClasses.add(clazz.asSubclass(Processor.class)); + } catch (ClassNotFoundException e) { + thrown = e; + } catch (ClassCastException e) { + thrown = e; + } + if (thrown != null) { + System.err.println("Ignoring " + procName + " (" + thrown + ")"); + } + } Factory cldrFactory = Factory.make(sourceDir, ".*"); Set<String> locales = cldrFactory.getAvailable(); - if (DEBUG) { - locales = - new HashSet<String>(Arrays.asList(new String[] { - "root", "en", "zh", "zh_Hans", "zh_Hant", "zh_CN", "zh_TW"})); + if (restrictLocales != null) { + Set<String> newLocales = new HashSet<String>(); + newLocales.add("root"); // always include root or things break + for (String locale : restrictLocales.split(",")) { + if (!locales.contains(locale)) { + System.err.println("Ignoring non-existent locale " + locale); + continue; + } + newLocales.add(locale); + } + locales = newLocales; } System.out.println("Processing " + locales.size() + " locales"); File outputDir = new File(targetDir); LocaleData localeData = new LocaleData(factory, locales); - for (Class<? extends Processor> processorClass : PROCESSORS) { + for (Class<? extends Processor> processorClass : processorClasses) { Constructor<? extends Processor> ctor = processorClass.getConstructor(File.class, Factory.class, LocaleData.class); Processor processor = ctor.newInstance(outputDir, cldrFactory, localeData); @@ -88,7 +110,11 @@ return UOption.create("outdir", 'o', UOption.REQUIRES_ARG); } + private static UOption processors() { + return UOption.create("processors", 'p', UOption.REQUIRES_ARG); + } + private static UOption restrictLocales() { - return UOption.create("restrictLocales", 'r', UOption.NO_ARG); + return UOption.create("restrictLocales", 'r', UOption.REQUIRES_ARG); } }
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/ListFormattingProcessor.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/ListFormattingProcessor.java index a37f9ee..81e8662 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/ListFormattingProcessor.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/ListFormattingProcessor.java
@@ -17,7 +17,7 @@ import com.google.gwt.i18n.shared.GwtLocale; -import org.unicode.cldr.util.CLDRFile.Factory; +import org.unicode.cldr.util.Factory; import java.io.File; import java.io.IOException; @@ -41,6 +41,7 @@ @Override protected void loadData() throws IOException { System.out.println("Loading data for list formatting"); + localeData.addVersions(cldrFactory); localeData.addEntries("list", cldrFactory, "//ldml/listPatterns/listPattern", "listPatternPart", "type"); } @@ -52,6 +53,9 @@ for (Map.Entry<String, String> entry : localeData.getEntries("list", locale).entrySet()) { if (pw == null) { pw = createOutputFile("rebind/cldr/ListPatterns_" + locale.getAsString() + ".properties"); + printPropertiesHeader(pw); + pw.println(); + printVersion(pw, locale, "# "); } pw.println(entry.getKey() + "=" + entry.getValue()); }
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/LocaleData.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/LocaleData.java index 7fedd06..b9707f9 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/LocaleData.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/LocaleData.java
@@ -19,7 +19,7 @@ import com.google.gwt.i18n.shared.GwtLocaleFactory; import org.unicode.cldr.util.CLDRFile; -import org.unicode.cldr.util.CLDRFile.Factory; +import org.unicode.cldr.util.Factory; import org.unicode.cldr.util.XPathParts; import java.util.Arrays; @@ -68,6 +68,8 @@ private String pattern; + private int rounding; + private String symbol; public Currency(String code) { @@ -87,7 +89,8 @@ && equalsNullCheck(symbol, other.symbol) && equalsNullCheck(pattern, other.pattern) && equalsNullCheck(decimalSeparator, other.decimalSeparator) && equalsNullCheck(groupingSeparator, other.groupingSeparator) - && decimalDigits == other.decimalDigits && inUse == other.inUse; + && decimalDigits == other.decimalDigits && inUse == other.inUse + && rounding == other.rounding; } public String getCode() { @@ -118,6 +121,10 @@ return pattern; } + public int getRounding() { + return rounding; + } + public String getSymbol() { return symbol; } @@ -126,7 +133,8 @@ public int hashCode() { return code.hashCode() + 17 * hashCodeNullCheck(displayName) + 19 * hashCodeNullCheck(symbol) + 23 * hashCodeNullCheck(pattern) + 29 * hashCodeNullCheck(decimalSeparator) + 31 - * hashCodeNullCheck(groupingSeparator) + 37 * decimalDigits + (inUse ? 41 : 0); + * hashCodeNullCheck(groupingSeparator) + 37 * decimalDigits + (inUse ? 41 : 0) + + 43 * rounding; } /** @@ -160,6 +168,10 @@ this.pattern = pattern; } + public void setRounding(int rounding) { + this.rounding = rounding; + } + public void setSymbol(String symbol) { this.symbol = symbol; } @@ -191,6 +203,7 @@ * Comparator that orders locales based the inheritance depth. */ private class LocaleComparator implements Comparator<GwtLocale> { + @Override public int compare(GwtLocale a, GwtLocale b) { Integer depthA = localeDepth.get(a); Integer depthB = localeDepth.get(b); @@ -353,21 +366,23 @@ * locale-independent data * @param defaultCurrencyFraction * @param stillInUse + * @param rounding */ public void addCurrencyEntries(String category, Factory cldrFactory, - Map<String, Integer> currencyFractions, int defaultCurrencyFraction, Set<String> stillInUse) { + Map<String, Integer> currencyFractions, int defaultCurrencyFraction, Set<String> stillInUse, + Map<String, Integer> rounding) { for (GwtLocale locale : allLocales.keySet()) { // skip the "default" locale for now if (locale.isDefault()) { continue; } addCurrencyEntries(category, locale, cldrFactory, currencyFractions, defaultCurrencyFraction, - stillInUse); + stillInUse, rounding); } // run the "default" locale last, to override inherited entries GwtLocale locale = localeFactory.getDefault(); addCurrencyEntries(category, locale, cldrFactory, currencyFractions, defaultCurrencyFraction, - stillInUse); + stillInUse, rounding); } public void addDateTimeFormatEntries(String group, Factory cldrFactory) { @@ -407,9 +422,9 @@ if (attr == null) { continue; } - String key = attr.get(keyAttribute); String value = cldr.getStringValue(path); boolean draft = parts.containsAttribute("draft"); + String key = keyAttribute != null ? attr.get(keyAttribute) : "value"; if (!draft || !map.containsKey(key)) { map.put(key, value); } @@ -461,7 +476,7 @@ */ public void addTerritoryEntries(String category, Factory cldrFactory, RegionLanguageData regionLanguageData, String prefix, String tag, String keyAttribute) { - CLDRFile supp = cldrFactory.make("supplementalData", true); + CLDRFile supp = cldrFactory.getSupplementalData(); Map<String, String> map = new HashMap<String, String>(); XPathParts parts = new XPathParts(); Iterator<String> iterator = supp.iterator(prefix); @@ -485,49 +500,30 @@ if (regionLanguageData != null) { // find the choice used by most literate speakers of each language // based on region-based preferences. - for (GwtLocale locale : allLocales.keySet()) { - if (locale.getRegion() != null || locale.getLanguage() == null) { - // skip any that have a region or don't have a language + summarizeTerritoryEntries(category, regionLanguageData, tag, map); + } + } + + public void addVersions(Factory cldrFactory) { + for (GwtLocale locale : allLocales.keySet()) { + Map<String, String> map = getMap("version", locale); + CLDRFile cldr = cldrFactory.make(allLocales.get(locale), true); + XPathParts parts = new XPathParts(); + Iterator<String> iterator = cldr.iterator("//ldml/identity"); + while (iterator.hasNext()) { + String path = iterator.next(); + String fullXPath = cldr.getFullXPath(path); + if (fullXPath == null) { + fullXPath = path; + } + parts.set(fullXPath); + Map<String, String> attr = parts.getAttributes(2); + if (attr == null) { continue; } - String language = locale.getAsString(); - Map<String, Double> langMap = new HashMap<String, Double>(); - for (RegionLanguageData.RegionPopulation langData : regionLanguageData.getRegions(language)) { - String region = langData.getRegion(); - GwtLocale regionLocale = localeFactory.fromString(language + "_" + region); - String day = getTerritoryData(regionLocale, map); - if (day != null) { - Double pop = langMap.get(day); - if (pop == null) { - pop = 0.0; - } - pop += langData.getLiteratePopulation(); - langMap.put(day, pop); - } + for (Map.Entry<String, String> entry : attr.entrySet()) { + map.put(entry.getKey(), entry.getValue()); } - double max = 0; - String maxDay = null; - for (Map.Entry<String, Double> entry : langMap.entrySet()) { - if (entry.getValue() > max) { - max = entry.getValue(); - maxDay = entry.getKey(); - } - } - if (maxDay != null) { - addEntry(category, locale, tag, maxDay); - } - } - } - - // map locales to territory data - for (GwtLocale locale : allLocales.keySet()) { - if (getEntry(category, locale, tag) != null) { - // don't override what we set above - continue; - } - String day = getTerritoryData(locale, map); - if (day != null) { - addEntry(category, locale, tag, day); } } } @@ -634,6 +630,24 @@ } /** + * Return a single value, following locale inheritance. + * + * @param category + * @param locale + * @param key + * @return the requested value, or null if not present + */ + public String getInheritedEntry(String category, GwtLocale locale, String key) { + for (GwtLocale search : locale.getCompleteSearchList()) { + String value = getEntry(category, search, key); + if (value != null) { + return value; + } + } + return null; + } + + /** * @return all locales that have some data associated with them. */ public Set<GwtLocale> getNonEmptyLocales() { @@ -649,6 +663,7 @@ } /** + * @param category * @return all locales that have some data associated with them in the * specified category. */ @@ -675,7 +690,9 @@ GwtLocale parent = inheritsFrom.get(locale); while (parent != null && parent != defaultLocale) { for (Map.Entry<MapKey, Map<String, String>> entry : maps.entrySet()) { - if (entry.getKey().getLocale().equals(parent)) { + MapKey key = entry.getKey(); + // Version entries are always present, so ignore them + if (!"version".equals(key.getCategory()) && key.getLocale().equals(parent)) { Map<String, String> map = entry.getValue(); if (!map.isEmpty()) { return parent; @@ -823,6 +840,62 @@ maps.clear(); } + /** + * Summarize values known by territories to bare languages, based on + * populations using a particular value. + * + * @param category + * @param key + * @param values map of region codes to values + */ + public void summarizeTerritoryEntries(String category, RegionLanguageData regionLanguageData, + String key, Map<String, String> values) { + for (GwtLocale locale : allLocales.keySet()) { + if (locale.getRegion() != null || locale.getLanguage() == null) { + // skip any that have a region or don't have a language + continue; + } + String language = locale.getAsString(); + Map<String, Double> valueMap = new HashMap<String, Double>(); + for (RegionLanguageData.RegionPopulation langData : regionLanguageData.getRegions(language)) { + String region = langData.getRegion(); + GwtLocale regionLocale = localeFactory.fromString(language + "_" + region); + String value = getTerritoryData(regionLocale, values); + if (value != null) { + Double pop = valueMap.get(value); + if (pop == null) { + pop = 0.0; + } + pop += langData.getLiteratePopulation(); + valueMap.put(value, pop); + } + } + double max = 0; + String maxValue = null; + for (Map.Entry<String, Double> entry : valueMap.entrySet()) { + if (entry.getValue() > max) { + max = entry.getValue(); + maxValue = entry.getKey(); + } + } + if (maxValue != null) { + addEntry(category, locale, key, maxValue); + } + } + + // map locales to territory data + for (GwtLocale locale : allLocales.keySet()) { + if (getEntry(category, locale, key) != null) { + // don't override what we set above + continue; + } + String value = getTerritoryData(locale, values); + if (value != null) { + addEntry(category, locale, key, value); + } + } + } + private void addAttributeEntries(String category, Factory cldrFactory, String prefix, String tag, String key, String attribute) { for (GwtLocale locale : allLocales.keySet()) { @@ -843,9 +916,11 @@ * locale-independent data * @param defaultCurrencyFraction * @param stillInUse + * @param rounding */ private void addCurrencyEntries(String category, GwtLocale locale, Factory cldrFactory, - Map<String, Integer> currencyFractions, int defaultCurrencyFraction, Set<String> stillInUse) { + Map<String, Integer> currencyFractions, int defaultCurrencyFraction, Set<String> stillInUse, + Map<String, Integer> rounding) { Map<String, String> outputMap = getMap(category, locale); Map<String, String> defaultMap = null; if (!locale.isDefault()) { @@ -857,8 +932,11 @@ Iterator<String> iterator = cldr.iterator("//ldml/numbers/currencies"); while (iterator.hasNext()) { String path = iterator.next(); - path = cldr.getFullXPath(path); - parts.set(path); + String fullPath = cldr.getFullXPath(path); + if (fullPath == null) { + fullPath = path; + } + parts.set(fullPath); Map<String, String> attr = parts.findAttributes("currency"); if (attr == null) { continue; @@ -874,9 +952,13 @@ } currency.setInUse(stillInUse.contains(currencyCode)); tempMap.put(currencyCode, currency); + Integer roundingMult = rounding.get(currencyCode); + if (roundingMult != null) { + currency.setRounding(roundingMult); + } } String field = parts.getElement(4); - String value = cldr.getStringValue(path); + String value = cldr.getStringValue(fullPath); attr = parts.findAttributes(field); if (attr == null) { attr = Collections.emptyMap(); @@ -991,6 +1073,12 @@ } if (!currency.isInUse()) { buf.append(skipped).append("|1"); + skipped = ""; + } else { + skipped += "|"; + } + if (currency.getRounding() != 0) { + buf.append(skipped).append("|").append(currency.getRounding()); } return buf.toString(); } @@ -1022,6 +1110,7 @@ Arrays.sort(keys, new Comparator<MapKey>() { private final Comparator<GwtLocale> depthComparator = new LocaleComparator(); + @Override public int compare(MapKey a, MapKey b) { return depthComparator.compare(a.getLocale(), b.getLocale()); }
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/LocalizedNamesProcessor.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/LocalizedNamesProcessor.java index 02ee2b7..86dd7eb 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/LocalizedNamesProcessor.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/LocalizedNamesProcessor.java
@@ -18,7 +18,7 @@ import com.google.gwt.i18n.shared.GwtLocale; import com.google.gwt.tools.cldr.RegionLanguageData.RegionPopulation; -import org.unicode.cldr.util.CLDRFile.Factory; +import org.unicode.cldr.util.Factory; import java.io.File; import java.io.IOException; @@ -48,6 +48,7 @@ this.key = collator.getCollationKey(value); } + @Override public int compareTo(IndexedName o) { return key.compareTo(o.key); } @@ -144,6 +145,7 @@ @Override protected void loadData() throws IOException { System.out.println("Loading data for localized names"); + localeData.addVersions(cldrFactory); localeData.addEntries("territory", cldrFactory, "//ldml/localeDisplayNames/territories", "territory", "type"); localeData.addEntries("language", cldrFactory, "//ldml/localeDisplayNames/languages", @@ -152,10 +154,6 @@ "type"); localeData.addEntries("variant", cldrFactory, "//ldml/localeDisplayNames/variants", "variant", "type"); - localeData.addEntries("localePattern", cldrFactory, "//ldml/localeDisplayNames/localePattern", - "localePattern", "unused"); - localeData.addEntries("localeSeparator", cldrFactory, - "//ldml/localeDisplayNames/localeSeparator", "localeSeparator", "unused"); } @Override @@ -178,19 +176,20 @@ // sort for deterministic output Collections.sort(regionCodesWithNames); if (locale.isDefault()) { - generateDefaultLocale(namesMap, regionCodesWithNames, sortOrder, likelyOrder); + generateDefaultLocale(locale, namesMap, regionCodesWithNames, sortOrder, likelyOrder); } generateLocale(locale, namesMap, regionCodesWithNames, sortOrder, likelyOrder); } } /** + * @param locale * @param namesMap * @param regionCodesWithNames * @param sortOrder * @param likelyOrder */ - private void generateDefaultLocale(Map<String, String> namesMap, + private void generateDefaultLocale(GwtLocale locale, Map<String, String> namesMap, List<String> regionCodesWithNames, String[] sortOrder, String[] likelyOrder) throws IOException { PrintWriter pw = null; @@ -199,7 +198,7 @@ printHeader(pw); pw.println("package com.google.gwt.i18n.client;"); pw.println(); - pw.println("// DO NOT EDIT - GENERATED FROM CLDR DATA"); + printVersion(pw, locale, "// "); pw.println(); pw.println("/**"); pw.println(" * Default LocalizedNames implementation."); @@ -250,7 +249,7 @@ pw.println("import com.google.gwt.core.client.JavaScriptObject;"); pw.println(); } - pw.println("// DO NOT EDIT - GENERATED FROM CLDR DATA"); + printVersion(pw, locale, "// "); pw.println(); pw.println("/**"); pw.println(" * Localized names for the \"" + locale + "\" locale.");
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/Processor.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/Processor.java index 0d351f8..6b243d1 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/Processor.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/Processor.java
@@ -15,9 +15,12 @@ */ package com.google.gwt.tools.cldr; +import com.google.gwt.codegen.server.AbortablePrintWriter; +import com.google.gwt.codegen.server.JavaSourceWriterBuilder; +import com.google.gwt.codegen.server.LoggingCodeGenContext; import com.google.gwt.i18n.shared.GwtLocale; -import org.unicode.cldr.util.CLDRFile.Factory; +import org.unicode.cldr.util.Factory; import java.io.BufferedWriter; import java.io.File; @@ -26,12 +29,42 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.util.Calendar; +import java.util.Map; /** * Base class for CLDR processors that generate GWT i18n resources. */ public abstract class Processor { + /** + * A CodeGenContext implementation that logs to j.u.logging and creates + * output files using {@link Processor#createOutputFile(String, String)}. + */ + protected class ProcessorCodeGenContext extends LoggingCodeGenContext { + @Override + public JavaSourceWriterBuilder addClass(String superPkg, String pkgName, String className) { + String pkgPath = superPkg == null ? pkgName : superPkg + '/' + pkgName; + if (pkgPath.length() > 0) { + pkgPath = pkgPath.replace('.', '/') + '/'; + } + String classPath = className.replace('.', '_'); + String fileName = pkgPath + classPath + ".java"; + try { + PrintWriter pw = createOutputFile("", fileName); + AbortablePrintWriter apw = new AbortablePrintWriter(pw); + printHeader(apw); + return new JavaSourceWriterBuilder(apw, pkgName, className); + } catch (FileNotFoundException e) { + error("Unable to create " + fileName, e); + return null; + } catch (IOException e) { + error("Unable to create " + fileName, e); + return null; + } + } + } + protected static final String I18N_PACKAGE_PATH = "user/src/com/google/gwt/i18n/"; protected static <T> String join(String joiner, Iterable<T> objects) { @@ -137,9 +170,8 @@ parent.mkdirs(); } f.createNewFile(); - pw = - new PrintWriter( - new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8")), false); + pw = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), + "UTF-8")), false); return pw; } @@ -190,8 +222,13 @@ protected abstract void loadData() throws IOException; protected void printHeader(PrintWriter pw) { + printJavaHeader(pw); + } + + protected void printJavaHeader(PrintWriter pw) { + int year = Calendar.getInstance().get(Calendar.YEAR); pw.println("/*"); - pw.println(" * Copyright 2010 Google Inc."); + pw.println(" * Copyright " + year + " Google Inc."); pw.println(" * "); pw.println(" * Licensed under the Apache License, Version 2.0 (the " + "\"License\"); you may not"); @@ -211,6 +248,36 @@ pw.println(" */"); } + protected void printPropertiesHeader(PrintWriter pw) { + int year = Calendar.getInstance().get(Calendar.YEAR); + pw.println("# Copyright " + year + " Google Inc."); + pw.println("# "); + pw.println("# Licensed under the Apache License, Version 2.0 (the " + + "\"License\"); you may not"); + pw.println("# use this file except in compliance with the License. You " + + "may obtain a copy of"); + pw.println("# the License at"); + pw.println("# "); + pw.println("# http://www.apache.org/licenses/LICENSE-2.0"); + pw.println("# "); + pw.println("# Unless required by applicable law or agreed to in writing, " + "software"); + pw.println("# distributed under the License is distributed on an \"AS " + + "IS\" BASIS, WITHOUT"); + pw.println("# WARRANTIES OR CONDITIONS OF ANY KIND, either express or " + "implied. See the"); + pw.println("# License for the specific language governing permissions and " + + "limitations under"); + pw.println("# the License."); + } + + protected void printVersion(PrintWriter pw, GwtLocale locale, String prefix) { + pw.println(prefix + "DO NOT EDIT - GENERATED FROM CLDR DATA:"); + Map<String, String> map = localeData.getEntries("version", locale); + for (Map.Entry<String, String> entry : map.entrySet()) { + pw.println(prefix + " " + entry.getKey() + "=" + entry.getValue()); + } + pw.println(); + } + /** * Set whether method definitions should use @Override. *
diff --git a/tools/cldr-import/src/com/google/gwt/tools/cldr/RegionLanguageData.java b/tools/cldr-import/src/com/google/gwt/tools/cldr/RegionLanguageData.java index 2cb6447..fd6b9c8 100644 --- a/tools/cldr-import/src/com/google/gwt/tools/cldr/RegionLanguageData.java +++ b/tools/cldr-import/src/com/google/gwt/tools/cldr/RegionLanguageData.java
@@ -16,9 +16,8 @@ package com.google.gwt.tools.cldr; import org.unicode.cldr.util.CLDRFile; +import org.unicode.cldr.util.Factory; import org.unicode.cldr.util.XPathParts; -import org.unicode.cldr.util.CLDRFile.DraftStatus; -import org.unicode.cldr.util.CLDRFile.Factory; import java.util.Collections; import java.util.HashMap; @@ -47,6 +46,7 @@ this.official = official; } + @Override public int compareTo(LanguagePopulation other) { int c = -Double.compare(literatePopulation, other.literatePopulation); if (c == 0) { @@ -106,6 +106,7 @@ this.official = official; } + @Override public int compareTo(RegionPopulation other) { int c = -Double.compare(literatePopulation, other.literatePopulation); if (c == 0) { @@ -184,7 +185,7 @@ } regionMap = new HashMap<String, SortedSet<LanguagePopulation>>(); languageMap = new HashMap<String, SortedSet<RegionPopulation>>(); - CLDRFile supp = cldrFactory.make("supplementalData", true, DraftStatus.approved); + CLDRFile supp = cldrFactory.getSupplementalData(); XPathParts parts = new XPathParts(); Iterator<String> iterator = supp.iterator("//supplementalData/territoryInfo/territory"); while (iterator.hasNext()) {