Using MultiForkByKeyProvider for all datagen

provider/cldr/src/cldr_paths.rs

@@ -142,7 +142,6 @@ impl Default for CldrPathsLocal {
 /// let paths = CldrPathsAllInOne {
 ///     cldr_json_root: PathBuf::from("/path/to/cldr-json"),
 ///     locale_subset: "full".to_string(),
-///     uprops_root: Some(PathBuf::from("path/to/uprops")),
 /// };
 ///
 /// assert_eq!(paths.cldr_misc().unwrap(), PathBuf::from("/path/to/cldr-json/cldr-misc-full"))

provider/cldr/src/download/cldr_allinone.rs

@@ -26,8 +26,7 @@ use std::path::PathBuf;
 /// let downloader = CldrAllInOneDownloader::try_new_from_github("38.1.0", "modern")
 ///     .expect("Cache directory not found");
 ///
-/// let paths: Box<dyn CldrPaths> = Box::new(downloader.download(None)
-///     .expect("The data should download successfully"));
+/// let paths: Box<dyn CldrPaths> = Box::new(downloader.download().expect("The data should download successfully"));
 /// ```
 #[derive(Debug)]
 pub struct CldrAllInOneDownloader {

provider/cldr/src/lib.rs

@@ -32,4 +32,78 @@ pub use cldr_paths::CldrPaths;
 pub use cldr_paths::CldrPathsAllInOne;
 pub use cldr_paths::CldrPathsLocal;
 pub use error::Error as CldrError;
-pub use transform::CldrJsonDataProvider;
+
+use icu_provider::fork::by_key::MultiForkByKeyProvider;
+use icu_provider::iter::IterableDynProvider;
+use icu_provider::prelude::*;
+use std::convert::TryFrom;
+use std::path::PathBuf;
+use transform::calendar::japanese::JapaneseErasProvider;
+use transform::datetime::patterns::DatePatternsProvider;
+use transform::datetime::skeletons::DateSkeletonPatternsProvider;
+use transform::datetime::symbols::DateSymbolsProvider;
+use transform::decimal::NumbersProvider;
+#[cfg(feature = "icu_list")]
+use transform::list::ListProvider;
+use transform::locale_canonicalizer::aliases::AliasesProvider;
+use transform::locale_canonicalizer::likely_subtags::LikelySubtagsProvider;
+use transform::plurals::PluralsProvider;
+use transform::time_zones::TimeZonesProvider;
+
+pub fn create_exportable_properties_provider<T: DataMarker>(
+    cldr_paths: &dyn CldrPaths,
+    uprops_root: PathBuf,
+) -> Result<MultiForkByKeyProvider<Box<dyn IterableDynProvider<T>>>, CldrError>
+where
+    AliasesProvider: IterableDynProvider<T>,
+    DateSymbolsProvider: IterableDynProvider<T>,
+    DateSkeletonPatternsProvider: IterableDynProvider<T>,
+    DatePatternsProvider: IterableDynProvider<T>,
+    JapaneseErasProvider: IterableDynProvider<T>,
+    LikelySubtagsProvider: IterableDynProvider<T>,
+    NumbersProvider: IterableDynProvider<T>,
+    PluralsProvider: IterableDynProvider<T>,
+    TimeZonesProvider: IterableDynProvider<T>,
+    ListProvider: IterableDynProvider<T>,
+{
+    #[allow(unused_variables)] // uprops_root is only used if icu_list
+    Ok(MultiForkByKeyProvider {
+        providers: vec![
+            Box::new(AliasesProvider::try_from(cldr_paths)?),
+            Box::new(DateSymbolsProvider::try_from(cldr_paths)?),
+            Box::new(DateSkeletonPatternsProvider::try_from(cldr_paths)?),
+            Box::new(DatePatternsProvider::try_from(cldr_paths)?),
+            Box::new(JapaneseErasProvider::try_from(cldr_paths)?),
+            Box::new(LikelySubtagsProvider::try_from(cldr_paths)?),
+            Box::new(NumbersProvider::try_from(cldr_paths)?),
+            Box::new(PluralsProvider::try_from(cldr_paths)?),
+            Box::new(TimeZonesProvider::try_from(cldr_paths)?),
+            #[cfg(feature = "icu_list")]
+            Box::new(ListProvider::try_from(cldr_paths, uprops_root)?),
+        ],
+    })
+}
+
+pub const ALL_KEYS: [ResourceKey; if cfg!(feature = "icu_list") { 18 } else { 15 }] = [
+    icu_calendar::provider::JapaneseErasV1Marker::KEY,
+    icu_datetime::provider::calendar::DatePatternsV1Marker::KEY,
+    icu_datetime::provider::calendar::DateSkeletonPatternsV1Marker::KEY,
+    icu_datetime::provider::calendar::DateSymbolsV1Marker::KEY,
+    icu_datetime::provider::time_zones::TimeZoneFormatsV1Marker::KEY,
+    icu_datetime::provider::time_zones::ExemplarCitiesV1Marker::KEY,
+    icu_datetime::provider::time_zones::MetaZoneGenericNamesLongV1Marker::KEY,
+    icu_datetime::provider::time_zones::MetaZoneGenericNamesShortV1Marker::KEY,
+    icu_datetime::provider::time_zones::MetaZoneSpecificNamesLongV1Marker::KEY,
+    icu_datetime::provider::time_zones::MetaZoneSpecificNamesShortV1Marker::KEY,
+    icu_decimal::provider::DecimalSymbolsV1Marker::KEY,
+    #[cfg(feature = "icu_list")]
+    icu_list::provider::AndListV1Marker::KEY,
+    #[cfg(feature = "icu_list")]
+    icu_list::provider::OrListV1Marker::KEY,
+    #[cfg(feature = "icu_list")]
+    icu_list::provider::UnitListV1Marker::KEY,
+    icu_locale_canonicalizer::provider::AliasesV1Marker::KEY,
+    icu_locale_canonicalizer::provider::LikelySubtagsV1Marker::KEY,
+    icu_plurals::provider::CardinalV1Marker::KEY,
+    icu_plurals::provider::OrdinalV1Marker::KEY,
+];

provider/cldr/src/transform/list/mod.rs

@@ -177,7 +177,8 @@ mod tests {
     macro_rules! test {
         ($langid:literal, $type:ident, $(($input:expr, $output:literal),)+) => {
             let cldr_paths = crate::cldr_paths::for_test();
-            let provider = ListProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap();
+            let provider = ListProvider::try_from(
+                &cldr_paths as &dyn CldrPaths, icu_testdata::paths::uprops_toml_root()).unwrap();
             let f = ListFormatter::$type(langid!($langid), &provider, ListStyle::Wide).unwrap();
             $(
                 assert_writeable_eq!(f.format($input.iter()), $output);

provider/cldr/src/transform/mod.rs

@@ -6,83 +6,11 @@
 //!
 //! Every ICU4X component should have its own private submodule and then export the types from here.
 
-mod calendar;
-mod datetime;
-mod decimal;
+pub(crate) mod calendar;
+pub(crate) mod datetime;
+pub(crate) mod decimal;
 #[cfg(feature = "icu_list")]
-mod list;
-mod locale_canonicalizer;
-mod plurals;
-mod time_zones;
-
-use crate::error::Error;
-use crate::CldrPaths;
-use icu_provider::fork::by_key::MultiForkByKeyProvider;
-use icu_provider::iter::IterableDynProvider;
-use icu_provider::prelude::*;
-use icu_provider::serde::SerializeMarker;
-use std::convert::TryFrom;
-use std::path::PathBuf;
-
-pub struct CldrJsonDataProvider;
-
-impl CldrJsonDataProvider {
-    pub fn try_new(
-        cldr_paths: &dyn CldrPaths,
-        uprops_root: PathBuf,
-    ) -> Result<MultiForkByKeyProvider<Box<dyn IterableDynProvider<SerializeMarker>>>, Error> {
-        Ok(MultiForkByKeyProvider {
-            providers: vec![
-                Box::new(locale_canonicalizer::aliases::AliasesProvider::try_from(
-                    cldr_paths,
-                )?),
-                Box::new(datetime::symbols::DateSymbolsProvider::try_from(
-                    cldr_paths,
-                )?),
-                Box::new(datetime::skeletons::DateSkeletonPatternsProvider::try_from(
-                    cldr_paths,
-                )?),
-                Box::new(datetime::patterns::DatePatternsProvider::try_from(
-                    cldr_paths,
-                )?),
-                Box::new(calendar::japanese::JapaneseErasProvider::try_from(
-                    cldr_paths,
-                )?),
-                Box::new(
-                    locale_canonicalizer::likely_subtags::LikelySubtagsProvider::try_from(
-                        cldr_paths,
-                    )?,
-                ),
-                Box::new(decimal::NumbersProvider::try_from(cldr_paths)?),
-                Box::new(plurals::PluralsProvider::try_from(cldr_paths)?),
-                Box::new(time_zones::TimeZonesProvider::try_from(cldr_paths)?),
-                #[cfg(feature = "icu_list")]
-                Box::new(list::ListProvider::try_from(cldr_paths, uprops_root)?),
-            ],
-        })
-    }
-
-    pub const ALL_KEYS: [ResourceKey; if cfg!(feature = "icu_list") { 18 } else { 15 }] = [
-        icu_calendar::provider::JapaneseErasV1Marker::KEY,
-        icu_datetime::provider::calendar::DatePatternsV1Marker::KEY,
-        icu_datetime::provider::calendar::DateSkeletonPatternsV1Marker::KEY,
-        icu_datetime::provider::calendar::DateSymbolsV1Marker::KEY,
-        icu_datetime::provider::time_zones::TimeZoneFormatsV1Marker::KEY,
-        icu_datetime::provider::time_zones::ExemplarCitiesV1Marker::KEY,
-        icu_datetime::provider::time_zones::MetaZoneGenericNamesLongV1Marker::KEY,
-        icu_datetime::provider::time_zones::MetaZoneGenericNamesShortV1Marker::KEY,
-        icu_datetime::provider::time_zones::MetaZoneSpecificNamesLongV1Marker::KEY,
-        icu_datetime::provider::time_zones::MetaZoneSpecificNamesShortV1Marker::KEY,
-        icu_decimal::provider::DecimalSymbolsV1Marker::KEY,
-        #[cfg(feature = "icu_list")]
-        icu_list::provider::AndListV1Marker::KEY,
-        #[cfg(feature = "icu_list")]
-        icu_list::provider::OrListV1Marker::KEY,
-        #[cfg(feature = "icu_list")]
-        icu_list::provider::UnitListV1Marker::KEY,
-        icu_locale_canonicalizer::provider::AliasesV1Marker::KEY,
-        icu_locale_canonicalizer::provider::LikelySubtagsV1Marker::KEY,
-        icu_plurals::provider::CardinalV1Marker::KEY,
-        icu_plurals::provider::OrdinalV1Marker::KEY,
-    ];
-}
+pub(crate) mod list;
+pub(crate) mod locale_canonicalizer;
+pub(crate) mod plurals;
+pub(crate) mod time_zones;

provider/uprops/src/lib.rs

@@ -20,13 +20,37 @@
 mod bin_uniset;
 mod enum_codepointtrie;
 mod enum_uniset;
-mod provider;
 mod reader;
 mod script;
 mod uprops_helpers;
 mod uprops_serde;
 
-pub use provider::PropertiesDataProvider;
-
-// Required by icu_provider_cldr::transform::list
+pub use bin_uniset::BinaryPropertyUnicodeSetDataProvider;
+pub use enum_codepointtrie::EnumeratedPropertyCodePointTrieProvider;
 pub use enum_uniset::EnumeratedPropertyUnicodeSetDataProvider;
+pub use script::ScriptWithExtensionsPropertyProvider;
+
+use icu_provider::fork::by_key::MultiForkByKeyProvider;
+use icu_provider::iter::IterableDynProvider;
+use icu_provider::DataMarker;
+use std::path::Path;
+
+pub fn create_exportable_properties_provider<T: DataMarker>(
+    root_dir: &Path,
+) -> eyre::Result<MultiForkByKeyProvider<Box<dyn IterableDynProvider<T>>>>
+where
+    EnumeratedPropertyCodePointTrieProvider: IterableDynProvider<T>,
+    ScriptWithExtensionsPropertyProvider: IterableDynProvider<T>,
+    EnumeratedPropertyUnicodeSetDataProvider: IterableDynProvider<T>,
+    BinaryPropertyUnicodeSetDataProvider: IterableDynProvider<T>,
+{
+    Ok(MultiForkByKeyProvider {
+        providers: vec![
+            Box::new(EnumeratedPropertyCodePointTrieProvider::try_new(root_dir)?),
+            Box::new(ScriptWithExtensionsPropertyProvider::try_new(root_dir)?),
+            Box::new(EnumeratedPropertyUnicodeSetDataProvider::try_new(root_dir)?),
+            // Has to go last as it matches all props/ keys.
+            Box::new(BinaryPropertyUnicodeSetDataProvider::try_new(root_dir)?),
+        ],
+    })
+}

tools/datagen/src/bin/datagen.rs

@@ -5,7 +5,6 @@
 use clap::{App, Arg, ArgGroup, ArgMatches};
 use eyre::WrapErr;
 use icu_locid::LanguageIdentifier;
-use icu_properties::provider::key::{ALL_MAP_KEYS, ALL_SCRIPT_EXTENSIONS_KEYS, ALL_SET_KEYS};
 use icu_provider::export::DataExporter;
 use icu_provider::filter::Filterable;
 use icu_provider::fork::by_key::MultiForkByKeyProvider;
@@ -15,13 +14,11 @@ use icu_provider::prelude::*;
 use icu_provider::serde::SerializeMarker;
 use icu_provider_blob::export::BlobExporter;
 use icu_provider_cldr::download::CldrAllInOneDownloader;
-use icu_provider_cldr::CldrJsonDataProvider;
 use icu_provider_cldr::CldrPathsAllInOne;
 use icu_provider_fs::export::fs_exporter;
 use icu_provider_fs::export::serializers;
 use icu_provider_fs::export::FilesystemExporter;
 use icu_provider_fs::manifest;
-use icu_provider_uprops::PropertiesDataProvider;
 use simple_logger::SimpleLogger;
 use std::borrow::Cow;
 use std::collections::HashSet;
@@ -260,12 +257,12 @@ fn main() -> eyre::Result<()> {
     };
 
     let all_keys = {
-        // TODO: Use central key repository
+        // TODO(#1512): Use central key repository
         let mut v = vec![];
-        v.extend(CldrJsonDataProvider::ALL_KEYS);
-        v.extend(ALL_MAP_KEYS);
-        v.extend(ALL_SCRIPT_EXTENSIONS_KEYS);
-        v.extend(ALL_SET_KEYS);
+        v.extend(icu_provider_cldr::ALL_KEYS);
+        v.extend(icu_properties::provider::key::ALL_MAP_KEYS);
+        v.extend(icu_properties::provider::key::ALL_SCRIPT_EXTENSIONS_KEYS);
+        v.extend(icu_properties::provider::key::ALL_SET_KEYS);
         v
     };
 
@@ -340,14 +337,14 @@ fn main() -> eyre::Result<()> {
 
         Box::new(MultiForkByKeyProvider {
             providers: vec![
-                Box::new(CldrJsonDataProvider::try_new(
+                Box::new(icu_provider_cldr::create_exportable_properties_provider(
                     cldr_paths.as_ref(),
                     uprops_root.clone(),
                 )?) as Box<dyn IterableDynProvider<SerializeMarker>>,
-                Box::new(PropertiesDataProvider::try_new(&uprops_root)?)
-                    as Box<dyn IterableDynProvider<SerializeMarker>>,
-                Box::new(HelloWorldProvider::new_with_placeholder_data())
-                    as Box<dyn IterableDynProvider<SerializeMarker>>,
+                Box::new(icu_provider_uprops::create_exportable_properties_provider(
+                    &uprops_root,
+                )?),
+                Box::new(HelloWorldProvider::new_with_placeholder_data()),
             ],
         })
     };