From f3995b3fabfe890f89b41b2bbd57aa5586e96472 Mon Sep 17 00:00:00 2001
From: Robert Bastian <robertbastian@users.noreply.github.com>
Date: Wed, 16 Feb 2022 15:25:21 +0100
Subject: [PATCH] MultiForkByKeyProvider for all datagen

---
 provider/blob/src/export/blob_exporter.rs |  58 +++++++++++-----------
 provider/fs/src/export/fs_exporter.rs     |   3 ++
 provider/testdata/data/testdata.postcard  | Bin 1670453 -> 1670477 bytes
 3 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/provider/blob/src/export/blob_exporter.rs b/provider/blob/src/export/blob_exporter.rs
index e2b58976f55..aadf31cec33 100644
--- a/provider/blob/src/export/blob_exporter.rs
+++ b/provider/blob/src/export/blob_exporter.rs
@@ -7,31 +7,34 @@ use icu_provider::export::DataExporter;
 use icu_provider::iter::IterableDynProvider;
 use icu_provider::prelude::*;
 use icu_provider::serde::SerializeMarker;
-use litemap::LiteMap;
 use writeable::Writeable;
 use zerovec::map2d::ZeroMap2d;
+use std::sync::Mutex;
 
 /// A data exporter that writes data to a single-file blob.
 /// See the module-level docs for an example.
 pub struct BlobExporter {
-    resources: LiteMap<(ResourceKeyHash, String), Vec<u8>>,
-    sink: Box<dyn std::io::Write>,
+    resources: Mutex<Vec<(ResourceKeyHash, String, Vec<u8>)>>,
+    sink: Mutex<Option<Box<dyn std::io::Write>>>,
 }
 
+unsafe impl Sync for BlobExporter {}
+unsafe impl Send for BlobExporter {}
+
 impl BlobExporter {
     /// Create a [`BlobExporter`] that writes to the given I/O stream.
     pub fn new_with_sink(sink: Box<dyn std::io::Write>) -> Self {
         Self {
-            resources: LiteMap::new(),
-            sink,
+            resources: Mutex::new(Vec::new()),
+            sink: Mutex::new(Some(sink)),
         }
     }
 }
 
 impl Drop for BlobExporter {
     fn drop(&mut self) {
-        if !self.resources.is_empty() {
-            panic!("Please call close before dropping FilesystemExporter");
+        if self.sink.lock().unwrap().is_some() {
+            panic!("Please call close before dropping BlobExporter");
         }
     }
 }
@@ -52,34 +55,31 @@ impl DataExporter<SerializeMarker> for BlobExporter {
                 .load_payload(key, req)?
                 .take_payload()?
                 .serialize(&mut <dyn erased_serde::Serializer>::erase(&mut serializer))?;
-            self.resources.insert(
-                (
-                    key.get_hash(),
-                    req.options.writeable_to_string().into_owned(),
-                ),
+            self.resources.lock().unwrap().push((
+                key.get_hash(),
+                req.options.writeable_to_string().into_owned(),
                 serializer.output.0,
-            );
+            ));
         }
         Ok(())
     }
 
     fn close(&mut self) -> Result<(), DataError> {
-        // Convert from LiteMap<(String, String), Vec<u8>> to ZeroMap2d<str, str, [u8]>
-        let mut zm: ZeroMap2d<ResourceKeyHash, str, [u8]> =
-            ZeroMap2d::with_capacity(self.resources.len());
-        for ((key, option), bytes) in self.resources.iter() {
-            zm.insert(key, option, bytes);
+        if let Some(mut sink) = self.sink.lock().unwrap().take() {
+            // We are the first `close` and own the sink now
+            let zm = self.resources.lock().unwrap().drain(..).collect::<ZeroMap2d<_, _, _>>();
+            let blob = BlobSchema::V001(BlobSchemaV1 {
+                resources: zm.as_borrowed(),
+            });
+            log::info!("Serializing blob to output stream...");
+            let mut serializer = postcard::Serializer {
+                output: postcard::flavors::AllocVec(Vec::new()),
+            };
+            serde::Serialize::serialize(&blob, &mut serializer)?;
+            sink.write_all(&serializer.output.0)?;
+            Ok(())
+        } else {
+            Err(DataError::custom("Close was called twice"))
         }
-        let blob = BlobSchema::V001(BlobSchemaV1 {
-            resources: zm.as_borrowed(),
-        });
-        log::info!("Serializing blob to output stream...");
-        let mut serializer = postcard::Serializer {
-            output: postcard::flavors::AllocVec(Vec::new()),
-        };
-        serde::Serialize::serialize(&blob, &mut serializer)?;
-        self.sink.write_all(&serializer.output.0)?;
-        self.resources.clear();
-        Ok(())
     }
 }
diff --git a/provider/fs/src/export/fs_exporter.rs b/provider/fs/src/export/fs_exporter.rs
index 0eeea8277dd..b3c3b9f6d86 100644
--- a/provider/fs/src/export/fs_exporter.rs
+++ b/provider/fs/src/export/fs_exporter.rs
@@ -59,6 +59,9 @@ pub struct FilesystemExporter {
     serializer: Box<dyn AbstractSerializer>,
 }
 
+unsafe impl Sync for FilesystemExporter {}
+unsafe impl Send for FilesystemExporter {}
+
 impl DataExporter<SerializeMarker> for FilesystemExporter {
     fn put_key_with_options(
         &mut self,
diff --git a/provider/testdata/data/testdata.postcard b/provider/testdata/data/testdata.postcard
index b754b0620490913ac8f0f04470ccb47603a09d56..43546afd81dcf9221a18228d67e58aede584e5ef 100644
GIT binary patch
delta 2753
zcmW;O2~<?~702=KV^e0{e}-j%85FCWC^is4+)=?TabIx5CGK&F`%cnm2C>l;gSAG5
zM5{5{RO{NR5v{0kOVt`R?m=<ER5UShi5o`kubp!~_uM!C_wK#_|D5yAY_A+~ymG{r
z>2dzEdq;c_!2dttit&;t4P=o8ceR!f1cQQMV+cH`0!>3<vjcoKOf#_F1qa=_R)KyI
z@JS?G^uYLND2jo@Ua(`KM>UvL9oELd4>h26P4Ew{1v~!N^%dw=TQ}5!r3rAQt_r>i
zJvB{164XytffO}Ng>UOYYJC``DOWUvJB^@u6PVu=eolvW&EfMFa6AKUWkN<PnBE#T
zw1M)rdgcu{(heTC#}6GerKEP$q_VaXcsfI_{3vgCfveKDE1Z-@-JnETc83F!oTc4M
zau1jz8)WVns5=(2WW8E!ljBk*H|2?h=0a?)l0+p9q>HSOx^s2JGGLx|`-_@M@<Z)P
zhR9^elLK-?o=fy2txJc;(B_Gbz7iJ5*OK~7r}0vQ$kG6d*})dQ92QL?Ee+$bD2%pf
z=e6h^Yf-bh#jqL{@ii@i;w`?cZ85E`MOvc8y(Ek9G>eM*7PT8%WHhnJZe|hO%Hp+7
z77<-6o^-V+>u%Asr$tt8i(}ck?q~66fQ4^}MapoC*GH<~Xp1Z3E&RVswCFq8!hO=>
zaB={bmIV-K0(m$ikcz8;ly(dvX?qaqzX@jVAA<SEgJ24qgfMDf2#eCHP_m{9%}iC&
z7gnWLWmV1$59P8P@`ti<Sty@;70T>~q1+C2(4&TfU1<)6cW_WK*g^bsho7hd2bo_u
zXu8)yT$uywxr5mePR2EHvZsra{qk6nx;hyqMUvl5Ytpy7lk<|C<)pKWk<Vn0wCLev
zw>0bNq@U!-63Oi4q+TE0EA#vKoh(+eUG_*qwv$w8Dy^lH#0_-PNJhy!vPyQzF*zd(
zraD<E#d1+Tp5|n^Y?MQ1oRrC32{`Mdmb8&6GGEsC75ZOr5>~3l(q8(@6v>y*Wv5)1
zN^xFt^6-%ced1(@ye$iyHd}0)i*jF{NszG_6l;?sLsM<Wh^L-SZSgm-DU!7I$}7g$
z{BEqxe0d?Sj<e|_$7I}in_Kded^5pj-b9;^<gc<pw#zxWDb^&LYLX-^q=$@>nbK&g
z&E4rX!82@XNdsvsZ_cp&Oj9yn*2zveD&=D5Xb11xOq3!ynrHLMM>elXhO{rR`F^2I
z-eNT?vWZ`AvvH+Olhrl{zR;lSZT_=Gb+_60iftC`wwbigX5T@Zup>4@j_XuT`8Cm5
zo7ER=+?Q?MyJmCvdz;wX@{`u?+thtz^YWRFG{BG(Vi@Z%yx}(Ni!|hU4Q=BLA@PRa
zCm0ST8g{1``qei)YGgQ^ZdjUOsMXrA;dMjf4tigAHT3Ukh|4w<`1>1v9b_mQW=I}o
zcz>+nn~8>#Qw{TS459BCB4qIghQG)jIV}|uHrJ3U-DR}o%FIs<?`%?2*(9a%SRywY
zQl!1SD<8{N@$XhRBq#P8Y927ulPTv7bL3a)a>1}d?n>*6S}!%smYhq5k(ageGD8Cy
zCI{r6T)m=tS2gT4!;cbsU5zD6hRGa>D>u|BH-1u-w3I9<f2yWpDh)LxMJBsk%#c90
zi&>H<f0kk?k^S<u{98`RIf)5#@l?{oU38MSBss=Kru?azi%;d4oT=t_aYxDHIxf5k
zE*3R&k<{G9mr|vLi_kVMTFY>mDJ7EJ)<t7!DeL5@^qk;gn#_{NlU%II)%Q{=QFC1E
z`lpN2;@Rz@wj@hqY42CKby3r{bThV>n>7;E+s)7Y+|b`mz8sK&1Kqq_=4RD*Zf+!o
zk(VFF>OEnU+z;b?op8zrg!6D=I4ci@bG<U0)aDV4niau~k_d*rh@dzvlCr!==I)53
z@P|k~leCwS_=hVjmLpOX55JX;GENF)lbn$I5@~s8D19VHie#sJC(ooH*h99%!)SR|
z@?@(Vl*yfSEpH9=FmkwuiE?U$$4})54@tQmTFCO9Ds<n&lvYtx{4EOWTohg_nv|qy
zHVug8S$;IR#nHsxil&D*hClX=q585I)?JRFjqPRdn_hO!^Riz~%ME!ht`EI@nC~TG
zrI$n*ztv0jHZMin{9gX9<fxR&4e7JP%VxP(tOs^_nRCa>8QJr|%L$q6^l?;9OK?pe
z(UQ}^$6EPH>Zki?D_vxeoR<oDD#M!jm?{V5xV*?v_*V-b*Q9lZ54)F-D5)cv(p|ol
za_QXLN0@#jk|k9#tIjD*m{*u!Nq_`OkOWJJRFSF@Dh_dqEoMhyf~)NWt7hSru|99e
hRUi5*H1GBjm$)TN!X-i?#UoJ?EipT8FY$J|`Cq?YwMPH|

delta 2747
zcmW;N2~^c(8prYPA3@;W|9daX1@1+0DFqXb3vLW5YHA9&;I3Ga8k6CIrD$r?<!I`N
zOE{;Yl4j6cW=74_5it=dNhgyUJ*DD~nyED|<Cfym{Lb8SKF@jn=Y9Xr^Stjl_ug+C
za?2ZXw-v<stEzJs1@Qk*sA8-<D@jsh!Bwp_4}{S{u%R*h5)2)+@ox?Yw_&P*buQTJ
z)^j)vi-1*;aMT02(NG)%`@9gS!Wm7Vped}0gG<dIt_AqVw1m%|fd}e3ptUx{!^#9W
z(MAQ^LS`Z?(lm+fRG_^Yc7Sg>LYq#I(-}&-z~!#cIRzH}4z8p^uO6_vCmcwF3%#Ig
zADH$$tnUk_`{|t*;L8DUb0B^m4AhkADwv^3Wlbj7gWxUsQ6>zAI>{OWUrW28P%bH1
zP$}`lbb5&&4)bNbypsnlra-!^^TQT7Ajjpr+!Cu0A_|o}t0YPKONq2ttSgogOLW@1
zY9{gbbSfDuugg5ClrwTqT=%sueSU@R4|Menuvn_4%_H5$ZyH2a23X7uvKZvB=n!dX
z7>~t!(H6bD7K39gyiF~#n_0xRuz1|kVry%QsckIUCR$uevItGKsPAME*Ttf1ip9|G
z7EgLxw9d3}47Rv6#Nv3CMaK~q>7y)a#^`yR#f|Y6;jdUEOtR?td-a=aablW<|3CQ_
zS#MYbAF<fiE`T3O19)UN;`;1H)YmuS`;0(Zl?RgCJcx>=L2SAc#Cs`?$@!u&MajXG
zuMeh^2_d;Sgh34<)J_WJm{j>g*-#qFsxL#CdoPrWp$;;dIoOfxV0^lR`Wy$b1r9%E
zg@bMz9dxXA5OvPM!^aNhMmTvj#mTP0PWH%6X*I-2wiL^Pp<0uyEGPe#_+d`^$wc{3
zc1f4vPIgMC5l)6lzO0aLBb_`qT6<;TXup%?N<Nic(tM1QHqud2rLRQ2<fNVC$PD>N
zc1VrX%Ho+$K9Fs4RF=(h^1f`4svn#jm#fkszGF_hOP(y0kNpb6>zss~R%7Wc!zE7^
z$ZFXxr{uOYI^*Q}eGU4+$yk{ni=8%qw{4Ef&vHi|%jj5}d>PllW}?^~ZQ^83XPaVa
zJJ7~opJ(&t6q|)|U!Iw2Gem0S)oC^t<R>{eU7Pc5-jy<0FQ3YHa$X)v<P4iu(nT^P
zM`lU8nKoAoY@Ud3woQ`skdd=(KU0+~l(n*5_RDDroTC%GWAmC6%l-v6O_$rWmafu!
zh0WPwn|bf6*@rf<AKPpwv+1zTrgEbO-E4Dio9gbg3EyqAc(2XmFKl*yWfOA9X6(1R
zm7{)5bj+r-&L;S*&CCln`!3r={46)LcH5@KeVbn&=}Ij_eq+NVhat^v*d1w@=QZ?*
zGdyi+css#Rm1x-6-Y~3_;YL@(zEs1?G=uMX!}|V)_UVQPgAKz+7^21)7WuOccV9Lf
zpJ<4G%`kh4;b6Yu>zM}s9D_CA;E?5u41bYba#-pmWU--*43t;oEt$32Fk`El%0~HK
zZi@3WLxS{{H)NS?7XMC#D*4xbgSW=;oaCJ_%$K{;zs^u1S0(kN)=wMeO8yzc%V%|R
zNjhi9mP)xMb>~&DUc+86{3sC@)mYLcTjopDB}21I#!rHh6iJuU57ktH8Vu2rAg{Yz
zOqWMe;C3-j{w&+1T=qz{d?knEJ8_4(_^%{~yXY$~N_>oqZnCtAi$6(?)Hd<ExUA%6
zyo;~|7e(D&wCdqvtNbQbUl*w|US>(T#P@U2UQ%SO?3c{xY9hIEeukP9y4WE#@??RF
zPbyth%40EmT|`TB>EKs5deX&(-fl*WbhBKZ$cb@ouF9L)ZZ=E*m)u+_b+h!8o7%)M
z@`}P(wl9ngx5KE859jdsaD5xXS#ltpZyLh!_K0BUA0nu&iXi=I1Z$EbIk+H_=@pUq
ze~ILe68oF-Oog{)yWEm+JS59VDUcGWkW=zV;w%qok|Tw(R`$y!DG2h=%i$qQCP<!?
z$`%<nNY658f`=iKJdBagb3J}8=6diHdT1^0ey&2dJ!JQZ;>e~bt{scgA9XY_Nzs&y
zkLFTQG?RBn6L2k>RBsG(#>Nm>8pEoyF(lev2E6FylO<j@Nu|`vWw|diio7)Z)k|cV
zmr?)p(rc%e!kvCEA1T=(HBu{S6<$7&le_hTOu6o5uWY>Ir9#Fzee94*x!&BzQ_1P<
z(@(08e@K&5A4$?hUXW@zA{Qj1yN~g*Mat!;t_s_F`Zy$Q(|p_=;p2(8M*2vQ6xkz(
zrSm8sA=Cl&3(FFglqFaaAdMtYf~2toONfMuL!4rZsVGZu^_%VvQyJPx2k8~EB%s74
YZV8iciI7O~NR&iNOhrJ6H}lH>0MqBIr2qf`