[Test](tvf) add regression tests for testing orc reader #41606 #42188 (…

…#42120) cherry pick from #42031 #42188 --------- Co-authored-by: Tiewei Fang <[email protected]> Co-authored-by: TieweiFang <[email protected]>
apache · Oct 21, 2024 · 084434e · 084434e
1 parent ac3d64c
commit 084434e
Show file tree

Hide file tree

Showing 17 changed files with 1,435 additions and 10 deletions.
diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
@@ -32,8 +32,6 @@ enum {
 };
 
 namespace doris::vectorized {
-static const int64_t timestamp_threshold = -2177481943;
-static const int64_t timestamp_diff = 343;
 static const int64_t micr_to_nano_second = 1000;
 
 Status DataTypeDateTimeV2SerDe::serialize_column_to_json(const IColumn& column, int start_idx,
@@ -232,14 +230,6 @@ Status DataTypeDateTimeV2SerDe::write_column_to_orc(const std::string& timezone,
             return Status::InternalError("get unix timestamp error.");
         }
 
-        // -2177481943 represent '1900-12-31 23:54:17'
-        // but -2177481944 represent '1900-12-31 23:59:59'
-        // so for timestamp <= -2177481944, we subtract 343 (5min 43s)
-        // Reference: https://www.timeanddate.com/time/change/china/shanghai?year=1900
-        if (timezone == TimezoneUtils::default_time_zone && timestamp < timestamp_threshold) {
-            timestamp -= timestamp_diff;
-        }
-
         cur_batch->data[row_id] = timestamp;
         cur_batch->nanoseconds[row_id] = datetime_val.microsecond() * micr_to_nano_second;
     }

diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out
diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group1_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group1_orc_files.out
@@ -0,0 +1,141 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_0 --
+0200-02-10
+0947-02-16
+1012-02-27
+1014-02-17
+1947-02-11
+2012-02-21
+2014-02-11
+8200-02-11
+
+-- !test_3 --
+2	foo	0.8	1	1969-12-31T16:00
+5	eat	0.8	6	1969-12-31T16:00:20
+13	bar	80.0	2	1969-12-31T16:00:05
+29	cat	8.0	3	1969-12-31T16:00:10
+70	dog	1.8	4	1969-12-31T16:00:15
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+100	zebra	8.0	0	1969-12-31T16:04:10
+
+-- !test_4 --
+105685	almond antique violet chocolate turquoise	Manufacturer#2	Brand#22	MEDIUM ANODIZED COPPER	14	MED CAN	1690.68	ly pending requ
+110592	almond antique salmon chartreuse burlywood	Manufacturer#1	Brand#15	PROMO BURNISHED NICKEL	6	JUMBO PKG	1602.59	 to the furiously
+112398	almond antique metallic orange dim	Manufacturer#3	Brand#32	MEDIUM BURNISHED BRASS	19	JUMBO JAR	1410.39	ole car
+121152	almond antique burnished rose metallic	Manufacturer#1	Brand#14	PROMO PLATED TIN	2	JUMBO BOX	1173.15	e pinto beans h
+121152	almond antique burnished rose metallic	Manufacturer#1	Brand#14	PROMO PLATED TIN	2	JUMBO BOX	1173.15	e pinto beans h
+132666	almond aquamarine rose maroon antique	Manufacturer#2	Brand#24	SMALL POLISHED NICKEL	25	MED BOX	1698.66	even 
+144293	almond antique olive coral navajo	Manufacturer#3	Brand#34	STANDARD POLISHED STEEL	45	JUMBO CAN	1337.29	ag furiously about 
+146985	almond aquamarine midnight light salmon	Manufacturer#2	Brand#23	MEDIUM BURNISHED COPPER	2	SM CASE	2031.98	s cajole caref
+15103	almond aquamarine dodger light gainsboro	Manufacturer#5	Brand#53	ECONOMY BURNISHED STEEL	46	LG PACK	1018.1	packages hinder carefu
+155733	almond antique sky peru orange	Manufacturer#5	Brand#53	SMALL PLATED BRASS	2	WRAP DRUM	1788.73	furiously. bra
+17273	almond antique forest lavender goldenrod	Manufacturer#3	Brand#35	PROMO ANODIZED TIN	14	JUMBO CASE	1190.27	along the
+17927	almond aquamarine yellow dodger mint	Manufacturer#4	Brand#41	ECONOMY BRUSHED COPPER	7	SM PKG	1844.92	ites. eve
+191709	almond antique violet turquoise frosted	Manufacturer#2	Brand#22	ECONOMY POLISHED STEEL	40	MED BOX	1800.7	 haggle
+192697	almond antique blue firebrick mint	Manufacturer#5	Brand#52	MEDIUM BURNISHED TIN	31	LG DRUM	1789.69	ickly ir
+195606	almond aquamarine sandy cyan gainsboro	Manufacturer#2	Brand#25	STANDARD PLATED TIN	18	SM PKG	1701.6	ic de
+33357	almond azure aquamarine papaya violet	Manufacturer#4	Brand#41	STANDARD ANODIZED TIN	12	WRAP CASE	1290.35	reful
+40982	almond antique misty red olive	Manufacturer#3	Brand#32	ECONOMY PLATED COPPER	1	LG PKG	1922.98	c foxes can s
+42669	almond antique medium spring khaki	Manufacturer#5	Brand#51	STANDARD BURNISHED TIN	6	MED CAN	1611.66	sits haggl
+45261	almond aquamarine floral ivory bisque	Manufacturer#4	Brand#42	SMALL PLATED STEEL	27	WRAP CASE	1206.26	careful
+48427	almond antique violet mint lemon	Manufacturer#4	Brand#42	PROMO POLISHED STEEL	39	SM CASE	1375.42	hely ironic i
+49671	almond antique gainsboro frosted violet	Manufacturer#4	Brand#41	SMALL BRUSHED BRASS	10	SM BOX	1620.67	ccounts run quick
+65667	almond aquamarine pink moccasin thistle	Manufacturer#1	Brand#12	LARGE BURNISHED STEEL	42	JUMBO CASE	1632.66	e across the expr
+78486	almond azure blanched chiffon midnight	Manufacturer#5	Brand#52	LARGE BRUSHED BRASS	23	MED BAG	1464.48	hely blith
+85768	almond antique chartreuse lavender yellow	Manufacturer#1	Brand#12	LARGE BRUSHED STEEL	34	SM BAG	1753.76	refull
+86428	almond aquamarine burnished black steel	Manufacturer#1	Brand#12	STANDARD ANODIZED STEEL	28	WRAP BAG	1414.42	arefully 
+90681	almond antique chartreuse khaki white	Manufacturer#3	Brand#31	MEDIUM BURNISHED TIN	17	SM CASE	1671.68	are slyly after the sl
+
diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group2_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group2_orc_files.out
@@ -0,0 +1,23 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_0 --
+row 000000
+row 000001
+row 000002
+row 000003
+row 000004
+row 000005
+row 000006
+row 000007
+row 000008
+row 000009
+
+-- !test_1 --
+1200-01-08
+
+-- !test_2 --
+0
+
+-- !test_3 --
+Alyssa	\N	[3, 9, 15, 20]
+Ben	red	[]
+
diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group3_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group3_orc_files.out
@@ -0,0 +1,106 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_0 --
+42
+
+-- !test_1 --
+-1155099828	0	-8072133231410116475
+-836442134	1	1238145679872042884
+-995758198	2	-7837173574477374810
+-634239373	3	-6265997042966714236
+-830330741	4	1226587808838328746
+1767665612	5	-3483296404361882349
+-234435524	6	-2492777959717761890
+-1237341112	7	-1655110853862828106
+-3666739	8	762716722992837914
+594038518	9	-414822555163282702
+680485277	10	-8438545694061423851
+1990579537	11	1725616428769673202
+1904347123	12	7487194301641871634
+318397575	13	5284947443410977522
+726871265	14	-4971309310269585972
+470886284	15	-1048347571663351969
+-469827848	16	5914087625780064167
+-1935288453	17	-7901283747506674198
+1091416458	18	6617893815478315911
+589802492	19	-715526239534668540
+1433595053	20	7812935714614531047
+1463956372	21	7736151544221720702
+538577990	22	4280956441582620229
+261903418	23	8643666910095216441
+726547892	24	4589536732286802996
+1827934852	25	5220768139842995762
+-2142712056	26	8452212129931893439
+-1979643329	27	9192676079737912910
+1439902652	28	-5485358416046815779
+1509471580	29	-2900537060362111313
+231072361	30	-1166373669027043549
+-697207184	31	-4941710932145227910
+-1897073668	32	-1214632250906731010
+492728889	33	882146759164402004
+441673793	34	-1623585845533786862
+1053599723	35	-7650598155209059322
+1220936946	36	733466437358115738
+1770977837	37	-1075813735198997759
+332258178	38	5380612601496486616
+-1323562223	39	-2783026025528108869
+2127527772	40	4508443200032579318
+1127644012	41	-6689049007808433632
+-1518802603	42	5499312000257318891
+1211783764	43	-4583514533579607671
+673031799	44	2616088513476383289
+1357179832	45	7306204410731777268
+641449848	46	-7300459700312065882
+1678603587	47	-2031057129963086472
+-479585417	48	5657078495382185794
+218433295	49	-4753393794365190618
+-168375338	50	641301706817379655
+-168670764	51	-5560416134812969260
+-688521145	52	-2939353597904605496
+1521405587	53	-7202421235678721139
+280766823	54	1482820802614970407
+2006072305	55	2582426602654761877
+2841986	56	-3096925422848703303
+1823577017	57	3525728051657376574
+86980171	58	6283812381833889592
+-729958587	59	6505103417183468045
+58615730	60	2130961958634360268
+959699894	61	2427093929580813013
+-1125707998	62	-3191632577313116988
+1004001096	63	8703825967927764437
+1139572680	64	6580459837064512104
+-474634457	65	-1099847229116957438
+-140317777	66	2216635117068313445
+492523913	67	-5990170212825458438
+-842003748	68	5108694483378094778
+794552832	69	-6096899967171477161
+-844195667	70	5179278291182829888
+1984499260	71	6237468570917148003
+-1177191130	72	-7377861005267193838
+-1938669641	73	8248065364431718993
+272665329	74	5506944610710795910
+1705319172	75	-6584616164208436602
+-1108396995	76	1345540141198804862
+440543153	77	-85607681367308891
+2032289729	78	-5986629849796103379
+-842603518	79	-1064662868388789428
+-361562994	80	1801059877260196266
+-1174579731	81	-5036539402851796276
+-1831395724	82	8364289038127439850
+-1980899554	83	-702572046954790847
+743792160	84	4229501672312896502
+1422058480	85	-391320938771541039
+1492414283	86	-1809927736944097678
+-1820060267	87	-6396528855664002559
+1375472775	88	1126092692710756057
+-839029264	89	-3470724079771464614
+1119484814	90	87521117139371251
+-365152316	91	3748318374606874591
+-1851648474	92	560370592390608245
+-524654524	93	2263722670392996598
+-1318061333	94	8352976901787855054
+1720264500	95	-5276809390947146025
+503688873	96	1116640964922907260
+-1033291508	97	8338246827928889510
+1306784267	98	-4929504406024123242
+-179756314	99	2571427694455332022
+