From cbe406c4e0e713269f5bc149a6864062156c0e4a Mon Sep 17 00:00:00 2001 From: Madhu Kadiri Date: Tue, 3 Dec 2024 10:16:50 +0000 Subject: [PATCH] DMS and GlueJob files re-aligned - v1 --- ...g4s_emsys_tpims_task_tables_selection.json | 10 +++ .../D_Comments_V2_DDL.sql | 59 ++++++++++++++++++ .../D_Comments_V2_Insert_Into_Select.sql | 31 +++++++++ .../glue-job/glue_data_validation_lib.zip | Bin 6706 -> 0 bytes .../reusable_module/rds_transform_queries.py | 12 ++++ .../D_Comments_V2_DDL.sql | 59 ------------------ .../D_Comments_V2_Insert_Into_Select.sql | 31 --------- 7 files changed, 112 insertions(+), 90 deletions(-) create mode 100644 terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql create mode 100644 terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql delete mode 100644 terraform/environments/electronic-monitoring-data/glue-job/glue_data_validation_lib.zip delete mode 100644 terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql delete mode 100644 terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql diff --git a/terraform/environments/electronic-monitoring-data/dms_g4s_emsys_tpims_task_tables_selection.json b/terraform/environments/electronic-monitoring-data/dms_g4s_emsys_tpims_task_tables_selection.json index 94d11c03ce7..97f36bad40e 100644 --- a/terraform/environments/electronic-monitoring-data/dms_g4s_emsys_tpims_task_tables_selection.json +++ b/terraform/environments/electronic-monitoring-data/dms_g4s_emsys_tpims_task_tables_selection.json @@ -22,6 +22,16 @@ "table-name": "CurfewSegment" }, "rule-action": "exclude" + }, + { + "rule-type": "selection", + "rule-id": "03", + "rule-name": "03_exclude", + "object-locator": { + "schema-name": "dbo", + "table-name": "GPSPositionLatest" + }, + "rule-action": "exclude" } ] } diff --git a/terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql b/terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql new file mode 100644 index 00000000000..53f8db73c44 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql @@ -0,0 +1,59 @@ +-- SET ANSI_NULLS ON +-- GO +-- SET QUOTED_IDENTIFIER ON +-- GO +-- CREATE TABLE [dbo].[D_Comments_V2]( +-- [CommentSID] [int] IDENTITY(1,1) NOT NULL, +-- [VisitID] [int] NULL, +-- [ActivityID] [uniqueidentifier] NULL, +-- [Comments] [varchar](4200) NULL, +-- [CommentType] [varchar](50) NULL +-- ) ON [PRIMARY] +-- GO +-- CREATE CLUSTERED INDEX [PK_D_Comments_V2] ON [dbo].[D_Comments_V2] +-- ( +-- [CommentSID] ASC +-- )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +-- GO +-- SET ANSI_PADDING ON +-- GO +-- CREATE NONCLUSTERED INDEX [I1_D_Comments_V2] ON [dbo].[D_Comments_V2] +-- ( +-- [VisitID] ASC, +-- [CommentType] ASC +-- ) +-- INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +-- GO +-- SET ANSI_PADDING ON +-- GO +-- CREATE NONCLUSTERED INDEX [I2_D_Comments_V2] ON [dbo].[D_Comments_V2] +-- ( +-- [CommentType] ASC +-- ) +-- INCLUDE([CommentSID],[ActivityID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +-- GO +-- CREATE NONCLUSTERED INDEX [I3_D_Comments_V2] ON [dbo].[D_Comments_V2] +-- ( +-- [VisitID] ASC, +-- [ActivityID] ASC +-- )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +-- GO +-- SET ANSI_PADDING ON +-- GO +-- CREATE NONCLUSTERED INDEX [I4_D_Comments_V2] ON [dbo].[D_Comments_V2] +-- ( +-- [ActivityID] ASC, +-- [CommentType] ASC +-- ) +-- INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +-- GO +-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentSID' +-- GO +-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'VisitID' +-- GO +-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'ActivityID' +-- GO +-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Free text comments summarising either a phone call or a visit adding detail that isnt captured else where ' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'Comments' +-- GO +-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'identifies if the comment belongs to either a phone call or a visit' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentType' +-- GO diff --git a/terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql b/terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql new file mode 100644 index 00000000000..2c101c64c9b --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql @@ -0,0 +1,31 @@ +-- SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 ON +-- ; + +-- truncate table [g4s_cap_dw].[dbo].[D_Comments_V2]; + +-- INSERT INTO g4s_cap_dw.dbo.D_Comments_V2 (CommentSID, VisitID, ActivityID, Comments, CommentType) +-- SELECT CommentSID, VisitID, ActivityID, +-- trim(replace(replace(Comments, char(141), ''), char(129), '')) AS Comments, +-- CommentType +-- FROM g4s_cap_dw.dbo.D_Comments +-- ; + +-- SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 OFF; + + +-- -- TESTING QUERIES -- +-- -- +-- -- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments_V2; -- 49695569 +-- -- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments; -- 49695569 +-- -- + + +-- -- +-- -- SELECT 'D_Comments' AS TableName, Comments +-- -- FROM g4s_cap_dw.dbo.D_Comments +-- -- WHERE CommentSID = 26837791 +-- -- UNION +-- -- SELECT 'D_Comments_V2' AS TableName, Comments +-- -- FROM g4s_cap_dw.dbo.D_Comments_V2 +-- -- WHERE CommentSID = 26837791 +-- -- \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/glue-job/glue_data_validation_lib.zip b/terraform/environments/electronic-monitoring-data/glue-job/glue_data_validation_lib.zip deleted file mode 100644 index b3757dd3162529d4941aeb619ed04a7afbb535c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6706 zcmaKR1yCGJ(T(XK~L0i!Sc&?gaM`+}%BpEbgu$xWn<*-Fx-j zU-e!0)J#uRch^*P_e@PcDhh~5gm7rD3bG%4*CCRsr*;#cdr_jb}r^1Gh=6CkgKts^{ZfQ4+hy;o4k5E7FmFhDWsgj3EyqvlY*-&>lhY3fdyY%bJ>pCs9@nk?_+G5B z%~Bty@%>~9(^Lv;KPV&#kx5iE{KbMbdQYGlShK6T?tqV? z7SY*DMKRhu;uQKDZ&+Zl-OO0^mf_sIo~hqXOq|F z&$wVGjzY^cQ6n2~SIZc_=Q|s=&)Dyw-M?j#+c8M`#SK-vr#ZE*jm{4&tZ+CugQ9Hj zbU(2J0*KJvD=;l7thlM5hS%H=w$j8Sp>O7}3erPO!&*r4lirljYuC3L)$4|IV@u!^ ze1ioy4VyL>)ZK7z4TCU@C>IYb>s+3xEGBAv}O@l{?FRy}TR=n6xSZdSN-O*c^6#F={3;S~sFKv~cC8~O->tORGLt-*#*L3s4oXMt$O=tA zx$Q9bwTN+vUFOm^4C={Zqm4i^sQb!2t|S2sZo`OE&g$F?j(J?{h0lP_^9dO_kQo%XN-qKF#>-6qG7+Aw2(KG+CRKs5w_kCT1IeM!X&{|oVX50`%p4zXsD}6!Z;Pw1J7u26DZqf*2@Ua0kojq4+QT-b z#3(-b2!SFNADmC@mg7hIkPaxZ;|$ZE{hPV1a1nAW@OMs`M2{c8DZxGz8e*!c3)UgED>{g{nSwX-GACdR z7Nmyqezg!?4}dyp6EP0=q7{JDr+^YChf`TV}$wF$P`^M7xS|c6?rN}36`#M}yk>82)^U>^43U@7- zfroH!!?`94#Sd1l2kI6g7SJ_`F97g^eH9x6NQIrNtA@U~=0(Y0EK|;p9yU%?{0(Bd z?UIXZ8`Q1(xY#yS6h2HUmoaK|*T{0=DA5ebFk-;YrnIyS7)klbgd4+Qspz{n6AB)qO zq%kIFn8%37vg4V?(zqNAuRIFP2WC{rFOuo9(C|_31rPHbt=HZVUWB1=6>V%?9%Jv) z5Ygxq22{SVGhKcd}O8x)Q~ zM{A5L6HPW3ukjVtaXUB?@Th7`pBv|A8`gt&`R%IsGi7h#%dXA* zOv4-%79;27{Qb2rkf$p*4jgR7#S{SSgfEtfGg9*mNJBd$389qN84u2n$ES;Dbe^2qJmJw70tk&+rSTs=coHNW4qG@J zQ=i{ZtG-cYwgKn7@o0k%Bsw8W_EO^lHleKDAlhwsoQ8C!$Dr)6(4=qxqc{%?<-j>J zHYbegE;&)N`xFlq6`89Km-~3MmWCUx*GrIw>0TQUH*!_n9V{a+Ii0#ENlG0XylxUx zri?JYt(`w^_}dY$NMQYMYELHSnP9ig!m!Y^>?F6_pP_BHtV0ejvqmr559h)h_3AST zYXP^7h_N^QEpr1cI8;3{J@ON`XIvqMCaTO*{+P6W$z%-mPO%+Deu>j84qQS0n*Hx53G16F&t!8B1)%5JY0*Dj%Ncu)$@M&a)f z4GH)&jA`VQFcRZ#S}H^Y&h0N-qSq3wqt1Gn;+w~3Lze?Ra(R%SP$PUl2?BWX+&AmZ zL2xom!Q{wpjL8%wDC%QaJT(`iZ{VSCAp}R#B|@a9-xApINGZ4`UAZS_aar!HYdlf& z^v!sM?qqcW-|Y?EOO__NJF_@jM$Cv>f#~o?I|bSBd?un6h1UXyW3%q*N}eNkep9ua zq&sC2t94nrE|D3#9bD#qJ{_oY&8W3CJ;lL%i6PFpw$OAd#~5VtKPh1H4#3XX)feCQ zt7-odU!;Dl<1$!hJG_zDt)Du~+z=KbLe0rCwo_2z!FH(;NTeJCTOY9}vWRGQB2AiG zi*@`7yxhAf+}-+C<&#vwcteFIOP2=qY*7g)D7F^qbypuR7{m@vXkjM95j;N@Jae!H ziWaG8B}h~cH~fG|?K1ey3CRc&eeZf-#UnolOJ%kt=7iWebpWTSPQLs)R1>)LBeXt( zbd;(qLK~7yeRXW4PwG!-EVM4j&DHs)>Y~|Lb3Bq6aUDxxgnA%cxduj^q}9Keo4eD*Bb9sEbsZQ#ao|5=eHWb~>15*I5<6_jLX; zTo84f4Isg?>@l)z_7gC(n)M;DT%x*ut~NRaIf}jy?`%sdQGTMQx!0RGr$@$`_~T-`7jQwSSXtI9VJ4I_d{~)7i6dlGo5v= za~PG)sgqO2(G+e}bNLj%MH{p2AAyal+TxM(Mml5Lg^72wBjIhKkECgJB1y}Kcm6_E zv*o0Sz+9N@>J!pjQ{V=B`yCREi=5vl&$qcMqC$k^geYo-%QU5hPL13POKY(gS9GSZ zoDvQ*UlZREt&!;h?>FtxS(~^fGQ|&B5;%sy#g&_eGQ_sSw#<1NPgm9x*$ra${R*~; z=l!yV@2lQI6+C17l-d-$3i>s?d6{CGgyja9a(%z$;w40Ni$FsJ``?F48x_do13762 zC0?Xg{+z>KJp43INDeZ=x5EsS42eQVSJP*@41yb(F0p>piQ(Do5~en8AONI06K~90 zHiRlW;iOvz$)`4H??lS~fh3mi7^df}psVpkk0`fUmsH71s&${+8AN@=K-ABUOu*|Z zZJi1JJGh(v2{rb9%ZVxBNmWOg<1a}W(d>oGjJQg{qSMA_nd z5(RGRiSZ~Z=^36naEeQ_!F#?j^x`pXU*I>@hI?+%)+2t*^#plvACnMEMXp)Amw2C* zpx%NH>EDMzQK-t*YiKTZe<87^@@G(^EhIrGPq_ygxAMX^yxaf$`r&K)eQXoF_>-y+ zgHCrf~!A`XrJ>cR) zB=P`e|5=RG5Msf$INtg2zPqH^Ck_@-^#L^s?4pbD@FDK6q;GK0f;~g(ZBSJ*ov*`e zAiiKtqMSbynhhQWU!Go+B$=VE%n4TD?_eRR)r0B)4*K-H(6JY%=62Hf&r1}^Ya6=b z+Zew@yN2K+N^CdLe&)7Q#oS*Hz;;v&ubEEC8p&hV{W4J!NR`0SdG0i7S%2pYF=a|( z)6RaSva{>&a!M(ZuKAEJ%uzA%Q!!n`uB1S`UGD70BJ`JyaU%l1-v7{4fG~j|wj{lj zmzarMdLi(A0fFE7SW7q~9VMaU1i$lJxtg*XeTYFZJJIbO_(Sa`_S})5a@x2Wn>fl= zn`3TyHnq;OYeup;t|Eq|wNFegWitf%3e)-u(Nn+Ytu(sECNvMx4)x3rfGd0JUrYN6 zOz<^`lC+6RreJKkjjbdsDvnkh=UD{P>h%@tL-w0q&SFQKjP=Fbr?p0AIk<=Z+|TZg+~n9Hhy~-=Jek=)&5GYc1Te3 zxMRQfscb(n&i;3l^iEXrJ*faOth$5=f=&SNk%Ne&He2qx6Jmp}gq%)N(% z8duDY{~|)ZmtxT$)vo5!9A}McpJ4%0jQ7hULOP^RLQ|EJG7v`y^^;gn$zs$>DBrp* zIWJOUvZo3-Or(w0Uum`5tRxr2xuzPDhnEviZ+^S9#3Kq^JRy+v5lO#a&av|U08QJYCkb+cxr9USGdoYK!eg_0NblW4%aQ> zeDVmM;rpp*PDD(H(S_$5VHl3iI+u8L3Y)kIZ!+c>-X_*mk#}{Lz8F$HbZTq;U9mEi zH|{zo8%cmW*1Q>rylMp68&uB5$!PG{=tnd0>iSh3$6W7| z#wQnsGpwwfUdhtS%PPB>gIm}E*5(ZthG2WLrQfp(DQXH(r{Zf$If$hO&wxw)(Ub&C zS()quZuh}D3f&M?HQu5aW0O$|v=T11D^ z%%+iiO^Js!-g0L90*pdv{p-&Kg>pym&HMAMTInv$AU=WaQ?;J>wfYFtU5pK;I5ku#*>^!hGC_JRQL{ z+39X!xbih})A_E)Yk~fSv1cVaGKSv&88C*HJ+9v=1&dd*AFTbYFxyyq1Pp7kTmpfu zJ(*Q@a<+)*-j&Bq;5VR<&DWJxcY^u%EQ(AirS&FiDa*IAm4Q(on*vA^)HgBr!f%9r z9R6)Jz6+0i6765Ig zK3-JeFkt^qJjS$z`A#il95YRmoki>O7cmBDoJkvtetKW%ljO|3Oz$boKdE+vL77nr z6(XJHJq^~f$56%B1gt07-fYVaJht)xF;(;=fwjaix#$%)gxDf&L8xOvM(em%y9Tw$3&4akAK< zD*dE7Gx5SFT;t@GgU$@`QR}ZvD^8h>N0}%i<$-{57Pr0g;BFHv9+`FzY!NH#lJwdO zaR`No|IbOj2KlGr4}eyV;?exdyL#UXkr7eI#D5;{dt7A?qmbIlR+i=I}PV z`1&c)Z|1E!m4=!<61usg@kh_s`Wf;I*pwe+!6Vbsj5QgOp#yo$pC#&c(DdT1h$gc2 zcA9jf-BoGZUWgwxI_?kQuTwp!G?ZzRer0KKI4-)v?iu zsssYlhY5=}OhRvmKP;Y*sXfGJ#=}h~PtTTICX|$BnPC@v)Iq|@X#q0lAs(I61U$xe z0gjN`)!$i|$QOB|!^)}v+Q}4=j(KP+)C40vmNtcxBKa_wL%|zKgiLipu6WiT6*Hx} zmIC%c%4ahY*o`{83WF1{`1p3|Mp7XCo^Vd9;-OZ|c4SQ8mX9%C!Pnw@} zP!%zgq}(|UOA$j= zfe4gyYt5xDqwxeAiF(!XVR4T?1b`Jud3NOGGjdWlD6}bX%>K z{QSL?(X#Y{@^8D_pbx*Pm>3SuHvkUqzqh+dU)$XfGbfNU#2D;kVGsEVa&$3=SerY& z_PV_i#O)UWxC5tmgsiz@5du#*zEX!h@J!+mvR-f*OHgHxu&Y*0$DiXgzjB|OQ8C?W z?yIk>_OlPJyDNpr0!-VsZ1JwLa`l!0Nd!_VE#qoXHz&Fv+-gw%PLpK0IZvi?@-4j& zkUa4uR+KvuP0n`E?$KJ;j%U6ns=6;R&@dTc-k`->M zkcc*Lk8LTl=pHV<6w^4M5kwi90%ne{Q|I<<K5I_K%7#QmDZ?zblYTG;$pzvPXsVt^e4aJUYlP@&?fenzpSL=V{Q5h*c_v zBV^6P7p?hAzr+rGy$ijXluYh~Y$^)z=Vk-^xtpBBs=RG&)@~3=PnvTOq<-|}Kb?tK zq&TDnzopCm*>}F9Q|YO*tT^jo%TMrGtOQXUSCKriOlE88NZdz%IM`)qWvF{Uq<8t? zpaXy8s^RS)#fVN~#CPEOTs6HNl1+Sd1rLG7nU6^xqhFR<^*&!6YF4N4