DMS and GlueJob files re-aligned - v1

terraform/environments/electronic-monitoring-data/dms_g4s_emsys_tpims_task_tables_selection.json

@@ -22,6 +22,16 @@
         "table-name": "CurfewSegment"
       },
       "rule-action": "exclude"
+    },
+    {
+      "rule-type": "selection",
+      "rule-id": "03",
+      "rule-name": "03_exclude",
+      "object-locator": {
+        "schema-name": "dbo",
+        "table-name": "GPSPositionLatest"
+      },
+      "rule-action": "exclude"
     }
   ]
 }

terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql

@@ -0,0 +1,59 @@
+-- SET ANSI_NULLS ON
+-- GO
+-- SET QUOTED_IDENTIFIER ON
+-- GO
+-- CREATE TABLE [dbo].[D_Comments_V2](
+-- 	[CommentSID] [int] IDENTITY(1,1) NOT NULL,
+-- 	[VisitID] [int] NULL,
+-- 	[ActivityID] [uniqueidentifier] NULL,
+-- 	[Comments] [varchar](4200) NULL,
+-- 	[CommentType] [varchar](50) NULL
+-- ) ON [PRIMARY]
+-- GO
+-- CREATE CLUSTERED INDEX [PK_D_Comments_V2] ON [dbo].[D_Comments_V2]
+-- (
+-- 	[CommentSID] ASC
+-- )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+-- GO
+-- SET ANSI_PADDING ON
+-- GO
+-- CREATE NONCLUSTERED INDEX [I1_D_Comments_V2] ON [dbo].[D_Comments_V2]
+-- (
+-- 	[VisitID] ASC,
+-- 	[CommentType] ASC
+-- )
+-- INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+-- GO
+-- SET ANSI_PADDING ON
+-- GO
+-- CREATE NONCLUSTERED INDEX [I2_D_Comments_V2] ON [dbo].[D_Comments_V2]
+-- (
+-- 	[CommentType] ASC
+-- )
+-- INCLUDE([CommentSID],[ActivityID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+-- GO
+-- CREATE NONCLUSTERED INDEX [I3_D_Comments_V2] ON [dbo].[D_Comments_V2]
+-- (
+-- 	[VisitID] ASC,
+-- 	[ActivityID] ASC
+-- )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+-- GO
+-- SET ANSI_PADDING ON
+-- GO
+-- CREATE NONCLUSTERED INDEX [I4_D_Comments_V2] ON [dbo].[D_Comments_V2]
+-- (
+-- 	[ActivityID] ASC,
+-- 	[CommentType] ASC
+-- )
+-- INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+-- GO
+-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentSID'
+-- GO
+-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'VisitID'
+-- GO
+-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'ActivityID'
+-- GO
+-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Free text comments summarising either a phone call or a visit adding detail that isnt captured else where ' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'Comments'
+-- GO
+-- EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'identifies if the comment  belongs to either a phone call or a visit' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentType'
+-- GO

terraform/environments/electronic-monitoring-data/glue-job/Archived/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql

@@ -0,0 +1,31 @@
+-- SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 ON
+-- ;
+
+-- truncate table [g4s_cap_dw].[dbo].[D_Comments_V2];
+
+-- INSERT INTO g4s_cap_dw.dbo.D_Comments_V2 (CommentSID, VisitID, ActivityID, Comments, CommentType)
+--  SELECT CommentSID, VisitID, ActivityID, 
+--         trim(replace(replace(Comments, char(141), ''), char(129), '')) AS Comments, 
+--         CommentType
+--   FROM g4s_cap_dw.dbo.D_Comments
+-- ;
+
+-- SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 OFF;
+
+
+-- -- TESTING QUERIES -- 
+-- --
+-- -- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments_V2; -- 49695569
+-- -- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments; -- 49695569
+-- --
+
+
+-- --
+-- -- SELECT 'D_Comments' AS TableName, Comments
+-- --   FROM g4s_cap_dw.dbo.D_Comments
+-- --  WHERE CommentSID = 26837791
+-- -- UNION
+-- -- SELECT 'D_Comments_V2' AS TableName, Comments
+-- --   FROM g4s_cap_dw.dbo.D_Comments_V2
+-- --  WHERE CommentSID = 26837791
+-- --

terraform/environments/electronic-monitoring-data/glue-job/reusable_module/rds_transform_queries.py

@@ -2,6 +2,16 @@
 class SQLServer_Extract_Transform:
 
     QUERY_STR_DICT = {
+
+    "g4s_cap_dw_dbo_D_Comments": """
+    SELECT [CommentSID]
+        ,[VisitID]
+        ,[ActivityID]
+        ,trim(replace(replace(Comments, char(141), ''), char(129), '')) AS Comments
+        ,[CommentType]
+    FROM [g4s_cap_dw].[dbo].[D_Comments]
+    """.strip(),
+
     "g4s_emsys_tpims_dbo_CurfewSegment": """
     SELECT [CurfewSegmentID]
         ,[CurfewID]
@@ -26,6 +36,7 @@ class SQLServer_Extract_Transform:
         ,[TravelTimeAfter]
     FROM [g4s_emsys_tpims].[dbo].[CurfewSegment]
     """.strip(),
+
     "g4s_emsys_tpims_dbo_GPSPositionLatest": """
     SELECT [GPSPositionID]
         ,[PersonID]
@@ -45,4 +56,5 @@ class SQLServer_Extract_Transform:
         ,[SeparationViolation]
     FROM [g4s_emsys_tpims].[dbo].[GPSPositionLatest]
     """.strip()
+
     }