[Prod] Boundary diagram, manual mark goals similar, Worker Transactions, Graceful Shutdown, Remove goals from TR, Support request

.circleci/config.yml

@@ -449,7 +449,7 @@ commands:
             #!/bin/bash
             CONTROL_FILE="/tmp/stop_tail"
             rm -f $CONTROL_FILE
-            
+
             # Start tailing logs
             cf logs tta-automation &
 
@@ -509,7 +509,7 @@ commands:
             # Signal the log monitoring to stop
             CONTROL_FILE="/tmp/stop_tail"
             touch $CONTROL_FILE
-            
+
             # Wait for the log monitoring process to terminate
             sleep 5
 
@@ -551,7 +551,7 @@ parameters:
     type: string
   dev_git_branch: # change to feature branch to test deployment
     description: "Name of github branch that will deploy to dev"
-    default: "al-ttahub-2939-add-fei-root-cause-to-goal-card"
+    default: "TTAHUB-3097/transactions-for-workers-round-2"
     type: string
   sandbox_git_branch:  # change to feature branch to test deployment
     default: "mb/TTAHUB-3198/training-report-alerts"
@@ -799,7 +799,7 @@ jobs:
           name: Run playwright tests
           command: yarn e2e:ci
       - store_artifacts:
-          path: playwright/e2e
+          path: tests/e2e
     resource_class: large
   test_api:
     executor: docker-postgres-executor
@@ -836,7 +836,7 @@ jobs:
           name: Run playwright tests
           command: yarn e2e:api
       - store_artifacts:
-          path: playwright/api
+          path: tests/api
     resource_class: large
   test_utils:
     executor: docker-postgres-executor
@@ -868,7 +868,7 @@ jobs:
           name: Run playwright tests
           command: yarn e2e:utils
       - store_artifacts:
-          path: playwright/utilsTests
+          path: tests/utilsTests
     resource_class: large
   cucumber_test:
     executor: docker-postgres-executor

docs/boundary_diagram.md

@@ -26,10 +26,37 @@ Boundary(aws, "AWS GovCloud") {
         Container(similarity_api, "Similarity API", "Python", "AI application to identify similarity of text")
         Container(clamav, "File scanning API", "ClamAV", "Internal application for scanning user uploads\n\n docker: ajilaag/clamav-rest:20211026")
       }
+      Boundary(auto, "Automation") {
+        Container(auto_prod_backup, "<&layers> Automation (Production backup)", "bash", "Streams database backup directly to s3(backup) in a password protected encrypted zip compressed format") #green
+        Container(auto_prod_restore, "<&layers> Automation (Restore for processing)", "bash", "Streams database backup directly from s3(backup) in a password protected encrypted zip compressed format to processing DB") #green
+        Container(auto_prod_proc, "<&layers> Automation (Processing to remove PII)", "NodeJS", "Processes restored production data to remove PII for use in lower environments") #green
+        Container(auto_proc_backup, "<&layers> Automation (Backup processed DB)", "bash", "Streams processed database backup directly to s3(backup) in a password protected encrypted zip compressed format") #green
+        Container(auto_proc_restore, "<&layers> Automation (Restore processed data to lower environments)", "bash", "Streams processed database backup directly from s3(backup) in a password protected encrypted zip compressed format to lower environment DB") #green
+      }
+      note right of auto
+          At any one time, a max of only one of these configurations is enabled.
+          All connections are unbound when not in use.
+          No routes are configured.
+          Resources (RAM, Disk) are minimized to only what is needed for the singular automation task.
+      end note
       Boundary(cloudgov_services, "Services") {
         ContainerDb(www_db, "PostgreSQL Database", "AWS RDS", "Contains content and configuration for the TTA Hub")
         ContainerDb(www_s3, "AWS S3 bucket", "AWS S3", "Stores static file assets")
         ContainerDb(www_redis, "Redis Database", "AWS Elasticache", "Queue of background jobs to work on")
+
+        ContainerDb(auto_db, "Automation PostgreSQL Database", "AWS RDS", "Used to process production data to remove PII") #green
+        ContainerDb(auto_s3, "Automation AWS S3 bucket", "AWS S3", "Stores production and processed DB backups") #green
+
+        ContainerDb(logs_s3, "Log transfer AWS S3 bucket", "AWS S3", "Stores production DB logs when requested from cloud.gov") #orange
+      }
+      Boundary(cloudgov_services_staging, "Services - Staging") {
+        ContainerDb(staging_db, "PostgreSQL Database", "AWS RDS", "Contains PII-free content and configuration for the TTA Hub testing") #violet
+      }
+      Boundary(cloudgov_services_dev, "Services - Dev") {
+        ContainerDb(dev_db, "PostgreSQL Database", "AWS RDS", "Contains PII-free content and configuration for the TTA Hub testing") #violet
+      }
+      Boundary(cloudgov_services_sandbox, "Services - Sandbox") {
+        ContainerDb(sandbox_db, "PostgreSQL Database", "AWS RDS", "Contains PII-free content and configuration for the TTA Hub testing") #violet
       }
     }
   }
@@ -48,6 +75,23 @@ Boundary(gsa_saas, "SaaS") {
 Boundary(gsa_fed_saas, "FedRAMP-approved SaaS") {
   System_Ext(newrelic, "New Relic", "Continuous Monitoring")
 }
+
+Rel(www_db, auto_prod_backup, "backup", "pg_dump")
+Rel(auto_prod_backup, auto_s3, "store", "aws s3")
+
+Rel(auto_s3, auto_prod_restore, "retrieve", "aws s3")
+Rel(auto_prod_restore, auto_db, "restore", "psql")
+
+BiRel(auto_db, auto_prod_proc, "process", "psql")
+
+Rel(auto_db, auto_proc_backup, "backup", "pg_dump")
+Rel(auto_proc_backup, auto_s3, "store", "aws s3")
+
+Rel(auto_s3, auto_proc_restore, "retrieve", "aws s3")
+Rel(auto_proc_restore, staging_db, "restore", "psql")
+Rel(auto_proc_restore, dev_db, "restore", "psql")
+Rel(auto_proc_restore, sandbox_db, "restore", "psql")
+
 Rel(developer, newrelic, "Manage performance & logging", "https GET/POST/PUT/DELETE (443)")
 Rel(www_app, newrelic, "reports telemetry", "tcp (443)")
 Rel(developer, google_tag_manager, "Configure tags")

docs/logical_data_model.puml

@@ -246,16 +246,6 @@ class Courses{
  deletedAt : timestamp with time zone
 }
 
-class EventReportPilotGoals{
- * id : integer : <generated>
- sessionId : integer : REFERENCES "SessionReportPilots".id
- * eventId : integer : REFERENCES "EventReportPilots".id
- * goalId : integer : REFERENCES "Goals".id
- * grantId : integer : REFERENCES "Grants".id
- * createdAt : timestamp with time zone
- * updatedAt : timestamp with time zone
-}
-
 class EventReportPilotNationalCenterUsers{
  * id : integer : <generated>
  * eventReportPilotId : integer : REFERENCES "EventReportPilots".id
@@ -1106,6 +1096,7 @@ class Users{
 enum enum_Users_flags {
  anv_statistics
  closed_goal_merge_override
+ manual_mark_goals_similar
  quality_assurance_dashboard
  regional_goal_dashboard
  training_reports_dashboard
@@ -1413,20 +1404,6 @@ class ZALDDL{
  session_sig : text
 }
 
-class ZALEventReportPilotGoals{
- * id : bigint : <generated>
- * data_id : bigint
- * dml_as : bigint
- * dml_by : bigint
- * dml_timestamp : timestamp with time zone
- * dml_txid : uuid
- * dml_type : enum
- descriptor_id : integer
- new_row_data : jsonb
- old_row_data : jsonb
- session_sig : text
-}
-
 class ZALEventReportPilotNationalCenterUsers{
  * id : bigint : <generated>
  * data_id : bigint
@@ -2478,7 +2455,6 @@ CollaboratorTypes "1" --[#black,dashed,thickness=2]--{  "n" ObjectiveCollaborato
 CommunicationLogs "1" --[#black,dashed,thickness=2]--{  "n" CommunicationLogFiles : communicationLog, communicationLogFiles
 Courses "1" --[#black,dashed,thickness=2]--{  "n" ActivityReportObjectiveCourses : course, activityReportObjectiveCourses
 Courses "1" --[#black,dashed,thickness=2]--{  "n" Courses : mapsFromCourse, mapsToCourse
-EventReportPilots "1" --[#black,dashed,thickness=2]--{  "n" EventReportPilotGoals : event, eventReportPilotGoals
 EventReportPilots "1" --[#black,dashed,thickness=2]--{  "n" EventReportPilotNationalCenterUsers : event, eventReportPilotNationalCenterUsers
 EventReportPilots "1" --[#black,dashed,thickness=2]--{  "n" SessionReportPilots : sessionReports, event
 Files "1" --[#black,dashed,thickness=2]--{  "n" ActivityReportFiles : file, reportFiles
@@ -2494,7 +2470,6 @@ GoalTemplates "1" --[#black,dashed,thickness=2]--{  "n" GoalTemplateObjectiveTem
 GoalTemplates "1" --[#black,dashed,thickness=2]--{  "n" GoalTemplateResources : goalTemplate, goalTemplateResources
 GoalTemplates "1" --[#black,dashed,thickness=2]--{  "n" Goals : goals, goalTemplate
 Goals "1" --[#black,dashed,thickness=2]--{  "n" ActivityReportGoals : goal, originalGoal, activityReportGoals, reassignedActivityReportGoals
-Goals "1" --[#black,dashed,thickness=2]--{  "n" EventReportPilotGoals : goal, eventReportPilotGoals
 Goals "1" --[#black,dashed,thickness=2]--{  "n" GoalCollaborators : goal, goalCollaborators
 Goals "1" --[#black,dashed,thickness=2]--{  "n" GoalFieldResponses : goal, responses
 Goals "1" --[#black,dashed,thickness=2]--{  "n" GoalResources : goal, goalResources
@@ -2507,7 +2482,6 @@ Goals "1" --[#black,dashed,thickness=2]--{  "n" SimScoreGoalCaches : scoreOne, s
 GrantNumberLinks "1" --[#black,dashed,thickness=2]--{  "n" MonitoringClassSummaries : monitoringClassSummaries, grantNumberLink
 GrantNumberLinks "1" --[#black,dashed,thickness=2]--{  "n" MonitoringReviewGrantees : monitoringReviewGrantees, grantNumberLink
 Grants "1" --[#black,dashed,thickness=2]--{  "n" ActivityRecipients : grant, activityRecipients
-Grants "1" --[#black,dashed,thickness=2]--{  "n" EventReportPilotGoals : grant, eventReportPilotGoals
 Grants "1" --[#black,dashed,thickness=2]--{  "n" Goals : grant, goals
 Grants "1" --[#black,dashed,thickness=2]--{  "n" Grants : oldGrants, grant
 Grants "1" --[#black,dashed,thickness=2]--{  "n" GroupGrants : groupGrants, grant
@@ -2570,7 +2544,6 @@ Roles "1" --[#black,dashed,thickness=2]--{  "n" CollaboratorRoles : role, collab
 Roles "1" --[#black,dashed,thickness=2]--{  "n" RoleTopics : role, roleTopics
 Roles "1" --[#black,dashed,thickness=2]--{  "n" UserRoles : userRoles, role
 Scopes "1" --[#black,dashed,thickness=2]--{  "n" Permissions : scope, permissions
-SessionReportPilots "1" --[#black,dashed,thickness=2]--{  "n" EventReportPilotGoals : session, eventReportPilotGoals
 SessionReportPilots "1" --[#black,dashed,thickness=2]--{  "n" SessionReportPilotFiles : sessionReport, sessionFiles
 SessionReportPilots "1" --[#black,dashed,thickness=2]--{  "n" SessionReportPilotSupportingAttachments : sessionReport, sessionSupportingAttachments
 Topics "1" --[#black,dashed,thickness=2]--{  "n" ActivityReportObjectiveTopics : topic, activityReportObjectiveTopics
@@ -2611,8 +2584,6 @@ CollaboratorTypes "n" }--[#black,dotted,thickness=2]--{ "n" Groups : groups, col
 CollaboratorTypes "n" }--[#black,dotted,thickness=2]--{ "n" Objectives : objectives, collaboratorTypes
 CollaboratorTypes "n" }--[#black,dotted,thickness=2]--{ "n" Users : usersThroughGoalCollaborators, usersThroughGroupCollaborators, usersThroughObjectiveCollaborators, collaboratorTypesThroughGoalCollaborator, collaboratorTypesThroughGroupCollaborator, collaboratorTypesThroughObjectiveCollaborator
 CommunicationLogs "n" }--[#black,dotted,thickness=2]--{ "n" Files : files, logs
-EventReportPilots "n" }--[#black,dotted,thickness=2]--{ "n" Goals : goals, eventReportPilots
-EventReportPilots "n" }--[#black,dotted,thickness=2]--{ "n" Grants : grants, eventReports
 Files "n" }--[#black,dotted,thickness=2]--{ "n" SessionReportPilots : sessions, sessionsWithSupportingAttachments, files, supportingAttachments
 GoalSimilarityGroups "n" }--[#black,dotted,thickness=2]--{ "n" Goals : goals, similarityGroups
 GoalTemplateFieldPrompts "n" }--[#black,dotted,thickness=2]--{ "n" Goals : goals, prompts

email_templates/changes_requested_by_manager/html.pug

@@ -2,7 +2,7 @@ style
   include ../email.css
 p Hello,
 p 
-p #{managerName} requested changed to report #{displayId}. 
+p #{managerName} requested changes to report #{displayId}. 
 if comments
   p #{managerName} provided the following comments:
   blockquote !{comments} 

email_templates/tr_poc_vision_complete/html.pug

@@ -0,0 +1,11 @@
+style
+  include ../email.css
+p Hello,
+p The POC has completed the vision section of Training Report #{displayId}.
+p Access this report in the TTA Hub:
+ul
+  li
+    a(href=reportPath)= displayId
+p Best Regards,
+  br
+  | Your TTA Hub team

email_templates/tr_poc_vision_complete/subject.pug

@@ -0,0 +1 @@
+= `The vision for Training Report ${displayId} has been completed`

frontend/src/components/GoalCards/GoalCard.js

@@ -20,7 +20,6 @@ import ExpanderButton from '../ExpanderButton';
 import './GoalCard.scss';
 import { goalPropTypes } from './constants';
 import colors from '../../colors';
-import SessionObjectiveCard from './SessionObjectiveCard';
 import Tooltip from '../Tooltip';
 import isAdmin, { hasApproveActivityReportInRegion, canEditOrCreateGoals } from '../../permissions';
 import UserContext from '../../UserContext';
@@ -29,34 +28,21 @@ import AppLoadingContext from '../../AppLoadingContext';
 import GoalStatusChangeAlert from './components/GoalStatusChangeAlert';
 import useObjectiveStatusMonitor from '../../hooks/useObjectiveStatusMonitor';
 
-const SESSION_TYPE = 'session';
-
 export const ObjectiveSwitch = ({
   objective,
   objectivesExpanded,
   regionId,
   goalStatus,
   dispatchStatusChange,
-}) => {
-  if (objective.type === SESSION_TYPE) {
-    return (
-      <SessionObjectiveCard
-        objective={objective}
-        objectivesExpanded={objectivesExpanded}
-      />
-    );
-  }
-
-  return (
-    <ObjectiveCard
-      objective={objective}
-      objectivesExpanded={objectivesExpanded}
-      goalStatus={goalStatus}
-      regionId={regionId}
-      dispatchStatusChange={dispatchStatusChange}
-    />
-  );
-};
+}) => (
+  <ObjectiveCard
+    objective={objective}
+    objectivesExpanded={objectivesExpanded}
+    goalStatus={goalStatus}
+    regionId={regionId}
+    dispatchStatusChange={dispatchStatusChange}
+  />
+);
 
 ObjectiveSwitch.propTypes = {
   objective: PropTypes.shape({
@@ -92,7 +78,6 @@ export default function GoalCard({
     objectiveCount,
     reasons,
     objectives,
-    sessionObjectives,
     previousStatus,
     createdVia,
     collaborators,
@@ -103,7 +88,7 @@ export default function GoalCard({
   const { user } = useContext(UserContext);
   const { setIsAppLoading } = useContext(AppLoadingContext);
   const [invalidStatusChangeAttempted, setInvalidStatusChangeAttempted] = useState();
-  const sortedObjectives = [...objectives, ...(sessionObjectives || [])];
+  const sortedObjectives = [...objectives];
   sortedObjectives.sort((a, b) => ((new Date(a.endDate) < new Date(b.endDate)) ? 1 : -1));
   const hasEditButtonPermissions = canEditOrCreateGoals(user, parseInt(regionId, DECIMAL_BASE));
   const {

frontend/src/components/GoalCards/GoalsCardsHeader.js

@@ -12,6 +12,8 @@ import { canEditOrCreateGoals } from '../../permissions';
 import colors from '../../colors';
 import SelectPagination from '../SelectPagination';
 import { similarity } from '../../fetchers/goals';
+import { markSimilarGoals } from '../../fetchers/recipient';
+import { getFeatureFlags } from '../../fetchers/users';
 
 export default function GoalCardsHeader({
   title,
@@ -41,7 +43,9 @@ export default function GoalCardsHeader({
   allSelectedGoalIds,
   goalBuckets,
 }) {
+  const [retrieveSimilarGoals, setRetrieveSimilarGoals] = useState(false);
   const [goalMergeGroups, setGoalMergeGroups] = useState([]);
+  const [hasManualMarkGoalsSimilar, setHasManualMarkGoalsSimilar] = useState(false);
   const history = useHistory();
   const { user } = useContext(UserContext);
   const hasButtonPermissions = canEditOrCreateGoals(user, parseInt(regionId, DECIMAL_BASE));
@@ -69,7 +73,16 @@ export default function GoalCardsHeader({
     if (canMergeGoals) {
       getSimilarGoals();
     }
-  }, [canMergeGoals, recipientId, regionId]);
+  }, [canMergeGoals, recipientId, regionId, retrieveSimilarGoals]);
+
+  useEffect(() => {
+    async function checkFeatureFlags() {
+      const flags = await getFeatureFlags();
+      setHasManualMarkGoalsSimilar(flags.includes('manual_mark_goals_similar'));
+    }
+
+    checkFeatureFlags();
+  }, []);
 
   const showAddNewButton = hasActiveGrants && hasButtonPermissions;
   const onPrint = () => {
@@ -92,6 +105,25 @@ export default function GoalCardsHeader({
     });
   };
 
+  const onMarkSimilarGoals = async () => {
+    let similarGoals = Object.keys(allSelectedGoalIds).filter(
+      (key) => allSelectedGoalIds[key],
+    ).map((key) => parseInt(key, DECIMAL_BASE));
+
+    // If we don't just print the page.
+    if (!similarGoals.length) {
+      similarGoals = pageGoalIds;
+    }
+    // Get all the goals and associated goals from the buckets.
+    similarGoals = goalBuckets.filter(
+      (bucket) => similarGoals.includes(bucket.id),
+    ).map((bucket) => bucket.goalIds).flat();
+
+    await markSimilarGoals(recipientId, similarGoals); // PUT request to mark similar goals
+    selectAllGoalCheckboxSelect({ target: { checked: false } }); // Deselect all goals
+    setRetrieveSimilarGoals(!retrieveSimilarGoals);
+  };
+
   const setSortBy = (e) => {
     const [sortBy, direction] = e.target.value.split('-');
     requestSort(sortBy, direction);
@@ -220,6 +252,17 @@ export default function GoalCardsHeader({
         >
           {`Preview and print ${hasGoalsSelected ? 'selected' : ''}`}
         </Button>
+        { numberOfSelectedGoals > 1
+          && hasManualMarkGoalsSimilar
+          && (
+            <Button
+              unstyled
+              className="display-flex flex-align-center margin-left-3 margin-y-0"
+              onClick={onMarkSimilarGoals}
+            >
+              Mark goals as similar
+            </Button>
+          )}
       </div>
       <div>
         {showRttapaValidation && (