From 69b1f7ce86e7b593d4748633ed03f73e2b350de4 Mon Sep 17 00:00:00 2001 From: heathdutton Date: Tue, 19 Feb 2019 10:40:48 -0500 Subject: [PATCH] [ENG-619] Corrections to queries and addition of caching. --- Command/HealthCommand.php | 39 +++---- Model/HealthModel.php | 207 +++++++++++++++++++++----------------- 2 files changed, 135 insertions(+), 111 deletions(-) diff --git a/Command/HealthCommand.php b/Command/HealthCommand.php index bf09859..6847218 100644 --- a/Command/HealthCommand.php +++ b/Command/HealthCommand.php @@ -15,6 +15,7 @@ use MauticPlugin\MauticHealthBundle\Model\HealthModel; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\NullOutput; use Symfony\Component\Console\Output\OutputInterface; /** @@ -67,7 +68,6 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output) { - $verbose = $input->getOption('verbose'); // $campaignRebuildDelay = $input->getOption('campaign-rebuild-delay'); $campaignKickoffDelay = $input->getOption('campaign-kickoff-delay'); $campaignScheduledDelay = $input->getOption('campaign-scheduled-delay'); @@ -76,15 +76,16 @@ protected function execute(InputInterface $input, OutputInterface $output) $container = $this->getContainer(); $translator = $container->get('translator'); + if ($quiet) { + $output = new NullOutput(); + } if (!$this->checkRunStatus($input, $output)) { return 0; } /** @var HealthModel $healthModel */ $healthModel = $container->get('mautic.health.model.health'); - if ($verbose) { - $output->writeln(''.$translator->trans('mautic.health.running').''); - } + $output->writeln(''.$translator->trans('mautic.health.running').''); $settings = []; // if ($campaignRebuildDelay) { // $settings['campaign_rebuild_delay'] = $campaignRebuildDelay; @@ -101,26 +102,26 @@ protected function execute(InputInterface $input, OutputInterface $output) if ($settings) { $healthModel->setSettings($settings); } - if ($verbose) { - $output->writeln(''.$translator->trans('mautic.health.kickoff').''); - } - $healthModel->campaignKickoffCheck($output, $verbose); - if ($verbose) { - $output->writeln(''.$translator->trans('mautic.health.scheduled').''); - } - $healthModel->campaignScheduledCheck($output, $verbose); + + $output->writeln(''.$translator->trans('mautic.health.kickoff').''); + $healthModel->campaignKickoffCheck($output); + + $output->writeln(''.$translator->trans('mautic.health.scheduled').''); + $healthModel->campaignScheduledCheck($output); + // @todo - Add negative action path check. // $healthModel->campaignRebuildCheck($output, $verbose); + $healthModel->setCache(); + + $test = $healthModel->getCache(); if (!$quiet) { $healthModel->reportIncidents($output); } - if ($verbose) { - $output->writeln( - ''.$translator->trans( - 'mautic.health.complete' - ).'' - ); - } + $output->writeln( + ''.$translator->trans( + 'mautic.health.complete' + ).'' + ); $this->completeRun(); return 0; diff --git a/Model/HealthModel.php b/Model/HealthModel.php index 2e6d0ef..a6bc26a 100644 --- a/Model/HealthModel.php +++ b/Model/HealthModel.php @@ -17,6 +17,7 @@ use Doctrine\ORM\EntityManager; use Mautic\CampaignBundle\Model\CampaignModel; use Mautic\CampaignBundle\Model\EventModel; +use Mautic\CoreBundle\Helper\CacheStorageHelper; use Mautic\PluginBundle\Helper\IntegrationHelper; use MauticPlugin\MauticHealthBundle\Integration\HealthIntegration; use Symfony\Component\Console\Output\OutputInterface; @@ -30,7 +31,7 @@ class HealthModel protected $em; /** @var array */ - protected $campaigns = []; + protected $delays = []; /** @var array */ protected $incidents; @@ -56,6 +57,9 @@ class HealthModel /** @var array */ protected $publishedEvents = []; + /** @var CacheStorageHelper */ + protected $cache; + /** * HealthModel constructor. * @@ -92,58 +96,53 @@ public function setSettings($settings) /** * @param OutputInterface|null $output - * @param bool $verbose */ - public function campaignKickoffCheck(OutputInterface $output = null, $verbose = false) + public function campaignKickoffCheck(OutputInterface $output = null) { $campaignIds = array_keys($this->getPublishedCampaigns()); if (!$campaignIds) { return; } - $delay = !empty($this->settings['campaign_kickoff_delay']) ? (int) $this->settings['campaign_kickoff_delay'] : 3600; + $limit = !empty($this->settings['campaign_kickoff_delay']) ? (int) $this->settings['campaign_kickoff_delay'] : 3600; $query = $this->slaveQueryBuilder(); $query->select( 'cl.campaign_id AS campaign_id, count(cl.lead_id) AS contact_count, ROUND(AVG(UNIX_TIMESTAMP() - UNIX_TIMESTAMP(cl.date_added))) as avg_delay_s' ); $query->from(MAUTIC_TABLE_PREFIX.'campaign_leads', 'cl'); $query->where('cl.date_added > DATE_ADD(NOW(), INTERVAL -1 HOUR)'); - $query->andWhere('cl.campaign_id IN (:campaigns)'); + $query->andWhere('cl.campaign_id IN ('.implode(',', $campaignIds).')'); // Adding the manually removed check causes an index miss in 2.15.0+ // $query->andWhere('cl.manually_removed IS NOT NULL AND cl.manually_removed = 0'); $query->andWhere( 'NOT EXISTS (SELECT null FROM '.MAUTIC_TABLE_PREFIX.'campaign_lead_event_log e WHERE cl.lead_id = e.lead_id AND e.campaign_id = cl.campaign_id)' ); - $query->setParameter(':campaigns', $campaignIds); $query->groupBy('cl.campaign_id'); $campaigns = $query->execute()->fetchAll(); foreach ($campaigns as $campaign) { - $id = $campaign['campaign_id']; - if (!isset($this->campaigns[$id])) { - $this->campaigns[$id] = []; - } - $this->campaigns[$id]['kickoff'] = $campaign['contact_count']; - if ($output) { - $body = 'Campaign '.$this->getPublishedCampaigns($id)['name']. + $id = $campaign['campaign_id']; + $delay = [ + 'campaign_id' => $id, + 'campaign_name' => $this->getPublishedCampaigns($id), + 'event_id' => null, + 'event_name' => null, + 'type' => 'kickoff', + 'contact_count' => $campaign['contact_count'], + 'avg_delay_s' => $campaign['avg_delay_s'], + 'body' => 'Campaign '.$this->getPublishedCampaigns($id). ' ('.$id.') has '.$campaign['contact_count']. ' contacts (not realtime) awaiting kickoff with an average of '. - $campaign['avg_delay_s'].'s delay.'; - if ($campaign['avg_delay_s'] > $delay) { - $body .= ' (max is '.$delay.')'; - $status = 'error'; - $this->incidents[$id]['kickoff'] = [ - 'contact_count' => $campaign['contact_count'], - 'body' => $body, - ]; - } else { - $status = 'info'; - if (!$verbose) { - continue; - } - } - $output->writeln( - '<'.$status.'>'.$body.'' - ); + $campaign['avg_delay_s'].'s delay.', + ]; + $this->delays[] = $delay; + if ($delay['avg_delay_s'] > $limit) { + $status = 'error'; + $this->incidents[] = $delay; + } else { + $status = 'info'; } + $output->writeln( + '<'.$status.'>'.$delay['body'].'' + ); } } @@ -156,7 +155,7 @@ private function getPublishedCampaigns($campaignId = null) { if (!$this->publishedCampaigns) { foreach ($this->campaignModel->getPublishedCampaigns(true) as $campaign) { - $this->publishedCampaigns[$campaign['id']] = $campaign; + $this->publishedCampaigns[$campaign['id']] = $campaign['name']; } } @@ -167,23 +166,6 @@ private function getPublishedCampaigns($campaignId = null) } } - /** - * Create a DBAL QueryBuilder preferring a slave connection if available. - * - * @return QueryBuilder - */ - private function slaveQueryBuilder() - { - /** @var Connection $connection */ - $connection = $this->em->getConnection(); - if ($connection instanceof MasterSlaveConnection) { - // Prefer a slave connection if available. - $connection->connect('slave'); - } - - return new QueryBuilder($connection); - } - // /** // * Discern the number of leads waiting on mautic:campaign:rebuild. // * This typically means a large segment has been given a campaign. @@ -209,10 +191,10 @@ private function slaveQueryBuilder() // $campaigns = $query->execute()->fetchAll(); // foreach ($campaigns as $campaign) { // $id = $campaign['campaign_id']; - // if (!isset($this->campaigns[$id])) { - // $this->campaigns[$id] = []; + // if (!isset($this->delays[$id])) { + // $this->delays[$id] = []; // } - // $this->campaigns[$id]['rebuilds'] = $campaign['contact_count']; + // $this->delays[$id]['rebuilds'] = $campaign['contact_count']; // if ($output) { // $body = 'Campaign '.$campaign['campaign_name'].' ('.$id.') has '.$campaign['contact_count'].' ('.$delay.') leads queued to enter the campaign from a segment.'; // if ($campaign['contact_count'] > $delay) { @@ -234,17 +216,65 @@ private function slaveQueryBuilder() // } // } + /** + * Create a DBAL QueryBuilder preferring a slave connection if available. + * + * @return QueryBuilder + */ + private function slaveQueryBuilder() + { + /** @var Connection $connection */ + $connection = $this->em->getConnection(); + if ($connection instanceof MasterSlaveConnection) { + // Prefer a slave connection if available. + $connection->connect('slave'); + } + + return new QueryBuilder($connection); + } + + /** + * Get last delays from the cache. + */ + public function getCache() + { + if (!$this->cache) { + $this->cache = new CacheStorageHelper( + CacheStorageHelper::ADAPTOR_DATABASE, + 'MauticHealthBundle', + $this->em->getConnection() + ); + } + + return $this->cache->get('delays'); + } + + /** + * Store current delays in the cache and purge. + */ + public function setCache() + { + if (!$this->cache) { + $this->cache = new CacheStorageHelper( + CacheStorageHelper::ADAPTOR_DATABASE, + 'MauticHealthBundle', + $this->em->getConnection() + ); + } + $this->cache->set('delays', $this->delays, null); + $this->delays = []; + } + /** * @param OutputInterface|null $output - * @param bool $verbose */ - public function campaignScheduledCheck(OutputInterface $output = null, $verbose = false) + public function campaignScheduledCheck(OutputInterface $output = null) { $eventIds = array_keys($this->getPublishedEvents()); if (!$eventIds) { return; } - $delay = !empty($this->settings['campaign_scheduled_delay']) ? (int) $this->settings['campaign_scheduled_delay'] : 3600; + $limit = !empty($this->settings['campaign_scheduled_delay']) ? (int) $this->settings['campaign_scheduled_delay'] : 3600; $query = $this->slaveQueryBuilder(); $query->select( 'el.campaign_id, el.event_id, COUNT(el.lead_id) as contact_count, ROUND(AVG(UNIX_TIMESTAMP() - UNIX_TIMESTAMP(el.trigger_date))) as avg_delay_s' @@ -252,38 +282,34 @@ public function campaignScheduledCheck(OutputInterface $output = null, $verbose $query->from(MAUTIC_TABLE_PREFIX.'campaign_lead_event_log', 'el'); $query->where('el.is_scheduled = 1'); $query->andWhere('el.trigger_date <= NOW()'); - $query->andWhere('el.event_id IN (:eventIds)'); - $query->setParameter(':eventIds', $eventIds); + $query->andWhere('el.event_id IN ('.implode(',', $eventIds).')'); $query->groupBy('el.event_id'); - $campaigns = $query->execute()->fetchAll(); - foreach ($campaigns as $campaign) { - $id = $campaign['campaign_id']; - if (!isset($this->campaigns[$id])) { - $this->campaigns[$id] = []; - } - $this->campaigns[$id]['kickoff'] = $campaign['contact_count']; - if ($output) { - $body = 'Campaign '.$this->getPublishedCampaigns($id)['name']. - ' ('.$id.') has '.$campaign['contact_count']. - ' contacts queued for scheduled events with an average of '. - $campaign['avg_delay_s'].'s delay.'; - if ($campaign['avg_delay_s'] > $delay) { - $body .= ' (max is '.$delay.')'; - $status = 'error'; - $this->incidents[$id]['kickoff'] = [ - 'contact_count' => $campaign['contact_count'], - 'body' => $body, - ]; - } else { - $status = 'info'; - if (!$verbose) { - continue; - } - } - $output->writeln( - '<'.$status.'>'.$body.'' - ); + $events = $query->execute()->fetchAll(); + foreach ($events as $event) { + $id = $event['campaign_id']; + $delay = [ + 'campaign_id' => $id, + 'campaign_name' => $this->getPublishedCampaigns($id), + 'event_id' => $event['event_id'], + 'event_name' => $this->getPublishedEvents($event['event_id']), + 'type' => 'scheduled', + 'contact_count' => $event['contact_count'], + 'avg_delay_s' => $event['avg_delay_s'], + 'body' => 'Campaign '.$this->getPublishedCampaigns($id). + ' ('.$id.') has '.$event['contact_count']. + ' contacts queued for scheduled event '.$event['event_name'].' ('.$event['event_id'].') with an average of '. + $event['avg_delay_s'].'s delay.', + ]; + $this->delays[] = $delay; + if ($delay['avg_delay_s'] > $limit) { + $status = 'error'; + $this->incidents[] = $delay; + } else { + $status = 'info'; } + $output->writeln( + '<'.$status.'>'.$delay['body'].'' + ); } } @@ -299,7 +325,7 @@ private function getPublishedEvents($eventId = null) if ($campaignIds) { foreach ($this->eventModel->getRepository()->getEntities( [ - 'filter' => [ + 'filter' => [ 'force' => [ [ 'column' => 'IDENTITY(e.campaign)', @@ -308,10 +334,9 @@ private function getPublishedEvents($eventId = null) ], ], ], - 'hydration_mode' => 'HYDRATE_ARRAY', ] ) as $event) { - $this->publishedEvents[$event['id']] = $event; + $this->publishedEvents[$event->getId()] = $event->getName(); } } } @@ -343,11 +368,9 @@ public function reportIncidents(OutputInterface $output = null) if ($this->incidents && !empty($this->settings['statuspage_component_id'])) { $name = 'Processing Delays'; $body = []; - foreach ($this->incidents as $campaignId => $campaign) { - foreach ($campaign as $incident) { - if (!empty($incident['body'])) { - $body[] = $incident['body']; - } + foreach ($this->incidents as $incident) { + if (!empty($incident['body'])) { + $body[] = $incident['body']; } } $body = implode('\n', $body);