diff --git a/Dockerfile b/Dockerfile
index c609305c9b..dd9769d970 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,6 +21,7 @@ RUN apt-get -qq update && apt-get -qq install \
php-xdebug \
gettext \
rsync \
+ mariadb-client \
--no-install-recommends && \
rm -r /var/lib/apt/lists/*
diff --git a/classes/AlertView/Standard.php b/classes/AlertView/Standard.php
index 0f07f09dcb..011d91fc1f 100644
--- a/classes/AlertView/Standard.php
+++ b/classes/AlertView/Standard.php
@@ -25,7 +25,9 @@ public function display() {
$this->checkInput();
$this->searchForConstituenciesAndMembers();
- if (!sizeof($this->data['errors']) && ($this->data['keyword'] || $this->data['pid'])) {
+ if ($this->data['step'] || $this->data['addword']) {
+ $this->processStep();
+ } elseif (!$this->data['results'] == 'changes-abandoned' && !sizeof($this->data['errors']) && $this->data['submitted'] && ($this->data['keyword'] || $this->data['pid'])) {
$this->addAlert();
}
@@ -37,6 +39,7 @@ public function display() {
return $this->data;
}
+ # This only happens if we have an alert and want to do something to it.
private function processAction() {
$token = get_http_var('t');
$alert = $this->alert->check_token($token);
@@ -48,7 +51,8 @@ private function processAction() {
$success = $this->confirmAlert($token);
if ($success) {
$this->data['results'] = 'alert-confirmed';
- $this->data['criteria'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($this->alert->criteria);
+ $this->data['criteria'] = $this->alert->criteria;
+ $this->data['display_criteria'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($this->alert->criteria);
}
} elseif ($action == 'Suspend') {
$success = $this->suspendAlert($token);
@@ -70,6 +74,8 @@ private function processAction() {
if ($success) {
$this->data['results'] = 'all-alerts-deleted';
}
+ } elseif ($action == 'Abandon') {
+ $this->data['results'] = 'changes-abandoned';
}
if (!$success) {
$this->data['results'] = 'alert-fail';
@@ -79,6 +85,50 @@ private function processAction() {
$this->data['alert'] = $alert;
}
+ # Process a screen in the alert creation wizard
+ private function processStep() {
+ # fetch a list of suggested terms. Need this for the define screen so we can filter out the suggested terms
+ # and not show them if the user goes back
+ if (($this->data['step'] == 'review' || $this->data['step'] == 'define') && !$this->data['shown_related']) {
+ $suggestions = [];
+ foreach ($this->data['keywords'] as $word) {
+ $terms = $this->alert->get_related_terms($word);
+ $terms = array_diff($terms, $this->data['keywords']);
+ if ($terms && count($terms)) {
+ $suggestions = array_merge($suggestions, $terms);
+ }
+ }
+
+ if (count($suggestions) > 0) {
+ $this->data['step'] = 'add_vector_related';
+ $this->data['suggestions'] = $suggestions;
+ }
+ # confirm the alert. Handles both creating and editing alerts
+ } elseif ($this->data['step'] == 'confirm') {
+ $success = true;
+ # if there's already an alert assume we are editing it and user must be logged in
+ if ($this->data['alert']) {
+ $success = $this->updateAlert($this->data['alert']['id'], $this->data);
+ if ($success) {
+ # reset all the data to stop anything getting confused
+ $this->data['results'] = 'alert-confirmed';
+ $this->data['step'] = '';
+ $this->data['pid'] = '';
+ $this->data['alertsearch'] = '';
+ $this->data['pc'] = '';
+ $this->data['members'] = false;
+ $this->data['constituencies'] = [];
+ } else {
+ $this->data['results'] = 'alert-fail';
+ $this->data['step'] = 'review';
+ }
+ } else {
+ $success = $this->addAlert();
+ $this->data['step'] = '';
+ }
+ }
+ }
+
private function getBasicData() {
global $this_page;
@@ -93,12 +143,102 @@ private function getBasicData() {
$this->data["email"] = trim(get_http_var("email"));
$this->data['email_verified'] = false;
}
+
+ $this->data['token'] = get_http_var('t');
+ $this->data['step'] = trim(get_http_var("step"));
+ $this->data['mp_step'] = trim(get_http_var("mp_step"));
+ $this->data['addword'] = trim(get_http_var("addword"));
+ $this->data['this_step'] = trim(get_http_var("this_step"));
+ $this->data['shown_related'] = get_http_var('shown_related');
+ $this->data['match_all'] = get_http_var('match_all') == 'on';
$this->data['keyword'] = trim(get_http_var("keyword"));
- $this->data['pid'] = trim(get_http_var("pid"));
+ $this->data['search_section'] = '';
$this->data['alertsearch'] = trim(get_http_var("alertsearch"));
+ $this->data['mp_search'] = trim(get_http_var("mp_search"));
+ $this->data['pid'] = trim(get_http_var("pid"));
$this->data['pc'] = get_http_var('pc');
- $this->data['submitted'] = get_http_var('submitted') || $this->data['pid'] || $this->data['keyword'];
- $this->data['token'] = get_http_var('t');
+ $this->data['submitted'] = get_http_var('submitted') || $this->data['pid'] || $this->data['keyword'] || $this->data['step'];
+
+ if ($this->data['addword'] || $this->data['step']) {
+ $alert = $this->alert->check_token($this->data['token']);
+
+ $criteria = '';
+ if ($alert) {
+ $criteria = $alert['criteria'];
+ }
+
+ $this->data['alert'] = $alert;
+
+ $this->data['alert_parts'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($criteria, true);
+
+ $existing_rep = '';
+ if (isset($this->data['alert_parts']['spokenby'])) {
+ $existing_rep = $this->data['alert_parts']['spokenby'][0];
+ }
+
+ $existing_section = '';
+ if (count($this->data['alert_parts']['sections'])) {
+ $existing_section = $this->data['alert_parts']['sections'][0];
+ }
+
+ if ($this->data['alert_parts']['match_all']) {
+ $this->data['match_all'] = true;
+ }
+
+ $words = get_http_var('words', $this->data['alert_parts']['words'], true);
+
+ $this->data['words'] = [];
+ $this->data['keywords'] = [];
+ foreach ($words as $word) {
+ if (trim($word) != '') {
+ $this->data['keywords'][] = $word;
+ $this->data['words'][] = $this->wrap_phrase_in_quotes($word);
+ }
+ }
+
+ $add_all_related = get_http_var('add_all_related');
+ $this->data['add_all_related'] = $add_all_related;
+ $this->data['skip_keyword_terms'] = [];
+
+ $selected_related_terms = get_http_var('selected_related_terms', [], true);
+ $this->data['selected_related_terms'] = $selected_related_terms;
+
+ if ($this->data['step'] !== 'define') {
+ if ($add_all_related) {
+ $this->data['selected_related_terms'] = [];
+ $related_terms = get_http_var('related_terms', [], true);
+ foreach ($related_terms as $term) {
+ $this->data['skip_keyword_terms'][] = $term;
+ $this->data['keywords'][] = $term;
+ $this->data['words'][] = $this->wrap_phrase_in_quotes($term);
+ }
+ } else {
+ $this->data['skip_keyword_terms'] = $selected_related_terms;
+ foreach ($selected_related_terms as $term) {
+ $this->data['keywords'][] = $term;
+ $this->data['words'][] = $this->wrap_phrase_in_quotes($term);
+ }
+ }
+ }
+ $this->data['exclusions'] = trim(get_http_var("exclusions", implode('', $this->data['alert_parts']['exclusions'])));
+ $this->data['representative'] = trim(get_http_var("representative", $existing_rep));
+
+ $this->data['search_section'] = trim(get_http_var("search_section", $existing_section));
+
+ $separator = ' OR ';
+ if ($this->data['match_all']) {
+ $separator = ' ';
+ }
+ $this->data['keyword'] = implode($separator, $this->data['words']);
+ if ($this->data['exclusions']) {
+ $this->data['keyword'] = '(' . $this->data['keyword'] . ') -' . $this->data["exclusions"];
+ }
+
+ $this->data['results'] = '';
+
+ $this->getSearchSections();
+ } # XXX probably should do something here if $alertsearch is set
+
$this->data['sign'] = get_http_var('sign');
$this->data['site'] = get_http_var('site');
$this->data['message'] = '';
@@ -106,13 +246,49 @@ private function getBasicData() {
$ACTIONURL = new \MySociety\TheyWorkForYou\Url($this_page);
$ACTIONURL->reset();
$this->data['actionurl'] = $ACTIONURL->generate();
+
}
- private function checkInput() {
+ private function wrap_phrase_in_quotes($phrase) {
+ if (strpos($phrase, ' ') > 0) {
+ $phrase = '"' . trim($phrase, '"') . '"';
+ }
+
+ return $phrase;
+ }
+
+ private function getRecentResults($text) {
+ global $SEARCHENGINE;
+ $se = new \SEARCHENGINE($text);
+ $this->data['search_result_count'] = $se->run_count(0, 10);
+ $se->run_search(0, 1, 'date');
+ }
+
+ private function getSearchSections() {
+ $this->data['sections'] = [];
+ if ($this->data['search_section']) {
+ foreach (explode(' ', $this->data['search_section']) as $section) {
+ $this->data['sections'][] = \MySociety\TheyWorkForYou\Utility\Alert::sectionToTitle($section);
+ }
+ }
+ }
+
+ protected function updateAlert($token) {
+ $success = $this->alert->update($token, $this->data);
+ return $success;
+ }
+
+ protected function checkInput() {
global $SEARCHENGINE;
$errors = [];
+ # these are the initial screens and so cannot have any errors as we've not submitted
+ if (!$this->data['submitted'] || $this->data['step'] == 'define' || $this->data['mp_step'] == 'mp_alert') {
+ $this->data['errors'] = $errors;
+ return;
+ }
+
// Check each of the things the user has input.
// If there is a problem with any of them, set an entry in the $errors array.
// This will then be used to (a) indicate there were errors and (b) display
@@ -131,6 +307,9 @@ private function checkInput() {
}
$text = $this->data['alertsearch'];
+ if ($this->data['mp_search']) {
+ $text = $this->data['mp_search'];
+ }
if (!$text) {
$text = $this->data['keyword'];
}
@@ -155,11 +334,49 @@ private function checkInput() {
$this->data['errors'] = $errors;
}
- private function searchForConstituenciesAndMembers() {
- // Do the search
- if ($this->data['alertsearch']) {
- $this->data['members'] = \MySociety\TheyWorkForYou\Utility\Search::searchMemberDbLookupWithNames($this->data['alertsearch'], true);
- [$this->data['constituencies'], $this->data['valid_postcode']] = \MySociety\TheyWorkForYou\Utility\Search::searchConstituenciesByQuery($this->data['alertsearch']);
+ protected function searchForConstituenciesAndMembers() {
+ if ($this->data['results'] == 'changes-abandoned') {
+ $this->data['members'] = false;
+ return;
+ }
+
+ $text = $this->data['alertsearch'];
+ if ($this->data['mp_search']) {
+ $text = $this->data['mp_search'];
+ }
+ $errors = [];
+ if ($text != '') {
+ //$members_from_pids = array_values(\MySociety\TheyWorkForYou\Utility\Search::membersForIDs($this->data['alertsearch']));
+ $members_from_names = [];
+ $names_from_pids = array_values(\MySociety\TheyWorkForYou\Utility\Search::speakerNamesForIDs($text));
+ foreach ($names_from_pids as $name) {
+ $members_from_names = array_merge($members_from_names,\MySociety\TheyWorkForYou\Utility\Search::searchMemberDbLookupWithNames($name));
+ }
+ $members_from_words = \MySociety\TheyWorkForYou\Utility\Search::searchMemberDbLookupWithNames($text, true);
+ $this->data['members'] = array_merge($members_from_words, $members_from_names);
+ [$this->data['constituencies'], $this->data['valid_postcode']] = \MySociety\TheyWorkForYou\Utility\Search::searchConstituenciesByQuery($text, false);
+ } elseif ($this->data['pid']) {
+ $MEMBER = new \MEMBER(['person_id' => $this->data['pid']]);
+ $this->data['members'] = [[
+ "person_id" => $MEMBER->person_id,
+ "given_name" => $MEMBER->given_name,
+ "family_name" => $MEMBER->family_name,
+ "house" => $MEMBER->house_disp,
+ "title" => $MEMBER->title,
+ "lordofname" => $MEMBER->lordofname,
+ "constituency" => $MEMBER->constituency,
+ ]];
+ } elseif (isset($this->data['representative']) && $this->data['representative'] != '') {
+ $this->data['members'] = \MySociety\TheyWorkForYou\Utility\Search::searchMemberDbLookupWithNames($this->data['representative'], true);
+
+ $member_count = count($this->data['members']);
+ if ($member_count == 0) {
+ $errors["representative"] = gettext("No matching representative found");
+ } elseif ($member_count > 1) {
+ $errors["representative"] = gettext("Multiple matching representatives found, please select one.");
+ } else {
+ $this->data['pid'] = $this->data['members'][0]['person_id'];
+ }
} else {
$this->data['members'] = [];
}
@@ -167,28 +384,46 @@ private function searchForConstituenciesAndMembers() {
# If the above search returned one result for constituency
# search by postcode, use it immediately
if (isset($this->data['constituencies']) && count($this->data['constituencies']) == 1 && $this->data['valid_postcode']) {
- $MEMBER = new \MEMBER(['constituency' => $this->data['constituencies'][0], 'house' => 1]);
+ $MEMBER = new \MEMBER(['constituency' => array_values($this->data['constituencies'])[0], 'house' => 1]);
$this->data['pid'] = $MEMBER->person_id();
- $this->data['pc'] = $this->data['alertsearch'];
+ $this->data['pc'] = $text;
unset($this->data['constituencies']);
- $this->data['alertsearch'] = '';
}
if (isset($this->data['constituencies'])) {
$cons = [];
foreach ($this->data['constituencies'] as $constituency) {
try {
- $MEMBER = new \MEMBER(['constituency' => $constituency, 'house' => 1]);
+ $MEMBER = new \MEMBER(['constituency' => $constituency]);
$cons[$constituency] = $MEMBER;
} catch (\MySociety\TheyWorkForYou\MemberException $e) {
// do nothing
}
}
$this->data['constituencies'] = $cons;
+ if (count($cons) == 1) {
+ $cons = array_values($cons);
+ $this->data['pid'] = $cons[0]->person_id();
+ }
+ }
+
+ if ($this->data['alertsearch'] && !$this->data['mp_step'] && ($this->data['pid'] || $this->data['members'] || $this->data['constituencies'])) {
+ if (count($this->data['members']) == 1) {
+ $this->data['pid'] = $this->data['members'][0]['person_id'];
+ }
+ $this->data['mp_step'] = 'mp_alert';
+ $this->data['mp_search'] = $this->data['alertsearch'];
+ $this->data['alertsearch'] = '';
+ }
+
+ if (count($this->data["errors"]) > 0) {
+ $this->data["errors"] = array_merge($this->data["errors"], $errors);
+ } else {
+ $this->data["errors"] = $errors;
}
}
- private function addAlert() {
+ protected function addAlert() {
$external_auth = auth_verify_with_shared_secret($this->data['email'], OPTION_AUTH_SHARED_SECRET, get_http_var('sign'));
if ($external_auth) {
$confirm = false;
@@ -203,8 +438,12 @@ private function addAlert() {
$success = $this->alert->add($this->data, $confirm);
if ($success > 0 && !$confirm) {
+ $this->data['step'] = '';
+ $this->data['mp_step'] = '';
$result = 'alert-added';
} elseif ($success > 0) {
+ $this->data['step'] = '';
+ $this->data['mp_step'] = '';
$result = 'alert-confirmation';
} elseif ($success == -2) {
// we need to make sure we know that the person attempting to sign up
@@ -230,11 +469,12 @@ private function addAlert() {
$this->data['pc'] = '';
$this->data['results'] = $result;
- $this->data['criteria'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($this->alert->criteria);
+ $this->data['criteria'] = $this->alert->criteria;
+ $this->data['display_criteria'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($this->alert->criteria);
}
- private function formatSearchTerms() {
+ protected function formatSearchTerms() {
if ($this->data['alertsearch']) {
$this->data['alertsearch_pretty'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($this->data['alertsearch']);
$this->data['search_text'] = $this->data['alertsearch'];
@@ -243,7 +483,7 @@ private function formatSearchTerms() {
}
}
- private function checkForCommonMistakes() {
+ protected function checkForCommonMistakes() {
$mistakes = [];
if (strstr($this->data['alertsearch'], ',') > -1) {
$mistakes['multiple'] = 1;
@@ -261,7 +501,7 @@ private function checkForCommonMistakes() {
$this->data['mistakes'] = $mistakes;
}
- private function formatSearchMemberData() {
+ protected function formatSearchMemberData() {
if (isset($this->data['postcode'])) {
try {
$postcode = $this->data['postcode'];
@@ -302,17 +542,122 @@ private function formatSearchMemberData() {
}
}
- private function setUserData() {
+ protected function setUserData() {
+ if (!isset($this->data['criteria'])) {
+ $criteria = $this->data['keyword'];
+ if (!$this->data['match_all']) {
+ $has_or = strpos($criteria, ' OR ') !== false;
+ $missing_braces = strpos($criteria, '(') === false;
+
+ if ($has_or && $missing_braces) {
+ $criteria = "($criteria)";
+ }
+ }
+ if ($this->data['search_section']) {
+ $criteria .= " section:" . $this->data['search_section'];
+ }
+ if ($this->data['pid']) {
+ $criteria .= " speaker:" . $this->data['pid'];
+ }
+ $this->getRecentResults($criteria);
+
+ $this->data['criteria'] = $criteria;
+ $this->data['display_criteria'] = \MySociety\TheyWorkForYou\Utility\Alert::prettifyCriteria($criteria);
+ }
+ if ($this->data['results'] == 'changes-abandoned') {
+ $this->data['members'] = false;
+ $this->data['alertsearch'] = '';
+ }
+
+ if ($this->data['alertsearch'] && !(isset($this->data['mistakes']['postcode_and']) || $this->data['members'] || $this->data['pid'])) {
+ $this->data['step'] = 'define';
+ $this->data['words'] = [$this->data['alertsearch']];
+ $this->data['keywords'] = [$this->data['alertsearch']];
+ $this->data['exclusions'] = '';
+ $this->data['representative'] = '';
+ } elseif ($this->data['alertsearch'] && ($this->data['members'] || $this->data['pid'])) {
+ $this->data['mp_step'] = 'mp_alert';
+ $this->data['mp_search'] = [$this->data['alertsearch']];
+ } elseif ($this->data['members'] && $this->data['mp_step'] == 'mp_search') {
+ $this->data['mp_step'] = '';
+ }
+
$this->data['current_mp'] = false;
$this->data['alerts'] = [];
+ $this->data['keyword_alerts'] = [];
+ $this->data['speaker_alerts'] = [];
+ $this->data['spoken_alerts'] = [];
+ $this->data['own_member_alerts'] = [];
+ $this->data['all_keywords'] = [];
+ $this->data['own_mp_criteria'] = '';
+ $own_mp_criteria = '';
+
if ($this->data['email_verified']) {
if ($this->user->postcode()) {
$current_mp = new \MEMBER(['postcode' => $this->user->postcode()]);
- if (!$this->alert->fetch_by_mp($this->data['email'], $current_mp->person_id())) {
+ if ($current_mp_alert = !$this->alert->fetch_by_mp($this->data['email'], $current_mp->person_id())) {
$this->data['current_mp'] = $current_mp;
+ $own_mp_criteria = sprintf('speaker:%s', $current_mp->person_id());
}
+ $own_mp_criteria = $current_mp->full_name();
+ $this->data['own_mp_criteria'] = $own_mp_criteria;
}
$this->data['alerts'] = \MySociety\TheyWorkForYou\Utility\Alert::forUser($this->data['email']);
+ foreach ($this->data['alerts'] as $alert) {
+ if (array_key_exists('spokenby', $alert) and sizeof($alert['spokenby']) == 1 and $alert['spokenby'][0] == $own_mp_criteria) {
+ $this->data['own_member_alerts'][] = $alert;
+ } elseif (array_key_exists('spokenby', $alert)) {
+ if (!array_key_exists($alert['spokenby'][0], $this->data['spoken_alerts'])) {
+ $this->data['spoken_alerts'][$alert['spokenby'][0]] = [];
+ }
+ $this->data['spoken_alerts'][$alert['spokenby'][0]][] = $alert;
+ }
+ }
+ foreach ($this->data['alerts'] as $alert) {
+ $term = implode(' ', $alert['words']);
+ $add = true;
+ if (array_key_exists('spokenby', $alert)) {
+ $add = false;
+ } elseif (array_key_exists($term, $this->data['spoken_alerts'])) {
+ $add = false;
+ $this->data['all_keywords'][] = $term;
+ $this->data['spoken_alerts'][$term][] = $alert;
+ } elseif ($term == $own_mp_criteria) {
+ $add = false;
+ $this->data['all_keywords'][] = $term;
+ $this->data['own_member_alerts'][] = $alert;
+ } elseif (\MySociety\TheyWorkForYou\Utility\Search::searchMemberDbLookupWithNames($term, true)) {
+ if (!array_key_exists($term, $this->data['spoken_alerts'])) {
+ $this->data['spoken_alerts'][$term] = [];
+ }
+ $add = false;
+ # need to add this to make it consistent so the front end know where to get the name
+ $alert['spokenby'] = [$term];
+ $this->data['all_keywords'][] = $term;
+ $this->data['spoken_alerts'][$term][] = $alert;
+ }
+ if ($add) {
+ $this->data['all_keywords'][] = $term;
+ $this->data['keyword_alerts'][] = $alert;
+ }
+ }
+ } else {
+ if ($this->data['alertsearch'] && $this->data['pc']) {
+ $this->data['mp_step'] = 'mp_alert';
+ }
+ }
+ if (count($this->data['alerts'])) {
+ $this->data['delete_token'] = $this->data['alerts'][0]['token'];
+ }
+ if ($this->data['addword'] != '' || ($this->data['step'] && count($this->data['errors']) > 0)) {
+ $this->data["step"] = get_http_var('this_step');
+ } else {
+ $this->data['this_step'] = '';
+ }
+
+ $this->data["search_term"] = $this->data['alertsearch'];
+ if ($this->data['mp_search']) {
+ $this->data["search_term"] = $this->data['mp_search'];
}
}
}
diff --git a/classes/Utility/Alert.php b/classes/Utility/Alert.php
index 89da17ec29..6e87183132 100644
--- a/classes/Utility/Alert.php
+++ b/classes/Utility/Alert.php
@@ -9,6 +9,27 @@
*/
class Alert {
+ #XXX don't calculate this every time
+ public static function sectionToTitle($section) {
+ $section_map = [
+ "uk" => gettext('All UK'),
+ "debates" => gettext('House of Commons debates'),
+ "whalls" => gettext('Westminster Hall debates'),
+ "lords" => gettext('House of Lords debates'),
+ "wrans" => gettext('Written answers'),
+ "wms" => gettext('Written ministerial statements'),
+ "standing" => gettext('Bill Committees'),
+ "future" => gettext('Future Business'),
+ "ni" => gettext('Northern Ireland Assembly Debates'),
+ "scotland" => gettext('All Scotland'),
+ "sp" => gettext('Scottish Parliament Debates'),
+ "spwrans" => gettext('Scottish Parliament Written answers'),
+ "wales" => gettext('Welsh parliament record'),
+ "lmqs" => gettext('Questions to the Mayor of London'),
+ ];
+
+ return $section_map[$section];
+ }
public static function detailsToCriteria($details) {
$criteria = [];
@@ -20,6 +41,10 @@ public static function detailsToCriteria($details) {
$criteria[] = 'speaker:' . $details['pid'];
}
+ if (!empty($details['search_section'])) {
+ $criteria[] = 'section:' . $details['search_section'];
+ }
+
$criteria = join(' ', $criteria);
return $criteria;
}
@@ -34,6 +59,7 @@ public static function forUser($email) {
$alerts = [];
foreach ($q as $row) {
$criteria = self::prettifyCriteria($row['criteria']);
+ $parts = self::prettifyCriteria($row['criteria'], true);
$token = $row['alert_id'] . '-' . $row['registrationtoken'];
$status = 'confirmed';
@@ -43,36 +69,87 @@ public static function forUser($email) {
$status = 'suspended';
}
- $alerts[] = [
+ $alert = [
'token' => $token,
'status' => $status,
'criteria' => $criteria,
'raw' => $row['criteria'],
+ 'keywords' => [],
+ 'exclusions' => [],
+ 'sections' => [],
];
+
+ $alert = array_merge($alert, $parts);
+
+ $alerts[] = $alert;
}
return $alerts;
}
- public static function prettifyCriteria($alert_criteria) {
+ public static function prettifyCriteria($alert_criteria, $as_parts = false) {
$text = '';
+ $parts = ['words' => [], 'sections' => [], 'exclusions' => [], 'match_all' => true];
if ($alert_criteria) {
- $criteria = explode(' ', $alert_criteria);
+ # check for phrases
+ if (strpos($alert_criteria, ' OR ') !== false) {
+ $parts['match_all'] = false;
+ }
+ $alert_criteria = str_replace(' OR ', ' ', $alert_criteria);
+ $alert_criteria = str_replace(['(', ')'], '', $alert_criteria);
+ if (strpos($alert_criteria, '"') !== false) {
+ # match phrases
+ preg_match_all('/"([^"]*)"/', $alert_criteria, $phrases);
+ # and then remove them from the criteria
+ $alert_criteria = trim(preg_replace('/ +/', ' ', str_replace($phrases[0], "", $alert_criteria)));
+
+ # and then create an array with the words and phrases
+ $criteria = [];
+ if ( $alert_criteria != "") {
+ $criteria = explode(' ', $alert_criteria);
+ }
+ $criteria = array_merge($criteria, $phrases[1]);
+ } else {
+ $criteria = explode(' ', $alert_criteria);
+ }
$words = [];
+ $exclusions = [];
+ $sections = [];
+ $sections_verbose = [];
$spokenby = array_values(\MySociety\TheyWorkForYou\Utility\Search::speakerNamesForIDs($alert_criteria));
foreach ($criteria as $c) {
- if (!preg_match('#^speaker:(\d+)#', $c, $m)) {
+ if (preg_match('#^section:(\w+)#', $c, $m)) {
+ $sections[] = $m[1];
+ $sections_verbose[] = self::sectionToTitle($m[1]);
+ } elseif (strpos($c, '-') === 0) {
+ $exclusions[] = str_replace('-', '', $c);
+ } elseif (!preg_match('#^speaker:(\d+)#', $c, $m)) {
$words[] = $c;
}
}
if ($spokenby && count($words)) {
$text = implode(' or ', $spokenby) . ' mentions [' . implode(' ', $words) . ']';
+ $parts['spokenby'] = $spokenby;
+ $parts['words'] = $words;
} elseif (count($words)) {
$text = '[' . implode(' ', $words) . ']' . ' is mentioned';
+ $parts['words'] = $words;
} elseif ($spokenby) {
$text = implode(' or ', $spokenby) . " speaks";
+ $parts['spokenby'] = $spokenby;
}
+
+ if ($sections) {
+ $text = $text . " in " . implode(' or ', $sections_verbose);
+ $parts['sections'] = $sections;
+ $parts['sections_verbose'] = $sections_verbose;
+ }
+
+ $parts['exclusions'] = $exclusions;
+ }
+ if ($as_parts) {
+ return $parts;
}
return $text;
}
diff --git a/classes/Utility/Search.php b/classes/Utility/Search.php
index 67200ee12e..8f2420cb6d 100644
--- a/classes/Utility/Search.php
+++ b/classes/Utility/Search.php
@@ -249,12 +249,19 @@ public static function searchMemberDbLookupWithNames($searchstring, $current_onl
* saying whether it was a postcode used.
*/
- public static function searchConstituenciesByQuery($searchterm) {
+ public static function searchConstituenciesByQuery($searchterm, $mp_only=true) {
if (validate_postcode($searchterm)) {
// Looks like a postcode - can we find the constituency?
- $constituency = Postcode::postcodeToConstituency($searchterm);
- if ($constituency) {
- return [ [$constituency], true ];
+ if ($mp_only) {
+ $constituency = Postcode::postcodeToConstituency($searchterm);
+ if ($constituency) {
+ return [ [$constituency], true ];
+ }
+ } else {
+ $constituencies = Postcode::postcodeToConstituencies($searchterm);
+ if ($constituencies) {
+ return [ $constituencies, true ];
+ }
}
}
@@ -297,6 +304,28 @@ public static function speakerNamesForIDs($searchstring) {
return $speakers;
}
+ /**
+ * get list of members of speaker IDs from search string
+ *
+ * @param string $searchstring The search string with the speaker:NNN text
+ *
+ * @return array Array with the speaker id string as key and speaker name as value
+ */
+
+ public static function membersForIDs($searchstring) {
+ $criteria = explode(' ', $searchstring);
+ $speakers = [];
+
+ foreach ($criteria as $c) {
+ if (preg_match('#^speaker:(\d+)#', $c, $m)) {
+ $MEMBER = new \MEMBER(['person_id' => $m[1]]);
+ $speakers[$m[1]] = $MEMBER;
+ }
+ }
+
+ return $speakers;
+ }
+
/**
* replace speaker:NNNN with speaker:Name in search string
*
diff --git a/db/0025-add-vector-search-suggestions.sql b/db/0025-add-vector-search-suggestions.sql
new file mode 100644
index 0000000000..8cc47224d9
--- /dev/null
+++ b/db/0025-add-vector-search-suggestions.sql
@@ -0,0 +1,5 @@
+CREATE TABLE `vector_search_suggestions` (
+ `search_term` varchar(100) NOT NULL default '',
+ `search_suggestion` varchar(100) NOT NULL default '',
+ KEY `search_term` (`search_term`)
+);
diff --git a/db/schema.sql b/db/schema.sql
index 605b2d70a7..ef43d99edb 100644
--- a/db/schema.sql
+++ b/db/schema.sql
@@ -214,6 +214,12 @@ CREATE TABLE `postcode_lookup` (
PRIMARY KEY (`postcode`)
);
+CREATE TABLE `vector_search_suggestions` (
+ `search_term` varchar(100) NOT NULL default '',
+ `search_suggestion` varchar(100) NOT NULL default '',
+ KEY `search_term` (`search_term`)
+);
+
-- each time we index, we increment the batch number;
-- can use this to speed up search
CREATE TABLE `indexbatch` (
diff --git a/scripts/import_search_suggestions.py b/scripts/import_search_suggestions.py
new file mode 100755
index 0000000000..3ff09bca55
--- /dev/null
+++ b/scripts/import_search_suggestions.py
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+"""
+import_search_suggestions.py - Import vector search suggestions
+
+See python scripts/import_search_suggestions.py --help for usage.
+
+"""
+
+import re
+import sys
+from pathlib import Path
+from typing import cast
+from warnings import filterwarnings
+
+import MySQLdb
+import pandas as pd
+import rich_click as click
+from pylib.mysociety import config
+from rich import print
+from rich.prompt import Prompt
+
+repository_path = Path(__file__).parent.parent
+
+config.set_file(repository_path / "conf" / "general")
+
+# suppress warnings about using mysqldb in pandas
+filterwarnings(
+ "ignore",
+ category=UserWarning,
+ message=".*pandas only supports SQLAlchemy connectable.*",
+)
+
+
+@click.group()
+def cli():
+ pass
+
+
+def get_twfy_db_connection() -> MySQLdb.Connection:
+ db_connection = cast(
+ MySQLdb.Connection,
+ MySQLdb.connect(
+ host=config.get("TWFY_DB_HOST"),
+ db=config.get("TWFY_DB_NAME"),
+ user=config.get("TWFY_DB_USER"),
+ passwd=config.get("TWFY_DB_PASS"),
+ charset="utf8",
+ ),
+ )
+ return db_connection
+
+
+def df_to_db(df: pd.DataFrame, verbose: bool = False):
+ """
+ add search suggestions to the database
+ """
+ df = df.dropna(how="any")
+ db_connection = get_twfy_db_connection()
+
+ with db_connection.cursor() as cursor:
+ # just remove everything and re-insert it all rather than trying to update things
+ cursor.execute("DELETE FROM vector_search_suggestions")
+ insert_command = "INSERT INTO vector_search_suggestions (search_term, search_suggestion) VALUES (%s, %s)"
+ suggestion_data = [
+ (row["original_query"], row["match"]) for _, row in df.iterrows()
+ ]
+ cursor.executemany(insert_command, suggestion_data)
+ db_connection.commit()
+
+ if verbose:
+ print(f"[green]{len(df)} rows updated.")
+
+ db_connection.close()
+
+
+def url_to_db(url: str, verbose: bool = False):
+ """
+ Pipe external URL into the update process.
+ """
+ df = pd.read_csv(url)
+
+ df_to_db(df, verbose=verbose)
+
+
+def file_to_db(file: str, verbose: bool = False):
+ """
+ Pipe file into the update process.
+ """
+ df = pd.read_csv(file)
+
+ df_to_db(df, verbose=verbose)
+
+
+@cli.command()
+@click.option(
+ "--url",
+ required=False,
+ default=None,
+ help="A csv file to update search suggestions from.",
+)
+@click.option(
+ "--file",
+ required=False,
+ default=None,
+ help="A csv file to update search suggestions from.",
+)
+@click.option("--verbose", is_flag=True, help="Show verbose output")
+def update_vector_search_suggestions(url: str, file: str, verbose: bool = False):
+ """
+ Update the vector search suggestions
+ """
+ if file:
+ file_to_db(file, verbose=verbose)
+ elif url:
+ url_to_db(url, verbose=verbose)
+
+
+@cli.command()
+def count_suggestions():
+ """
+ for diagnostics to check import has worked
+ """
+ db_connection = get_twfy_db_connection()
+ with db_connection.cursor() as cursor:
+ cursor.execute(
+ "select count(*) as num_suggestions from vector_search_suggestions"
+ )
+ count = cursor.fetchone()[0]
+ print(f"There are {count} suggestions in the db")
+
+ db_connection.close()
+
+
+def main():
+ cli()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/AlertsPageTest.php b/tests/AlertsPageTest.php
index 6d62a781a7..91a2b15f24 100644
--- a/tests/AlertsPageTest.php
+++ b/tests/AlertsPageTest.php
@@ -15,6 +15,10 @@ private function fetch_page($vars) {
return $this->base_fetch_page($vars, 'alert');
}
+ private function get_page($vars = []) {
+ return $this->base_fetch_page_user($vars, '1.fbb689a0c092f5534b929d302db2c8a9', 'alert');
+ }
+
public function testFetchPage() {
$page = $this->fetch_page([]);
$this->assertStringContainsString('TheyWorkForYou Email Alerts', $page);
@@ -22,12 +26,18 @@ public function testFetchPage() {
public function testKeywordOnly() {
$page = $this->fetch_page([ 'alertsearch' => 'elephant']);
- $this->assertStringContainsString('Receive alerts when [elephant] is mentioned', $page);
+ $this->assertStringContainsString('What word or phrase would you like to recieve alerts about', $page);
+ $this->assertStringContainsString('fetch_page([ 'alertsearch' => 'speaker:2']);
+ $this->assertStringContainsString('Mrs Test Current-MP', $page);
}
public function testPostCodeOnly() {
$page = $this->fetch_page([ 'alertsearch' => 'SE17 3HE']);
- $this->assertStringContainsString('when Mrs Test Current-MP', $page);
+ $this->assertStringContainsString('Mrs Test Current-MP', $page);
}
public function testPostCodeWithKeyWord() {
@@ -49,4 +59,73 @@ public function testPostcodeAndKeywordWithNoSittingMP() {
$this->assertStringContainsString('You have used a postcode and something else', $page);
$this->assertStringNotContainsString('Did you mean to get alerts for when your MP', $page);
}
+
+ public function testBasicKeyWordAlertsCreation() {
+ $page = $this->fetch_page([ 'step' => 'define']);
+ $this->assertStringContainsString('What word or phrase would you like to recieve alerts about', $page);
+ $this->assertStringContainsString('fetch_page([ 'step' => 'review', 'email' => 'test@example.org', 'words[]' => 'fish']);
+ $this->assertStringContainsString('Review Your Alert', $page);
+ $this->assertStringContainsString('fetch_page([ 'step' => 'confirm', 'email' => 'test@example.org', 'words[]' => 'fish']);
+ $this->assertStringContainsString('We’re nearly done', $page);
+ $this->assertStringContainsString('You should receive an email shortly', $page);
+ }
+
+ public function testMultipleKeyWordAlertsCreation() {
+ $page = $this->fetch_page([ 'step' => 'define']);
+ $this->assertStringContainsString('What word or phrase would you like to recieve alerts about', $page);
+ $this->assertStringContainsString('fetch_page([ 'step' => 'review', 'email' => 'test@example.org', 'words[]' => ['fish', 'salmon']]);
+ $this->assertStringContainsString('Review Your Alert', $page);
+ $this->assertStringContainsString('assertStringContainsString('fetch_page([ 'step' => 'confirm', 'email' => 'test@example.org', 'words[]' => ['fish', 'salmon']]);
+ $this->assertStringContainsString('You should receive an email shortly', $page);
+ }
+
+ public function testMultipleKeyWordAlertsCreationLoggedIn() {
+ $page = $this->get_page(['step' => 'define']);
+ $this->assertStringContainsString('What word or phrase would you like to recieve alerts about', $page);
+ $this->assertStringContainsString('get_page([ 'step' => 'review', 'words[]' => ['fish', 'salmon']]);
+ $this->assertStringContainsString('Review Your Alert', $page);
+ $this->assertStringContainsString('assertStringContainsString('get_page([ 'step' => 'confirm', 'words[]' => ['fish', 'salmon']]);
+ $this->assertStringContainsString('You will now receive email alerts on any day when [fish salmon] is mentioned in parliament', $page);
+ }
+
+ public function testKeyWordAndSectionAlertsCreationLoggedIn() {
+ $page = $this->get_page(['step' => 'define']);
+ $this->assertStringContainsString('What word or phrase would you like to recieve alerts about', $page);
+ $this->assertStringContainsString('get_page(['step' => 'review', 'words[]' => 'fish', 'search_section' => 'debates']);
+ $this->assertStringContainsString('Review Your Alert', $page);
+ $this->assertStringContainsString('get_page(['step' => 'confirm', 'words[]' => 'fish', 'search_section' => 'debates']);
+ $this->assertStringContainsString('You will now receive email alerts on any day when [fish] is mentioned in House of Commons debates', $page);
+ }
+
+ public function testKeyWordAndSpeakerAlertsCreationLoggedIn() {
+ $page = $this->get_page(['step' => 'define']);
+ $this->assertStringContainsString('What word or phrase would you like to recieve alerts about', $page);
+ $this->assertStringContainsString('get_page(['step' => 'review', 'words[]' => 'fish', 'representative' => 'Mrs Test Current-MP']);
+ $this->assertStringContainsString('Review Your Alert', $page);
+ $this->assertStringContainsString('assertStringContainsString('get_page([ 'step' => 'confirm', 'words[]' => 'fish', 'representative' => 'Mrs Test Current-MP']);
+ $this->assertStringContainsString('You will now receive email alerts on any day when Mrs Test Current-MP mentions [fish] in parliament', $page);
+ }
}
diff --git a/tests/FetchPageTestCase.php b/tests/FetchPageTestCase.php
index 352dfbfc47..07b6334544 100644
--- a/tests/FetchPageTestCase.php
+++ b/tests/FetchPageTestCase.php
@@ -6,7 +6,15 @@
abstract class FetchPageTestCase extends TWFY_Database_TestCase {
protected function base_fetch_page($vars, $dir, $page = 'index.php', $req_uri = '') {
foreach ($vars as $k => $v) {
- $vars[$k] = $k . '=' . urlencode($v);
+ if (strpos($k, '[') !== false) {
+ if (is_array($vars[$k])) {
+ $vars[$k] = $k . '=' . join("&$k=", $vars[$k]);
+ } else {
+ $vars[$k] = $k . '=' . urlencode($v);
+ }
+ } else {
+ $vars[$k] = $k . '=' . urlencode($v);
+ }
}
if (!$req_uri) {
@@ -22,7 +30,15 @@ protected function base_fetch_page($vars, $dir, $page = 'index.php', $req_uri =
protected function base_fetch_page_user($vars, $cookie, $dir, $page = 'index.php', $req_uri = '') {
foreach ($vars as $k => $v) {
- $vars[$k] = $k . '=' . urlencode($v);
+ if (strpos($k, '[') !== false) {
+ if (is_array($vars[$k])) {
+ $vars[$k] = $k . '=' . join("&$k=", $vars[$k]);
+ } else {
+ $vars[$k] = $k . '=' . urlencode($v);
+ }
+ } else {
+ $vars[$k] = $k . '=' . urlencode($v);
+ }
}
if (!$req_uri) {
diff --git a/tests/SearchTest.php b/tests/SearchTest.php
index a99e0a6db3..0bd2e38fa7 100644
--- a/tests/SearchTest.php
+++ b/tests/SearchTest.php
@@ -121,7 +121,7 @@ public function testSearchPage() {
public function testSearchPageMP() {
$page = $this->fetch_page([ 'q' => 'Mary Smith' ]);
$this->assertStringContainsString('Mary Smith', $page);
- $this->assertStringContainsString('MP, Amber Valley', $page);
+ $this->assertMatchesRegularExpression('/MP *for Amber Valley/', $page);
}
/**
@@ -169,9 +169,9 @@ public function testSearchPageMultipleCons() {
$page = $this->fetch_page([ 'q' => 'Liverpool' ]);
$this->assertStringContainsString('MPs in constituencies matching Liverpool', $page);
$this->assertStringContainsString('Susan Brown', $page);
- $this->assertStringContainsString('MP, Liverpool, Riverside', $page);
+ $this->assertMatchesRegularExpression('/MP *for Liverpool, Riverside/', $page);
$this->assertStringContainsString('Andrew Jones', $page);
- $this->assertStringContainsString('MP, Liverpool, Walton', $page);
+ $this->assertMatchesRegularExpression('/MP *for Liverpool, Walton/', $page);
}
/**
diff --git a/tests/_fixtures/alertspage.xml b/tests/_fixtures/alertspage.xml
index e237dec30d..b1c65e678c 100644
--- a/tests/_fixtures/alertspage.xml
+++ b/tests/_fixtures/alertspage.xml
@@ -150,6 +150,14 @@
' + $message + '
+ = gettext('To be alerted on an exact phrase, be sure to put it in quotes. Also use quotes around a word to avoid stemming (where ‘horse’ would also match ‘horses’).') ?> +
++ = gettext('You should only enter one term per alert – if you wish to receive alerts on more than one thing, or for more than one person, simply fill in this form as many times as you need, or use boolean OR.') ?> +
++ = gettext('For example, if you wish to receive alerts whenever the words horse or pony are mentioned in Parliament, please fill in this form once with the word horse and then again with the word pony (or you can put horse OR pony with the OR in capitals). Do not put horse, pony as that will only sign you up for alerts where both horse and pony are mentioned.') ?> +
++ = gettext('The mySociety blog has a number of posts on signing up for and managing alerts:') ?> +
+ ++ = gettext('You are subscribed to the following alerts about your MP.') ?> +
+ + + +Alert when = _htmlspecialchars($own_mp_criteria) ?> is mentioned
+ + + + 0) { ?> += _htmlspecialchars($alert['criteria']) ?> +
Alert when = _htmlspecialchars(implode(', ', $person_alerts[0]['spokenby'])) ?> is mentioned
+ + += _htmlspecialchars($alert['criteria']) ?>
+Alert when = _htmlspecialchars($own_mp_criteria) ?> is mentioned
+ + diff --git a/www/includes/easyparliament/templates/html/alert/index.php b/www/includes/easyparliament/templates/html/alert/index.php index 8f4deef191..12af929ee3 100644 --- a/www/includes/easyparliament/templates/html/alert/index.php +++ b/www/includes/easyparliament/templates/html/alert/index.php @@ -20,7 +20,7 @@= gettext('You will now receive email alerts for the following criteria:') ?>
-= gettext('This is normally the day after, but could conceivably be later due to issues at our or the parliament’s end.') ?>
@@ -83,7 +83,7 @@- = sprintf(gettext('You will now receive email alerts on any day when %s in parliament.'), _htmlspecialchars($criteria)) ?> + = sprintf(gettext('You will now receive email alerts on any day when %s in parliament.'), _htmlspecialchars($display_criteria)) ?>
@@ -104,6 +104,11 @@ = gettext('You should receive an email shortly which will contain a link. You will need to follow that link to confirm your email address and receive future alerts. Thanks.') ?> + ++ = gettext('Those changes have been abandoned and your alerts are unchanged.') ?> +
@@ -115,11 +120,22 @@ - +
= sprintf(gettext('If you join or sign in, you can suspend, resume and delete your email alerts from your profile page.'), '/user/?pg=join', '/user/login/?ret=%2Falert%2F') ?>
= gettext('Plus, you won’t need to confirm your email address for every alert you set.') ?>
- -