From 3e23fdf69113de3c46f4c337aa3ba9c3b5aa97c1 Mon Sep 17 00:00:00 2001 From: dmldc Date: Mon, 24 Jan 2022 17:31:09 -0500 Subject: [PATCH 1/2] Changes audience query to move subject criteria --- models/audience.js | 68 ++++++---------------------------------------- 1 file changed, 8 insertions(+), 60 deletions(-) diff --git a/models/audience.js b/models/audience.js index fe02e170..b0daa342 100644 --- a/models/audience.js +++ b/models/audience.js @@ -427,8 +427,6 @@ class Audience { let order = options.direction || 'DESC'; let params = []; const sortBy = 'ORDER BY first ASC'; // Default - // let query = 'SELECT id, username, email, `first`, `last`, `name`, ref, photo, start_date, type_id, `read`, pr, entries_count, entries_read, percentage_read, reach, viewed, accessed, repi '; - let query = ' SELECT DISTINCT \ `clcdc`.`account`.`id` AS `id`, \ `clcdc`.`account`.`username` AS `username`, \ @@ -463,8 +461,6 @@ class Audience { 0.0001) * 0.1)), \ 1) AS `repi` \ '; - - let whereParams = [this.id, _this.client_id, this.id, _this.client_id]; let rank = null; if (options.rank) { @@ -479,59 +475,30 @@ class Audience { query += rank; whereParams = whereParams.concat(whereParams); } - // query += 'FROM view_repi '; - // query += 'FROM \ - // `clcdc`.`account` \ - // LEFT JOIN `clcdc`.`account_subject_status` ON \ - // `clcdc`.`account_subject_status`.`account_id` = account.id \ - // LEFT JOIN `clcdc`.`account_subject_statistics` ON \ - // `clcdc`.`account_subject_statistics`.`account_id` = account.id \ - // LEFT JOIN `clcdc`.`account_profile` ON \ - // `clcdc`.`account`.`id` = `clcdc`.`account_profile`.`account_id` \ - // OR ISNULL(`clcdc`.`account_profile`.`account_id`) \ - // LEFT JOIN `clcdc`.`profile_subject` ON \ - // `clcdc`.`account_profile`.`profile_id` = `clcdc`.`profile_subject`.`profile_id` \ - // LEFT JOIN `clcdc`.`subject` \ - // ON `clcdc`.`subject`.`id` = `clcdc`.`profile_subject`.`subject_id` \ - // AND `clcdc`.`subject`.`published_date` IS NOT NULL \ - // LEFT JOIN `clcdc`.`module_subject` \ - // ON `clcdc`.`subject`.`id` = `clcdc`.`module_subject`.`subject_id` '; - query += ' \ FROM account_profile \ INNER JOIN account ON account_profile.account_id = account.id \ LEFT JOIN profile_subject ON account_profile.profile_id = profile_subject.profile_id \ - INNER JOIN subject ON profile_subject.subject_id = subject.id \ + AND (profile_subject.profile_id = ? AND profile_subject.client_id = ?) \ + LEFT JOIN subject ON profile_subject.subject_id = subject.id AND \ + ( \ + subject.published_date < now() AND \ + subject.status_id = 3 AND \ + date is not null \ + ) \ LEFT JOIN view_subject_stats ON \ (account_profile.account_id = view_subject_stats.account_id AND view_subject_stats.subject_id = profile_subject.subject_id) \ '; - let where = ' \ WHERE \ ( \ account_profile.profile_id = ? AND \ account_profile.client_id = ? \ - ) AND \ - ( \ - profile_subject.profile_id = ? AND \ - profile_subject.client_id = ? AND \ - subject.published_date < now() AND \ - subject.status_id = 3 AND \ - date is not null \ - ) \ - '; - + ) '; const groupBy = ' \ GROUP BY account_profile.account_id, \ username, email, `first`, `last`, `name`, ref, photo, start_date, type_id \ '; - - // let where = 'WHERE ((`clcdc`.`subject`.`status_id` = 3) \ - // OR ISNULL(`clcdc`.`subject`.`status_id`)) AND `clcdc`.`account`.`client_id` = ? AND `clcdc`.`account_profile`.`profile_id` = ? AND module_id is null AND (`clcdc`.`profile_subject`.`client_id` = ? OR `clcdc`.`profile_subject`.`client_id` is null) AND `clcdc`.`account`.`email` is not null '; - - // // let whereParams = [_this.client_id, this.id, _this.client_id, _this.client_id, this.id, _this.client_id]; - // let groupBy = 'GROUP BY `clcdc`.`account`.`id` , `clcdc`.`account`.`username` , `clcdc`.`account`.`email` , `clcdc`.`account`.`name` , `clcdc`.`account`.`first` , `clcdc`.`account`.`last` , `clcdc`.`account`.`ref` , `clcdc`.`account`.`photo` , `clcdc`.`account`.`start_date` , `clcdc`.`account`.`type_id` , `clcdc`.`account`.`client_id` , `clcdc`.`account_profile`.`profile_id` , `clcdc`.`module_subject`.`module_id` , `clcdc`.`profile_subject`.`client_id` '; - if (parseInt(this.id) === 0) { // view_repi_all greatly simplifies the lookup code for this and provides for rank to be returned // BUT it does not add any client_id/appId restriction to the program being returned @@ -560,30 +527,11 @@ class Audience { query += rank; whereParams.push(_this.client_id); } - // groupBy = 'GROUP BY `clcdc`.`account`.`id` , `clcdc`.`account`.`username` , `clcdc`.`account`.`email` , `clcdc`.`account`.`name` , `clcdc`.`account`.`first` , `clcdc`.`account`.`last` , `clcdc`.`account`.`ref` , `clcdc`.`account`.`photo` , `clcdc`.`account`.`start_date` , `clcdc`.`account`.`type_id` , `clcdc`.`account`.`client_id` , `clcdc`.`module_subject`.`module_id` '; - // query += 'FROM view_repi_all '; } if (options.accountId) { where += ' AND id = ? '; whereParams.push(options.accountId); } - // if (options.startdate) { - // if (options.enddate) { - // where += ' AND dateref >= ? AND dateref <= ? '; - // whereParams.push(options.startdate); - // whereParams.push(options.enddate); - // } - // else { - // where += ' AND dateref >= ? '; - // whereParams.push(options.startdate); - // } - // } - // else { - // if (options.enddate) { - // where += ' AND dateref <= ? '; - // whereParams.push(options.enddate); - // } - // } if (options.searchText) { where += ' AND (username LIKE ? OR email LIKE ? or first LIKE ? or last LIKE ? or name LIKE ? or account.id LIKE ? or account.ref LIKE ?) '; From d63bd60199f0ca417fe33c8f5b9fbf0040994f49 Mon Sep 17 00:00:00 2001 From: dmldc Date: Mon, 24 Jan 2022 17:58:51 -0500 Subject: [PATCH 2/2] Performance fix for audience member lookup --- models/audience.js | 404 +++++++++++++++++++++++++-------------------- 1 file changed, 221 insertions(+), 183 deletions(-) diff --git a/models/audience.js b/models/audience.js index b0daa342..0484b6ff 100644 --- a/models/audience.js +++ b/models/audience.js @@ -415,197 +415,215 @@ class Audience { } getMembers(options, page, done) { const _this = this; - options = options || {}; - options.baseUrl = - typeof options.baseUrl !== 'undefined' - ? options.baseUrl - : '/admin/audience/' + this.id + '/'; - options.pageParam = - typeof options.pageParam !== 'undefined' ? options.pageParam : 'page'; - options.pageSize = - typeof options.pageSize !== 'undefined' ? options.pageSize : 10; - let order = options.direction || 'DESC'; - let params = []; - const sortBy = 'ORDER BY first ASC'; // Default - let query = ' SELECT DISTINCT \ - `clcdc`.`account`.`id` AS `id`, \ - `clcdc`.`account`.`username` AS `username`, \ - `clcdc`.`account`.`email` AS `email`, \ - `clcdc`.`account`.`first` AS `first`, \ - `clcdc`.`account`.`last` AS `last`, \ - IF(ISNULL(`clcdc`.`account`.`name`), \ - `clcdc`.`account`.`first`, \ - `clcdc`.`account`.`name`) AS `name`, \ - `clcdc`.`account`.`ref` AS `ref`, \ - `clcdc`.`account`.`photo` AS `photo`, \ - `clcdc`.`account`.`start_date` AS `start_date`, \ - `clcdc`.`account`.`type_id` AS `type_id` \ - , account_profile.account_id \ - , count(DISTINCT profile_subject.subject_id) as pr \ - , ROUND(SUM(view_subject_stats.repi)/COUNT(DISTINCT profile_subject.subject_id),2) as newrepi \ - , SUM(percentage_read)/COUNT(DISTINCT profile_subject.subject_id) as percentage_read \ - , COUNT(DISTINCT view_subject_stats.subject_id) as `read` \ - , SUM(view_subject_stats.viewed) as viewed \ - , SUM(view_subject_stats.accessed) as accessed \ - , IF (((COUNT(DISTINCT subject.id) > 0) \ - AND (COUNT(DISTINCT view_subject_stats.subject_id) > 0)), \ - ROUND(((COUNT(DISTINCT view_subject_stats.subject_id) / COUNT(DISTINCT subject.id)) * 100), 1), 0) as reach \ - , ROUND(((((((IF((SUM(`view_subject_stats`.`entries_read`) > 0), \ - SUM(`view_subject_stats`.`entries_read`), \ - 0.0001) / IF((SUM(`view_subject_stats`.`entries_count`) > 0), \ - SUM(`view_subject_stats`.`entries_count`), \ - IF((SUM(`view_subject_stats`.`entries_read`) > 0), \ - SUM(`view_subject_stats`.`entries_read`), \ - 0.0001))) * 100) + ((COUNT(DISTINCT `view_subject_stats`.`subject_id`) / COUNT(DISTINCT `clcdc`.`subject`.`id`)) * 100)) / 2) + (SUM(`view_subject_stats`.`viewed`) * 0.01)) + (IF((SUM(`view_subject_stats`.`accessed`) > 0), \ - SUM(`view_subject_stats`.`accessed`), \ - 0.0001) * 0.1)), \ - 1) AS `repi` \ - '; - let whereParams = [this.id, _this.client_id, this.id, _this.client_id]; - let rank = null; - if (options.rank) { - rank = ', FIND_IN_SET(repi, ( \ - SELECT GROUP_CONCAT(DISTINCT repi \ - ORDER BY repi DESC ) \ - FROM view_repi \ - WHERE account_client_id = ? AND module_id is null \ - AND account_profile_id = ? \ - AND (profile_client_id = ? OR profile_client_id is null) \ - )) AS rank '; - query += rank; - whereParams = whereParams.concat(whereParams); - } - query += ' \ - FROM account_profile \ - INNER JOIN account ON account_profile.account_id = account.id \ - LEFT JOIN profile_subject ON account_profile.profile_id = profile_subject.profile_id \ - AND (profile_subject.profile_id = ? AND profile_subject.client_id = ?) \ - LEFT JOIN subject ON profile_subject.subject_id = subject.id AND \ - ( \ - subject.published_date < now() AND \ - subject.status_id = 3 AND \ - date is not null \ - ) \ - LEFT JOIN view_subject_stats ON \ - (account_profile.account_id = view_subject_stats.account_id AND view_subject_stats.subject_id = profile_subject.subject_id) \ - '; - let where = ' \ - WHERE \ - ( \ - account_profile.profile_id = ? AND \ - account_profile.client_id = ? \ - ) '; - const groupBy = ' \ - GROUP BY account_profile.account_id, \ - username, email, `first`, `last`, `name`, ref, photo, start_date, type_id \ - '; - if (parseInt(this.id) === 0) { - // view_repi_all greatly simplifies the lookup code for this and provides for rank to be returned - // BUT it does not add any client_id/appId restriction to the program being returned - // - this won't impact most cases but will need to be looked at if we roll out shared programs and shared ids - // query = 'SELECT id, username, email, `first`, `last`, `name`, ref, photo, start_date, type_id, `read`, pr, entries_count, entries_read, percentage_read, reach, viewed, accessed, repi '; - where = ' \ - WHERE \ - ( \ - account_profile.client_id = ? \ - ) AND \ - ( \ - profile_subject.client_id = ? AND \ - subject.published_date < now() AND \ - subject.status_id = 3 AND \ - date is not null \ - ) \ - '; - whereParams = [_this.client_id, _this.client_id]; + this.getSubjectCount(_this.id, _this.client_id, (subjectCount) => { + options = options || {}; + options.baseUrl = + typeof options.baseUrl !== 'undefined' + ? options.baseUrl + : '/admin/audience/' + this.id + '/'; + options.pageParam = + typeof options.pageParam !== 'undefined' ? options.pageParam : 'page'; + options.pageSize = + typeof options.pageSize !== 'undefined' ? options.pageSize : 10; + let order = options.direction || 'DESC'; + let params = []; + const sortBy = 'ORDER BY first ASC'; // Default + let query = ' SELECT DISTINCT \ + `clcdc`.`account`.`id` AS `id`, \ + `clcdc`.`account`.`username` AS `username`, \ + `clcdc`.`account`.`email` AS `email`, \ + `clcdc`.`account`.`first` AS `first`, \ + `clcdc`.`account`.`last` AS `last`, \ + IF(ISNULL(`clcdc`.`account`.`name`), \ + `clcdc`.`account`.`first`, \ + `clcdc`.`account`.`name`) AS `name`, \ + `clcdc`.`account`.`ref` AS `ref`, \ + `clcdc`.`account`.`photo` AS `photo`, \ + `clcdc`.`account`.`start_date` AS `start_date`, \ + `clcdc`.`account`.`type_id` AS `type_id` \ + , account_profile.account_id \ + , count(DISTINCT profile_subject.subject_id) as pr \ + , ROUND(SUM(view_subject_stats.repi)/COUNT(DISTINCT profile_subject.subject_id),2) as newrepi \ + , SUM(percentage_read)/COUNT(DISTINCT profile_subject.subject_id) as percentage_read \ + , COUNT(DISTINCT view_subject_stats.subject_id) as `read` \ + , SUM(view_subject_stats.viewed) as viewed \ + , SUM(view_subject_stats.accessed) as accessed \ + , IF (((COUNT(DISTINCT subject.id) > 0) \ + AND (COUNT(DISTINCT view_subject_stats.subject_id) > 0)), \ + ROUND(((COUNT(DISTINCT view_subject_stats.subject_id) / COUNT(DISTINCT subject.id)) * 100), 1), 0) as reach \ + , ROUND(((((((IF((SUM(`view_subject_stats`.`entries_read`) > 0), \ + SUM(`view_subject_stats`.`entries_read`), \ + 0.0001) / IF((SUM(`view_subject_stats`.`entries_count`) > 0), \ + SUM(`view_subject_stats`.`entries_count`), \ + IF((SUM(`view_subject_stats`.`entries_read`) > 0), \ + SUM(`view_subject_stats`.`entries_read`), \ + 0.0001))) * 100) + ((COUNT(DISTINCT `view_subject_stats`.`subject_id`) / COUNT(DISTINCT `clcdc`.`subject`.`id`)) * 100)) / 2) + (SUM(`view_subject_stats`.`viewed`) * 0.01)) + (IF((SUM(`view_subject_stats`.`accessed`) > 0), \ + SUM(`view_subject_stats`.`accessed`), \ + 0.0001) * 0.1)), \ + 1) AS `repi` \ + '; + let whereParams = [this.id, _this.client_id, this.id, _this.client_id]; + if (subjectCount > 0) { + whereParams = [this.id, _this.client_id, this.id, _this.client_id, this.id, _this.client_id]; + } + let rank = null; if (options.rank) { rank = ', FIND_IN_SET(repi, ( \ SELECT GROUP_CONCAT(DISTINCT repi \ ORDER BY repi DESC ) \ - FROM view_repi_all \ - WHERE client_id = ? AND module_id is null \ + FROM view_repi \ + WHERE account_client_id = ? AND module_id is null \ + AND account_profile_id = ? \ + AND (profile_client_id = ? OR profile_client_id is null) \ )) AS rank '; query += rank; - whereParams.push(_this.client_id); + whereParams = whereParams.concat(whereParams); } - } - if (options.accountId) { - where += ' AND id = ? '; - whereParams.push(options.accountId); - } - if (options.searchText) { - where += - ' AND (username LIKE ? OR email LIKE ? or first LIKE ? or last LIKE ? or name LIKE ? or account.id LIKE ? or account.ref LIKE ?) '; - whereParams.push(options.searchText); - whereParams.push(options.searchText); - whereParams.push(options.searchText); - whereParams.push(options.searchText); - whereParams.push(options.searchText); - whereParams.push(options.searchText); - whereParams.push(options.searchText); - } - if (options.sortBy) { - switch (options.sortBy) { - case '1': // By reach - query = query + where; - query = query + groupBy; - query = - query + - ' ORDER BY reach ' + - order + - ', first ASC'; - params = whereParams; - break; - case '2': // By id - order = options.direction || 'ASC'; - query = query + where; - query = query + groupBy; - query = query + 'ORDER BY id ' + order; - params = whereParams; - break; - case '3': // By created - query = query + where; - query = query + groupBy; - query = query + 'ORDER BY start_date ' + order; - params = whereParams; - break; - case '4': // By repi - query = query + where; - query = query + groupBy; - query = - query + - 'ORDER BY repi ' + - order; - params = whereParams; - break; - default: - query = query + where; - query = query + groupBy; - query = query + sortBy; - params = whereParams; - break; + query += ' \ + FROM account_profile \ + INNER JOIN account ON account_profile.account_id = account.id \ + LEFT JOIN profile_subject ON account_profile.profile_id = profile_subject.profile_id \ + AND (profile_subject.profile_id = ? AND profile_subject.client_id = ?) \ + LEFT JOIN subject ON profile_subject.subject_id = subject.id ' + if (subjectCount === 0) { + query += 'AND \ + ( \ + subject.published_date < now() AND \ + subject.status_id = 3 AND \ + date is not null \ + ) '; } - } - else { - query = query + where; - query = query + groupBy; - query = query + sortBy; - params = whereParams; - } - options.params = []; - options.params['by'] = options.sortBy; - paginator.MyPaginator.configureInstance({ - pageSize: options.pageSize, - baseUrl: options.baseUrl, - pageParam: options.pageParam, - params: options.params, - customQuery: query, - customParameters: params - }); - paginator.MyPaginator.getPage(page, (obj) => { - done(null, obj); - }, (err) => { - if (err) - return done(err); + query += 'LEFT JOIN view_subject_stats ON \ + (account_profile.account_id = view_subject_stats.account_id AND view_subject_stats.subject_id = profile_subject.subject_id) \ + '; + let where = ' \ + WHERE \ + ( \ + account_profile.profile_id = ? AND \ + account_profile.client_id = ? \ + ) '; + if (subjectCount > 0) { + where += 'AND \ + ( \ + profile_subject.profile_id = ? AND \ + profile_subject.client_id = ? AND \ \ + subject.published_date < now() AND \ + subject.status_id = 3 AND \ + date is not null \ + ) '; + } + const groupBy = ' \ + GROUP BY account_profile.account_id, \ + username, email, `first`, `last`, `name`, ref, photo, start_date, type_id \ + '; + if (parseInt(this.id) === 0) { + // view_repi_all greatly simplifies the lookup code for this and provides for rank to be returned + // BUT it does not add any client_id/appId restriction to the program being returned + // - this won't impact most cases but will need to be looked at if we roll out shared programs and shared ids + // query = 'SELECT id, username, email, `first`, `last`, `name`, ref, photo, start_date, type_id, `read`, pr, entries_count, entries_read, percentage_read, reach, viewed, accessed, repi '; + where = ' \ + WHERE \ + ( \ + account_profile.client_id = ? \ + ) AND \ + ( \ + profile_subject.client_id = ? AND \ + subject.published_date < now() AND \ + subject.status_id = 3 AND \ + date is not null \ + ) \ + '; + whereParams = [_this.client_id, _this.client_id]; + if (options.rank) { + rank = ', FIND_IN_SET(repi, ( \ + SELECT GROUP_CONCAT(DISTINCT repi \ + ORDER BY repi DESC ) \ + FROM view_repi_all \ + WHERE client_id = ? AND module_id is null \ + )) AS rank '; + query += rank; + whereParams.push(_this.client_id); + } + } + if (options.accountId) { + where += ' AND id = ? '; + whereParams.push(options.accountId); + } + if (options.searchText) { + where += + ' AND (username LIKE ? OR email LIKE ? or first LIKE ? or last LIKE ? or name LIKE ? or account.id LIKE ? or account.ref LIKE ?) '; + whereParams.push(options.searchText); + whereParams.push(options.searchText); + whereParams.push(options.searchText); + whereParams.push(options.searchText); + whereParams.push(options.searchText); + whereParams.push(options.searchText); + whereParams.push(options.searchText); + } + if (options.sortBy) { + switch (options.sortBy) { + case '1': // By reach + query = query + where; + query = query + groupBy; + query = + query + + ' ORDER BY reach ' + + order + + ', first ASC'; + params = whereParams; + break; + case '2': // By id + order = options.direction || 'ASC'; + query = query + where; + query = query + groupBy; + query = query + 'ORDER BY id ' + order; + params = whereParams; + break; + case '3': // By created + query = query + where; + query = query + groupBy; + query = query + 'ORDER BY start_date ' + order; + params = whereParams; + break; + case '4': // By repi + query = query + where; + query = query + groupBy; + query = + query + + 'ORDER BY repi ' + + order; + params = whereParams; + break; + default: + query = query + where; + query = query + groupBy; + query = query + sortBy; + params = whereParams; + break; + } + } + else { + query = query + where; + query = query + groupBy; + query = query + sortBy; + params = whereParams; + } + options.params = []; + options.params['by'] = options.sortBy; + paginator.MyPaginator.configureInstance({ + pageSize: options.pageSize, + baseUrl: options.baseUrl, + pageParam: options.pageParam, + params: options.params, + customQuery: query, + customParameters: params + }); + paginator.MyPaginator.getPage(page, (obj) => { + done(null, obj); + }, (err) => { + if (err) + return done(err); + }); }); } getNonUsers(options, page, done) { @@ -728,6 +746,26 @@ class Audience { return done(null, null); }); } + getSubjectCount(profile_id, client_id, done) { + let sql = 'SELECT Count(id) as subject_count FROM subject \ + INNER JOIN profile_subject ON subject.id = profile_subject.subject_id \ + WHERE \ + ( \ + profile_subject.profile_id = ? AND \ + profile_subject.client_id = ? AND \ + subject.published_date < now() AND \ + subject.status_id = 3 AND \ + date is not null \ + ) \ + '; + let params = [profile_id, client_id]; + db.get().query(sql, params, (err, result) => { + if (err) return done(0); + if (result.length > 0) + return done(result[0].subject_count); + return done(0); + }) + } getSubjects(options, page, done) { const _this = this; options = options || {};