feat: back-end implementation of ranked link seach (#210)

* feat: endpoint for url search

* fix: remove redundant log

* fix: inappropriate error message

* feat: rate limiting on search endpoint

* refactor: use table name from orm

* feat: hide link clicks from search response

* feat: support different search orders

* fix: update comments

* fix: imports

* fix: search order validation

* feat: add unit test for search controller

* fix: test request using wrong params

* feat: search ignores inactive links

* refactor: move stripping of clicks to service layer

* feat: additional tests for new methods

* feat: add more tests for textsearch

* refactor: remove redundant coalesce

* fix: error in sql statement for recency sort

* docs: add comment explaining ts_rank_cd normalization

* refactor: extract helper methods from search

* fix: packagelock

* fix: use more reasonable default limit

* fix: typo in documentation

* refactor: capitalize sql keywords

* fix: count including inactive urls and not using index

* feat: rate limit use real ip and logs when limit is reached

* fix: formatting

package.json

@@ -31,6 +31,7 @@
     "@hapi/joi": "^17.1.1",
     "@material-ui/core": "^4.10.2",
     "@sentry/browser": "^5.18.1",
+    "@types/express-rate-limit": "^5.0.0",
     "aws-sdk": "^2.706.0",
     "babel-polyfill": "^6.26.0",
     "bcrypt": "^5.0.0",
@@ -46,6 +47,7 @@
     "express-fileupload": "^1.1.7-alpha.3",
     "express-joi-validation": "^4.0.3",
     "express-session": "^1.17.1",
+    "express-rate-limit": "^5.1.3",
     "file-saver": "^2.0.2",
     "helmet": "^3.23.3",
     "history": "^4.10.1",

src/server/api/index.ts

@@ -9,6 +9,7 @@ router.use('/login', require('./login'))
 router.use('/stats', require('./statistics'))
 router.use('/sentry', require('./sentry'))
 router.use('/links', require('./links'))
+router.use('/search', require('./search'))
 
 /**
  * To protect private user routes.

src/server/api/search.ts

@@ -0,0 +1,43 @@
+import Express from 'express'
+import { createValidator } from 'express-joi-validation'
+import Joi from '@hapi/joi'
+import rateLimit from 'express-rate-limit'
+import { container } from '../util/inversify'
+import { DependencyIds } from '../constants'
+import { SearchControllerInterface } from '../controllers/interfaces/SearchControllerInterface'
+import { SearchResultsSortOrder } from '../repositories/enums'
+import getIp from '../util/request'
+import { logger } from '../config'
+
+const urlSearchRequestSchema = Joi.object({
+  query: Joi.string().required(),
+  order: Joi.string()
+    .required()
+    .allow(...Object.values(SearchResultsSortOrder))
+    .only(),
+  limit: Joi.number(),
+  offset: Joi.number(),
+})
+
+const apiLimiter = rateLimit({
+  keyGenerator: (req) => getIp(req) as string,
+  onLimitReached: (req) =>
+    logger.warn(`Rate limit reached for IP Address: ${getIp(req)}`),
+  windowMs: 1000, // 1 second
+  max: 20,
+})
+
+const router = Express.Router()
+const validator = createValidator()
+const searchController = container.get<SearchControllerInterface>(
+  DependencyIds.searchController,
+)
+
+router.get(
+  '/urls',
+  apiLimiter,
+  validator.query(urlSearchRequestSchema),
+  searchController.urlSearchPlainText,
+)
+
+module.exports = router

src/server/constants.ts

@@ -27,6 +27,8 @@ export const DependencyIds = {
   urlManagementService: Symbol.for('urlManagementService'),
   userController: Symbol.for('userController'),
   qrCodeService: Symbol.for('qrCodeService'),
+  urlSearchService: Symbol.for('urlSearchService'),
+  searchController: Symbol.for('searchController'),
   linkStatisticsController: Symbol.for('linkStatisticsController'),
   linkStatisticsService: Symbol.for('linkStatisticsService'),
   linkStatisticsRepository: Symbol.for('linkStatisticsRepository'),

src/server/controllers/SearchController.ts

@@ -0,0 +1,59 @@
+import { inject, injectable } from 'inversify'
+import Express from 'express'
+import { UrlSearchServiceInterface } from '../services/interfaces/UrlSearchServiceInterface'
+import { DependencyIds } from '../constants'
+import { logger } from '../config'
+import jsonMessage from '../util/json'
+import { SearchControllerInterface } from './interfaces/SearchControllerInterface'
+import { SearchResultsSortOrder } from '../repositories/enums'
+
+type UrlSearchRequest = {
+  query: string
+  order: string
+  limit?: number
+  offset?: number
+}
+
+@injectable()
+export class SearchController implements SearchControllerInterface {
+  private urlSearchService: UrlSearchServiceInterface
+
+  public constructor(
+    @inject(DependencyIds.urlSearchService)
+    urlSearchService: UrlSearchServiceInterface,
+  ) {
+    this.urlSearchService = urlSearchService
+  }
+
+  public urlSearchPlainText: (
+    req: Express.Request,
+    res: Express.Response,
+  ) => Promise<void> = async (req, res) => {
+    const {
+      query,
+      order,
+      limit = 100,
+      offset = 0,
+    } = req.query as UrlSearchRequest
+
+    try {
+      const { urls, count } = await this.urlSearchService.plainTextSearch(
+        query,
+        order as SearchResultsSortOrder,
+        limit,
+        offset,
+      )
+
+      res.ok({
+        urls,
+        count,
+      })
+      return
+    } catch (error) {
+      logger.error(`Error searching urls: ${error}`)
+      res.serverError(jsonMessage('Error retrieving URLs for search'))
+    }
+  }
+}
+
+export default SearchController

src/server/controllers/UserController.ts

@@ -150,8 +150,8 @@ export class UserController implements UserControllerInterface {
     res: Express.Response,
   ) => Promise<void> = async (req, res) => {
     const { userId } = req.body
-    let { limit = 10000, searchText = '' } = req.query
-    limit = Math.min(10000, limit)
+    let { limit = 1000, searchText = '' } = req.query
+    limit = Math.min(1000, limit)
     searchText = searchText.toLowerCase()
     const {
       offset = 0,

src/server/controllers/interfaces/SearchControllerInterface.ts

@@ -0,0 +1,5 @@
+import Express from 'express'
+
+export interface SearchControllerInterface {
+  urlSearchPlainText(req: Express.Request, res: Express.Response): Promise<void>
+}

src/server/db_migrations/20200619_add_search_index.sql

@@ -0,0 +1,12 @@
+-- This migration script adds the index for GoSearch
+
+DROP INDEX IF EXISTS urls_weighted_search_idx;
+
+-- Search will be run on a concatenation of vectors formed from short links and their
+-- description. Descriptions are given a lower weight than short links as short link
+-- words can be taken as the title and words there are likely to be more important than
+-- those in their corresponding description.
+-- Search queries will have to use this exact expresion to be able to utilize the index.
+CREATE INDEX urls_weighted_search_idx ON urls USING gin ((setweight(to_tsvector(
+'english', urls."shortUrl"), 'A') || setweight(to_tsvector('english',
+urls."description"), 'B'))) where urls.state = 'ACTIVE';

src/server/inversify.config.ts

@@ -29,6 +29,8 @@ import { LogoutController } from './controllers/LogoutController'
 import { UrlManagementService } from './services/UrlManagementService'
 import { UserController } from './controllers/UserController'
 import { QrCodeService } from './services/QrCodeService'
+import { SearchController } from './controllers/SearchController'
+import { UrlSearchService } from './services/UrlSearchService'
 import { LinkStatisticsController } from './controllers/LinkStatisticsController'
 import { LinkStatisticsService } from './services/LinkStatisticsService'
 import { LinkStatisticsRepository } from './repositories/LinkStatisticsRepository'
@@ -67,6 +69,8 @@ export default () => {
   bindIfUnbound(DependencyIds.urlManagementService, UrlManagementService)
   bindIfUnbound(DependencyIds.userController, UserController)
   bindIfUnbound(DependencyIds.qrCodeService, QrCodeService)
+  bindIfUnbound(DependencyIds.searchController, SearchController)
+  bindIfUnbound(DependencyIds.urlSearchService, UrlSearchService)
   bindIfUnbound(DependencyIds.deviceCheckService, DeviceCheckService)
 
   bindIfUnbound(