fix: migrate to utf8mb4

This commit migrates the database to UTF8MB4 in preparation for MySQL 8
which defaults to this.  It also produces better results for diverse
character sets.

Closes #2989.

server/lib/barcode.js

@@ -54,7 +54,7 @@ function reverseLookup(barcodeKey) {
 
   const query = `
     SELECT BUID(uuid) as uuid FROM ${documentDefinition.table}
-    WHERE BUID(uuid) LIKE '${partialUuid}%' COLLATE utf8_unicode_ci;
+    WHERE BUID(uuid) LIKE '${partialUuid}%';
   `;
 
   // search for full UUID

server/lib/db/index.js

@@ -3,13 +3,12 @@ const q = require('q');
 const mysql = require('mysql');
 const uuidParse = require('uuid-parse');
 const uuidv4 = require('uuid/v4');
-const Transaction = require('./transaction');
 const _ = require('lodash');
+const debug = require('debug')('db');
 
 const BadRequest = require('../errors/BadRequest');
 const NotFound = require('../errors/NotFound');
-
-const debug = require('debug')('db');
+const Transaction = require('./transaction');
 
 /**
  * @class DatabaseConnector
@@ -32,7 +31,7 @@ class DatabaseConnector {
       user     : process.env.DB_USER,
       password : process.env.DB_PASS,
       database : process.env.DB_NAME,
-      charset  : 'utf8_unicode_ci',
+      charset  : 'UTF8MB4_UNICODE_CI',
     };
 
     this.pool = mysql.createPool(params);
@@ -115,8 +114,8 @@ class DatabaseConnector {
   one(sql, params, id, entity = 'record') {
     return this.exec(sql, params)
       .then(rows => {
-        const errorMessage =
-          `Expected ${entity} to contain a single record with id ${id}, but ${rows.length} were found!`;
+        // eslint-disable-next-line max-len
+        const errorMessage = `Expected ${entity} to contain a single record with id ${id}, but ${rows.length} were found!`;
 
         if (rows.length < 1) {
           debug(`#one(): Found too few records!  Expected 1 but ${rows.length} found.`);

server/models/bhima.sql

@@ -2,9 +2,12 @@
 -- be overriden by individual enterprises however these are the defaults
 
 -- set variables
-SET names 'utf8';
-SET character_set_database = 'utf8';
-SET collation_database = 'utf8_unicode_ci';
+SET foreign_key_checks = 0;
+
+SET names 'utf8mb4';
+SET character_set_database = 'utf8mb4';
+SET collation_database = 'utf8mb4_unicode_ci';
+SET CHARACTER SET utf8mb4, CHARACTER_SET_CONNECTION = utf8mb4;
 
 -- units
 INSERT INTO unit VALUES

server/models/functions.sql

@@ -134,7 +134,7 @@ BEGIN
     SET accountType = 'expense';
   END IF;
 
-  SET accountTypeId = (SELECT id FROM account_type WHERE `type` = accountType COLLATE utf8_unicode_ci LIMIT 1);
+  SET accountTypeId = (SELECT id FROM account_type WHERE `type` = accountType COLLATE utf8mb4_unicode_ci LIMIT 1);
 
   RETURN accountTypeId;
 END

server/models/icd10.sql

@@ -28,7 +28,7 @@ CREATE TABLE `icd10` (
   `label` text NOT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `icd10_code` (`code`)
-) ENGINE=InnoDB AUTO_INCREMENT=16384 DEFAULT CHARSET=utf8mb4;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE = utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --

server/models/procedures.sql

@@ -2,6 +2,12 @@
   This file contains all the stored procedures used in bhima's database.  It
   should be loaded after functions.sql.
 */
+
+SET names 'utf8mb4';
+SET character_set_database = 'utf8mb4';
+SET collation_database = 'utf8mb4_unicode_ci';
+SET CHARACTER SET utf8mb4, CHARACTER_SET_CONNECTION = utf8mb4;
+
 DELIMITER $$
 
 /*

server/models/procedures/account.sql

@@ -28,31 +28,31 @@ BEGIN
   DECLARE IMPORT_DEFAULT_OHADA_ACCOUNT_OPTION TINYINT(1) DEFAULT 1;
 
   SET existAccount = (SELECT IF((SELECT COUNT(`number`) AS total FROM `account` WHERE `number` = accountNumber) > 0, 1, 0));
-  SET existAccountType = (SELECT IF((SELECT COUNT(*) AS total FROM `account_type` WHERE `type` = accountType COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET accountTypeId = (SELECT id FROM `account_type` WHERE `type` = accountType COLLATE utf8_unicode_ci LIMIT 1);
+  SET existAccountType = (SELECT IF((SELECT COUNT(*) AS total FROM `account_type` WHERE `type` = accountType COLLATE utf8mb4_unicode_ci) > 0, 1, 0));
+  SET accountTypeId = (SELECT id FROM `account_type` WHERE `type` = accountType COLLATE utf8mb4_unicode_ci LIMIT 1);
   SET existAccountParent = (SELECT IF((SELECT COUNT(*) AS total FROM `account` WHERE `number` = accountParent) > 0, 1, 0));
 
   SET accountLength = (SELECT CHAR_LENGTH(accountNumber));
 
-  /* 
+  /*
     Handle parent account for importing ohada list of accounts
     We assume that ohada main accounts are already loaded into the system
   */
   IF (existAccountParent = 1) THEN
     SET accountParentId = (SELECT id FROM `account` WHERE `number` = accountParent);
   END IF;
-
 
-  /* 
-    Create account if it doesn't exist 
+
+  /*
+    Create account if it doesn't exist
 
     if the account already exist skip because we are in a loop and
     we have to continue importing other accounts
   */
   IF (existAccount = 0 AND existAccountType = 1) THEN
     INSERT INTO `account` (`type_id`, `enterprise_id`, `number`, `label`, `parent`) VALUES (accountTypeId, enterpriseId, accountNumber, accountLabel, accountParentId);
 
-    /* 
+    /*
       Insert default accounts for a quick usage
 
       insert an child account if the option is default ohada and we have an account with four digit

server/models/procedures/inventory.sql

@@ -5,9 +5,8 @@
 
   This procedure import a new inventory into the system
   by creating one and performing a stock integration
-  if necessary
+  if necessary.
 */
-
 DROP PROCEDURE IF EXISTS ImportInventory;
 CREATE PROCEDURE ImportInventory (
   IN enterpriseId SMALLINT(5),
@@ -29,43 +28,45 @@ BEGIN
   DECLARE inventoryTypeId TINYINT(3);
   DECLARE inventoryUnitId SMALLINT(5);
 
-  SET existInventoryGroup = (SELECT IF((SELECT COUNT(`name`) AS total FROM `inventory_group` WHERE `name` = inventoryGroupName COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET existInventory = (SELECT IF((SELECT COUNT(`text`) AS total FROM `inventory` WHERE `code` = inventoryCode COLLATE utf8_unicode_ci OR `text` = inventoryText COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET existInventoryType = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_type` WHERE `text` = inventoryType COLLATE utf8_unicode_ci) > 0, 1, 0));
-  SET existInventoryUnit = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_unit` WHERE `text` = inventoryUnit COLLATE utf8_unicode_ci) > 0, 1, 0));
+  SET existInventoryGroup = (SELECT IF((SELECT COUNT(`name`) AS total FROM `inventory_group` WHERE `name` = inventoryGroupName) > 0, 1, 0));
+  SET existInventory = (SELECT IF((SELECT COUNT(`text`) AS total FROM `inventory` WHERE `code` = inventoryCode OR `text` = inventoryText) > 0, 1, 0));
+  SET existInventoryType = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_type` WHERE `text` = inventoryType) > 0, 1, 0));
+  SET existInventoryUnit = (SELECT IF((SELECT COUNT(*) AS total FROM `inventory_unit` WHERE `text` = inventoryUnit) > 0, 1, 0));
 
   /* Create group if doesn't exist */
   IF (existInventoryGroup = 0) THEN
     SET randomCode = (SELECT ROUND(RAND() * 10000000));
     SET inventoryGroupUuid = HUID(UUID());
     INSERT INTO `inventory_group` (`uuid`, `name`, `code`) VALUES (inventoryGroupUuid, inventoryGroupName, randomCode);
-  ELSE 
-    SET inventoryGroupUuid = (SELECT `uuid` FROM `inventory_group` WHERE `name` = inventoryGroupName COLLATE utf8_unicode_ci LIMIT 1);
+  ELSE
+    SET inventoryGroupUuid = (SELECT `uuid` FROM `inventory_group` WHERE `name` = inventoryGroupName LIMIT 1);
   END IF;
 
   /* Create type if doesn't exist */
-  IF (existInventoryType = 0) THEN 
+  IF (existInventoryType = 0) THEN
     SET inventoryTypeId = (SELECT MAX(`id`) + 1 FROM `inventory_type`);
     INSERT INTO `inventory_type` (`id`, `text`) VALUES (inventoryTypeId, inventoryType);
-  ELSE 
-    SET inventoryTypeId = (SELECT `id` FROM `inventory_type` WHERE LOWER(`text`) COLLATE utf8_unicode_ci = LOWER(inventoryType) LIMIT 1);
+  ELSE
+    SET inventoryTypeId = (SELECT `id` FROM `inventory_type` WHERE LOWER(`text`) = LOWER(inventoryType) LIMIT 1);
   END IF;
 
   /* Create unit if doesn't exist */
   IF (existInventoryUnit = 0) THEN
     SET inventoryUnitId = (SELECT MAX(`id`) + 1 FROM `inventory_unit`);
     INSERT INTO `inventory_unit` (`id`, `abbr`, `text`) VALUES (inventoryUnitId, inventoryUnit, inventoryUnit);
-  ELSE 
-    SET inventoryUnitId = (SELECT `id` FROM `inventory_unit` WHERE LOWER(`text`) COLLATE utf8_unicode_ci = LOWER(inventoryUnit) LIMIT 1);
+  ELSE
+    SET inventoryUnitId = (SELECT `id` FROM `inventory_unit` WHERE LOWER(`text`) = LOWER(inventoryUnit) LIMIT 1);
   END IF;
 
-  /* 
-    Create inventory if it doesn't exist 
+  /*
+    Create inventory if it doesn't exist
 
-    if the inventory already exist skip because we are in a loop and
+    If the inventory already exists, skip because we are in a loop and
     we have to continue importing other inventories
   */
   IF (existInventory = 0) THEN
-    INSERT INTO `inventory` (`enterprise_id`, `uuid`, `code`, `text`, `price`, `group_uuid`, `type_id`, `unit_id`) VALUES (enterpriseId, HUID(UUID()), inventoryCode, inventoryText, inventoryUnitPrice, inventoryGroupUuid, inventoryTypeId, inventoryUnitId);
+    INSERT INTO `inventory` (`enterprise_id`, `uuid`, `code`, `text`, `price`, `group_uuid`, `type_id`, `unit_id`)
+    VALUES
+    (enterpriseId, HUID(UUID()), inventoryCode, inventoryText, inventoryUnitPrice, inventoryGroupUuid, inventoryTypeId, inventoryUnitId);
   END IF;
 END $$