Skip to content

Commit

Permalink
Merge pull request #147 from Chia-Network/feat/fts-units
Browse files Browse the repository at this point in the history 
feat: fts params for units
  • Loading branch information
@MichaelTaylor3D
MichaelTaylor3D authored Jan 13, 2022
2 parents 3d5db62 + 3fa4ff9 commit ea69454
Show file tree
Hide file tree
Showing 8 changed files with 232 additions and 60 deletions.
8 changes: 6 additions & 2 deletions migrations/20211212200953-fulltext-search.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,22 @@ module.exports = {
await queryInterface.sequelize.query(`
CREATE VIRTUAL TABLE units_fts USING fts5(
warehouseUnitId,
countryJuridictionOfOwner,
orgUid,
unitOwnerOrgUid,
countryJurisdictionOfOwner,
inCountryJurisdictionOfOwner,
serialNumberBlock,
unitIdentifier,
unitType,
intendedBuyerOrgUid,
marketplace,
tags,
inCountryJuridictionOfOwner,
unitStatus,
unitTransactionType,
unitStatusReason,
tokenIssuanceHash,
marketplaceIdentifier,
unitsIssuanceLocation,
unitRegistryLink,
unitMarketplaceLink,
correspondingAdjustmentDeclaration,
Expand Down
32 changes: 24 additions & 8 deletions migrations/20211219184405-sqlite-triggers-units.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,44 @@ module.exports = {
CREATE TRIGGER unit_insert_fts AFTER INSERT ON units BEGIN
INSERT INTO units_fts(
warehouseUnitId,
countryJuridictionOfOwner,
orgUid,
unitOwnerOrgUid,
countryJurisdictionOfOwner,
inCountryJurisdictionOfOwner,
serialNumberBlock,
unitIdentifier,
unitType,
intendedBuyerOrgUid,
marketplace,
tags,
inCountryJuridictionOfOwner,
unitStatus,
unitTransactionType,
unitStatusReason,
tokenIssuanceHash,
marketplaceIdentifier,
unitsIssuanceLocation,
unitRegistryLink,
unitMarketplaceLink,
correspondingAdjustmentDeclaration,
correspondingAdjustmentStatus
) VALUES (
new.warehouseUnitId,
new.countryJuridictionOfOwner,
new.orgUid,
new.unitOwnerOrgUid,
new.countryJurisdictionOfOwner,
new.inCountryJurisdictionOfOwner,
new.serialNumberBlock,
new.unitIdentifier,
new.unitType,
new.intendedBuyerOrgUid,
new.marketplace,
new.tags,
new.inCountryJuridictionOfOwner,
new.unitStatus,
new.unitTransactionType,
new.unitStatusReason,
new.tokenIssuanceHash,
new.marketplaceIdentifier,
new.unitsIssuanceLocation,
new.unitRegistryLink,
new.unitMarketplaceLink,
new.correspondingAdjustmentDeclaration,
Expand All @@ -55,36 +63,44 @@ module.exports = {
DELETE FROM units_fts WHERE warehouseUnitId = old.warehouseUnitId;
INSERT INTO units_fts(
warehouseUnitId,
countryJuridictionOfOwner,
orgUid,
unitOwnerOrgUid,
countryJurisdictionOfOwner,
inCountryJurisdictionOfOwner,
serialNumberBlock,
unitIdentifier,
unitType,
intendedBuyerOrgUid,
marketplace,
tags,
inCountryJuridictionOfOwner,
unitStatus,
unitTransactionType,
unitStatusReason,
tokenIssuanceHash,
marketplaceIdentifier,
unitsIssuanceLocation,
unitRegistryLink,
unitMarketplaceLink,
correspondingAdjustmentDeclaration,
correspondingAdjustmentStatus
) VALUES (
new.warehouseUnitId,
new.countryJuridictionOfOwner,
new.orgUid,
new.unitOwnerOrgUid,
new.countryJurisdictionOfOwner,
new.inCountryJurisdictionOfOwner,
new.serialNumberBlock,
new.unitIdentifier,
new.unitType,
new.intendedBuyerOrgUid,
new.marketplace,
new.tags,
new.inCountryJuridictionOfOwner,
new.unitStatus,
new.unitTransactionType,
new.unitStatusReason,
new.tokenIssuanceHash,
new.marketplaceIdentifier,
new.unitsIssuanceLocation,
new.unitRegistryLink,
new.unitMarketplaceLink,
new.correspondingAdjustmentDeclaration,
Expand Down
37 changes: 23 additions & 14 deletions src/controllers/units.controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ export const create = async (req, res, next) => {
};

export const findAll = async (req, res) => {
let { page, limit, columns, orgUid } = req.query;
let { page, limit, columns, orgUid, search } = req.query;
let where = orgUid ? { orgUid } : undefined;

const includes = [Qualification];
Expand All @@ -69,19 +69,28 @@ export const findAll = async (req, res) => {
if (!columns.length) {
columns = ['warehouseUnitId'];
}

res.json(
optionallyPaginatedResponse(
await Unit.findAndCountAll({
where,
distinct: true,
...columnsToInclude(columns, includes),
...paginationParams(page, limit),
}),
page,
limit,
),
);

let results;

if (search) {
results = await Unit.fts(
search,
orgUid,
paginationParams(page, limit),
columns,
);
}

if (!results) {
results = await Unit.findAndCountAll({
where,
distinct: true,
...columnsToInclude(columns, includes),
...paginationParams(page, limit),
});
}

res.json(optionallyPaginatedResponse(results, page, limit));
};

export const findOne = async (req, res) => {
Expand Down
143 changes: 143 additions & 0 deletions src/models/units/units.model.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,29 @@ const virtualColumns = {
type: Sequelize.VIRTUAL,
get() {
const rawValue = this.getDataValue('serialNumberBlock');
if (!rawValue) {
return undefined;
}
return rawValue.split('-')[0];
},
},
unitBlockEnd: {
type: Sequelize.VIRTUAL,
get() {
const rawValue = this.getDataValue('serialNumberBlock');
if (!rawValue) {
return undefined;
}
return rawValue.split('-')[1];
},
},
unitCount: {
type: Sequelize.VIRTUAL,
get() {
const rawValue = this.getDataValue('serialNumberBlock');
if (!rawValue) {
return undefined;
}
const blocks = rawValue.split('-');
const blockStart = Number(blocks[0].split(/(\d+)/)[1]);
const blockEnd = Number(blocks[1].split(/(\d+)/)[1]);
Expand Down Expand Up @@ -86,6 +95,140 @@ class Unit extends Model {

return super.destroy(values);
}

static async fts(searchStr, orgUid, pagination, columns = []) {
const dialect = sequelize.getDialect();

const handlerMap = {
sqlite: Unit.findAllSqliteFts,
mysql: Unit.findAllMySQLFts,
};

// Check if we need to include the virtual field dep
for (const col of Object.keys(virtualColumns)) {
if (columns.includes(col)) {
if (!columns.includes('serialNumberBlock')) {
columns.push('serialNumberBlock');
}
break;
}
}

return handlerMap[dialect](
searchStr,
orgUid,
pagination,
columns
.filter((col) => ![
'createdAt',
'updatedAt',
'unitBlockStart',
'unitBlockEnd',
'unitCount'].includes(col))
);
}

static async findAllMySQLFts(searchStr, orgUid, pagination, columns = []) {
const { offset, limit } = pagination;

let fields = '*';
if (columns.length) {
fields = columns.join(', ');
}

let sql = `
SELECT ${fields} FROM units WHERE MATCH (
unitOwnerOrgUid,
countryJurisdictionOfOwner,
inCountryJurisdictionOfOwner,
serialNumberBlock,
unitIdentifier,
unitType,
intendedBuyerOrgUid,
marketplace,
tags,
unitStatus,
unitTransactionType,
unitStatusReason,
tokenIssuanceHash,
marketplaceIdentifier,
unitsIssuanceLocation,
unitRegistryLink,
unitMarketplaceLink,
cooresponingAdjustmentDeclaration,
correspondingAdjustmentStatus
) AGAINST ":search"
`;

if (orgUid) {
sql = `${sql} AND orgUid = :orgUid`;
}

const replacements = { search: searchStr, orgUid };

const count = (
await sequelize.query(sql, {
model: Unit,
mapToModel: true, // pass true here if you have any mapped fields
replacements,
})
).length;

if (limit && offset) {
sql = `${sql} ORDER BY relevance DESC LIMIT :limit OFFSET :offset`;
}

return {
count,
rows: await sequelize.query(sql, {
model: Unit,
replacements: { ...replacements, ...{ offset, limit } },
mapToModel: true, // pass true here if you have any mapped fields
offset,
limit,
}),
};
}

static async findAllSqliteFts(searchStr, orgUid, pagination, columns = []) {
const { offset, limit } = pagination;

let fields = '*';
if (columns.length) {
fields = columns.join(', ');
}

searchStr = searchStr = searchStr.replaceAll('-', '+');

let sql = `SELECT ${fields} FROM units_fts WHERE units_fts MATCH :search`;

if (orgUid) {
sql = `${sql} AND orgUid = :orgUid`;
}

const replacements = { search: `${searchStr}*`, orgUid };

const count = (
await sequelize.query(sql, {
model: Unit,
mapToModel: true, // pass true here if you have any mapped fields
replacements,
})
).length;

if (limit && offset) {
sql = `${sql} ORDER BY rank DESC LIMIT :limit OFFSET :offset`;
}

return {
count,
rows: await sequelize.query(sql, {
model: Unit,
mapToModel: true, // pass true here if you have any mapped fields
replacements: { ...replacements, ...{ offset, limit } },
}),
};
}
}

Unit.init(Object.assign({}, ModelTypes, virtualColumns), {
Expand Down
4 changes: 2 additions & 2 deletions src/models/units/units.modeltypes.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ module.exports = {
// organizations tables on the datalayer
orgUid: Sequelize.STRING,
unitOwnerOrgUid: Sequelize.STRING,
countryJuridictionOfOwner: Sequelize.STRING,
inCountryJuridictionOfOwner: Sequelize.STRING,
countryJurisdictionOfOwner: Sequelize.STRING,
inCountryJurisdictionOfOwner: Sequelize.STRING,
serialNumberBlock: Sequelize.STRING,
unitIdentifier: Sequelize.STRING,
unitType: Sequelize.STRING,
Expand Down
Loading

0 comments on commit ea69454

Please sign in to comment.