Skip to content
This repository has been archived by the owner on Jan 15, 2025. It is now read-only.

cross-train core library in bf-lu #601

Merged
merged 60 commits into from
Feb 28, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
d1dc10d
add qna cross training
feich-ms Jan 6, 2020
3242373
Revert "remove cross-train cli related changes and only keep the api"
feich-ms Jan 6, 2020
721245f
call qnaCrossTrain
feich-ms Jan 7, 2020
dbc0ec2
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 7, 2020
12a6c6a
add qna cross traini
feich-ms Jan 8, 2020
909a853
expose function
feich-ms Jan 8, 2020
306a557
optimize and add more corner test cases
feich-ms Jan 8, 2020
da83a88
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 8, 2020
d23892f
rename crossTrainer class
feich-ms Jan 8, 2020
9aac639
remove cli code pieces and optimize tests
feich-ms Jan 8, 2020
1d04faf
move cross-train.ts to lu folder
feich-ms Jan 9, 2020
368be78
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 13, 2020
063dabf
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 13, 2020
9ce4473
Merge branch 'feich/QnACrossTrain' of https://github.com/microsoft/bo…
feich-ms Jan 13, 2020
2bb3df7
optimize function headers and naming
feich-ms Jan 13, 2020
3650b57
typo: dialogname to dialogName
feich-ms Jan 13, 2020
8e9ef5c
add more test cases
feich-ms Jan 13, 2020
9c7f472
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 15, 2020
ec49691
remove unused '
feich-ms Jan 17, 2020
20acac9
add cross-train.ts cli file for test convenience and will remove it o…
feich-ms Jan 17, 2020
b8b4a6f
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 17, 2020
ab8eaf7
fix bugs
feich-ms Jan 17, 2020
5746690
Merge branch 'master' into feich/QnACrossTrain
feich-ms Jan 20, 2020
5a9e559
trigger ci
feich-ms Jan 20, 2020
d27e7cb
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 11, 2020
d975278
update qna cross train based on feedbacks
feich-ms Feb 12, 2020
b100a36
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 12, 2020
7f4901a
add config path parameter to specify the config file
feich-ms Feb 12, 2020
3d7bb4b
fix posttest failure
feich-ms Feb 12, 2020
6e31cdd
remove dup utterances in interuption intents
feich-ms Feb 13, 2020
f2a4ff1
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 13, 2020
df4fdd6
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 14, 2020
3bcb68c
remove dup 1
feich-ms Feb 14, 2020
e7af357
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 17, 2020
d2b0736
optimize all de-dup logic
feich-ms Feb 17, 2020
bb4160c
resolve all feedbacks
feich-ms Feb 18, 2020
125d01d
add test cases to test dedup functions
feich-ms Feb 18, 2020
ccdd91f
fix tslint errors
feich-ms Feb 18, 2020
52cbf82
add more friendly description for config CLI parameter
feich-ms Feb 18, 2020
857a114
fix config full path issue and remove uncessary loop validation
feich-ms Feb 19, 2020
01e592a
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 20, 2020
64674c2
move cross train CLI to a new package named cross-train
feich-ms Feb 20, 2020
d3db087
fix build error in CI validation
feich-ms Feb 20, 2020
1a1ddb1
add missing package in package.json
feich-ms Feb 20, 2020
9fa064a
remove unused file
feich-ms Feb 20, 2020
59bb856
fix typo
feich-ms Feb 20, 2020
700195d
optimize dedep logic and resolve feedbacks
feich-ms Feb 21, 2020
8bd1442
adjust test cases
feich-ms Feb 21, 2020
9e83d00
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 21, 2020
c7d54f2
support qna source, id, prompts reconstruction in cross training
feich-ms Feb 21, 2020
57c5a53
Merge branch 'master' into feich/QnACrossTrain
feich-ms Feb 24, 2020
19d29bb
typo: interuption to interruption
feich-ms Feb 24, 2020
2a37843
revert CLI changes
feich-ms Feb 25, 2020
5678af3
Merge branch 'master' into feich/QnACrossTrainCore
feich-ms Feb 25, 2020
29d39b9
minor fix
feich-ms Feb 25, 2020
1cfa404
Merge branch 'master' into feich/QnACrossTrainCore
vishwacsena Feb 25, 2020
1198603
Merge branch 'master' into feich/QnACrossTrainCore
feich-ms Feb 26, 2020
9288e8d
Merge branch 'master' into feich/QnACrossTrainCore
feich-ms Feb 27, 2020
a933210
Merge branch 'master' into feich/QnACrossTrainCore
feich-ms Feb 27, 2020
571d450
refactor cross train library tests
feich-ms Feb 27, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions packages/lu/src/parser/cross-train/cross-train.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*!
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License.
*/

const fs = require('fs-extra')
const path = require('path')
const file = require('../../utils/filehelper')
const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
const exception = require('../utils/exception')
const retCode = require('../utils/enums/CLI-errors');
const crossTrainer = require('./crossTrainer')

module.exports = {
/**
* Cross train lu and qna files.
* @param {string} input input lu and qna files folder.
* @param {string} intentName interruption intent name. Default value is _Interruption.
* @param {string} configPath path to config of mapping rules. If undefined, it will read config.json from input folder.
* @returns {luResult: any, qnaResult: any} trainedResult of luResult and qnaResult or undefined if no results.
*/
train: async function (input, intentName, configPath) {
let trainedResult

// Parse lu and qna objects
const luObjects = await file.getLuObjects(undefined, input, true, fileExtEnum.LUFile)
const qnaObjects = await file.getLuObjects(undefined, input, true, fileExtEnum.QnAFile)

let configObject
if (configPath && configPath !== '') {
configObject = await file.getConfigObject(configPath)
} else {
configObject = await file.getConfigObject(input)
}

if (configObject.rootIds.length > 0) {
let crossTrainConfig = {
rootIds: configObject.rootIds,
triggerRules: configObject.triggerRules,
intentName: intentName,
verbose: true
}

trainedResult = crossTrainer.crossTrain(luObjects, qnaObjects, JSON.stringify(crossTrainConfig))
} else {
throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'rootDialog property is required in config file'))
}

return trainedResult
},

/**
* Write lu and qna files
* @param {Map<string, any>} fileIdToLuResourceMap lu or qna file id to lu resource map.
* @param {string} out output folder name. If not specified, source lu and qna files will be updated.
* @throws {exception} Throws on errors.
*/
writeFiles: async function (fileIdToLuResourceMap, out) {
if (fileIdToLuResourceMap) {
let newFolder
if (out) {
newFolder = out
if (!path.isAbsolute(out)) {
newFolder = path.resolve(out)
}

if (!fs.existsSync(newFolder)) {
fs.mkdirSync(newFolder)
}
}

for (const fileId of fileIdToLuResourceMap.keys()) {
try {
if (newFolder) {
const fileName = path.basename(fileId)
const newFileId = path.join(newFolder, fileName)
await fs.writeFile(newFileId, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
} else {
await fs.writeFile(fileId, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
}
} catch (err) {
throw (new exception(retCode.errorCode.OUTPUT_FOLDER_INVALID, `Unable to write to file ${fileId}. Error: ${err.message}`))
}
}
}
}
}
Loading