From d7ee60e06cfff90d10031cce77eda6d2c36b7825 Mon Sep 17 00:00:00 2001 From: bab2min Date: Tue, 17 Dec 2024 12:45:06 +0000 Subject: [PATCH] deploy: 93831053259a3fb227f4b0a78ccd06b3758e8b5e --- api/cpp/Kiwi.html | 9 ++++- classkiwi_1_1HSDataset-members.html | 51 +++++++++++++------------- classkiwi_1_1HSDataset.html | 9 +++-- classkiwi_1_1KiwiBuilder-members.html | 33 +++++++++-------- classkiwi_1_1KiwiBuilder.html | 9 +++-- genindex.html | 8 ++-- objects.inv | Bin 25004 -> 25091 bytes searchindex.js | 2 +- 8 files changed, 68 insertions(+), 53 deletions(-) diff --git a/api/cpp/Kiwi.html b/api/cpp/Kiwi.html index 0decf609..07d15305 100644 --- a/api/cpp/Kiwi.html +++ b/api/cpp/Kiwi.html @@ -1018,8 +1018,13 @@
-
-HSDataset makeHSDataset(const std::vector<std::string> &inputPathes, size_t batchSize, size_t causalContextSize, size_t windowSize, size_t numWorkers, double dropoutProb = 0, const TokenFilter &tokenFilter = {}, const TokenFilter &windowFilter = {}, double splitRatio = 0, bool separateDefaultMorpheme = false, const std::string &morphemeDefPath = {}, size_t morphemeDefMinCnt = 0, HSDataset *splitDataset = nullptr) const
+
+void convertHSData(const std::vector<std::string> &inputPathes, const std::string &outputPath, const std::string &morphemeDefPath = {}, size_t morphemeDefMinCnt = 0) const
+
+ +
+
+HSDataset makeHSDataset(const std::vector<std::string> &inputPathes, size_t batchSize, size_t causalContextSize, size_t windowSize, size_t numWorkers, double dropoutProb = 0, double dropoutProbOnHistory = 0, const TokenFilter &tokenFilter = {}, const TokenFilter &windowFilter = {}, double splitRatio = 0, bool separateDefaultMorpheme = false, const std::string &morphemeDefPath = {}, size_t morphemeDefMinCnt = 0, HSDataset *splitDataset = nullptr) const
diff --git a/classkiwi_1_1HSDataset-members.html b/classkiwi_1_1HSDataset-members.html index 570ee896..dff2f1d1 100644 --- a/classkiwi_1_1HSDataset-members.html +++ b/classkiwi_1_1HSDataset-members.html @@ -80,31 +80,32 @@

This is the complete list of members for kiwi::HSDataset, including all inherited members.

- - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + +
estimVocabFrequency() const (defined in kiwi::HSDataset)kiwi::HSDataset
getAugmentedSent(size_t idx) (defined in kiwi::HSDataset)kiwi::HSDataset
getBatchSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
getCausalContextSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
getKnlmVocabSize() const (defined in kiwi::HSDataset)kiwi::HSDataset
getSent(size_t idx) const (defined in kiwi::HSDataset)kiwi::HSDataset
getWindowSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
getWindowTokenValidness() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
HSDataset(size_t _batchSize=0, size_t _causalContextSize=0, size_t _windowSize=0, size_t _workers=0, double _dropoutProb=0) (defined in kiwi::HSDataset)kiwi::HSDataset
HSDataset(const HSDataset &)=delete (defined in kiwi::HSDataset)kiwi::HSDataset
HSDataset(HSDataset &&) (defined in kiwi::HSDataset)kiwi::HSDataset
KiwiBuilder (defined in kiwi::HSDataset)kiwi::HSDatasetfriend
next(int32_t *in, int32_t *out, float *lmLProbs, uint32_t *outNgramNode, float &restLmOut, uint32_t &restLmCntOut) (defined in kiwi::HSDataset)kiwi::HSDataset
next(int64_t *in, int64_t *out, float *lmLProbs, int64_t *outNgramNode, float &restLmOut, uint32_t &restLmCntOut) (defined in kiwi::HSDataset)kiwi::HSDataset
ngramNodeSize() const (defined in kiwi::HSDataset)kiwi::HSDataset
numEstimBatches() const (defined in kiwi::HSDataset)kiwi::HSDataset
numSents() const (defined in kiwi::HSDataset)kiwi::HSDataset
numTokens() const (defined in kiwi::HSDataset)kiwi::HSDataset
operator=(const HSDataset &)=delete (defined in kiwi::HSDataset)kiwi::HSDataset
operator=(HSDataset &&) (defined in kiwi::HSDataset)kiwi::HSDataset
reset() (defined in kiwi::HSDataset)kiwi::HSDataset
seed(size_t newSeed) (defined in kiwi::HSDataset)kiwi::HSDataset
vocabForm(uint32_t vocab) const (defined in kiwi::HSDataset)kiwi::HSDataset
vocabInfo(uint32_t vocab) const (defined in kiwi::HSDataset)kiwi::HSDataset
vocabSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
~HSDataset() (defined in kiwi::HSDataset)kiwi::HSDataset
extractPrefixes(size_t minCnt, size_t maxLength, size_t numWorkers=1) const (defined in kiwi::HSDataset)kiwi::HSDataset
getAugmentedSent(size_t idx) (defined in kiwi::HSDataset)kiwi::HSDataset
getBatchSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
getCausalContextSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
getKnlmVocabSize() const (defined in kiwi::HSDataset)kiwi::HSDataset
getSent(size_t idx) const (defined in kiwi::HSDataset)kiwi::HSDataset
getWindowSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
getWindowTokenValidness() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
HSDataset(size_t _batchSize=0, size_t _causalContextSize=0, size_t _windowSize=0, size_t _workers=0, double _dropoutProb=0, double _dropoutProbOnHistory=0) (defined in kiwi::HSDataset)kiwi::HSDataset
HSDataset(const HSDataset &)=delete (defined in kiwi::HSDataset)kiwi::HSDataset
HSDataset(HSDataset &&) (defined in kiwi::HSDataset)kiwi::HSDataset
KiwiBuilder (defined in kiwi::HSDataset)kiwi::HSDatasetfriend
next(int32_t *in, int32_t *out, float *lmLProbs, uint32_t *outNgramNode, float &restLmOut, uint32_t &restLmCntOut) (defined in kiwi::HSDataset)kiwi::HSDataset
next(int64_t *in, int64_t *out, float *lmLProbs, int64_t *outNgramNode, float &restLmOut, uint32_t &restLmCntOut) (defined in kiwi::HSDataset)kiwi::HSDataset
ngramNodeSize() const (defined in kiwi::HSDataset)kiwi::HSDataset
numEstimBatches() const (defined in kiwi::HSDataset)kiwi::HSDataset
numSents() const (defined in kiwi::HSDataset)kiwi::HSDataset
numTokens() const (defined in kiwi::HSDataset)kiwi::HSDataset
operator=(const HSDataset &)=delete (defined in kiwi::HSDataset)kiwi::HSDataset
operator=(HSDataset &&) (defined in kiwi::HSDataset)kiwi::HSDataset
reset() (defined in kiwi::HSDataset)kiwi::HSDataset
seed(size_t newSeed) (defined in kiwi::HSDataset)kiwi::HSDataset
vocabForm(uint32_t vocab) const (defined in kiwi::HSDataset)kiwi::HSDataset
vocabInfo(uint32_t vocab) const (defined in kiwi::HSDataset)kiwi::HSDataset
vocabSize() const (defined in kiwi::HSDataset)kiwi::HSDatasetinline
~HSDataset() (defined in kiwi::HSDataset)kiwi::HSDataset