deploy: 9383105

bab2min · Dec 17, 2024 · d7ee60e · d7ee60e
1 parent 754c24b
commit d7ee60e
Show file tree

Hide file tree

Showing 8 changed files with 68 additions and 53 deletions.
diff --git a/api/cpp/Kiwi.html b/api/cpp/Kiwi.html
@@ -1018,8 +1018,13 @@
 <dd></dd></dl>
 
 <dl class="cpp function">
-<dt class="sig sig-object cpp" id="_CPPv4NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tdRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset">
-<span id="_CPPv3NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tdRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset"></span><span id="_CPPv2NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tdRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset"></span><span id="kiwi::KiwiBuilder::makeHSDataset__std::vector:ss:CR.s.s.s.s.double.TokenFilterCR.TokenFilterCR.double.b.ssCR.s.HSDatasetPC"></span><span class="target" id="classkiwi_1_1KiwiBuilder_1a27ee3cb063d1ad59ed8ec7d5e928aa18"></span><span class="n"><span class="pre">HSDataset</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">makeHSDataset</span></span></span><span class="sig-paren">(</span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">inputPathes</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">causalContextSize</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">windowSize</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">numWorkers</span></span>, <span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dropoutProb</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N4kiwi11KiwiBuilder11TokenFilterE" title="kiwi::KiwiBuilder::TokenFilter"><span class="n"><span class="pre">TokenFilter</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tokenFilter</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N4kiwi11KiwiBuilder11TokenFilterE" title="kiwi::KiwiBuilder::TokenFilter"><span class="n"><span class="pre">TokenFilter</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">windowFilter</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">splitRatio</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">separateDefaultMorpheme</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">morphemeDefPath</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">morphemeDefMinCnt</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="n"><span class="pre">HSDataset</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">splitDataset</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tdRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset" title="Link to this definition"></a><br /></dt>
+<dt class="sig sig-object cpp" id="_CPPv4NK4kiwi11KiwiBuilder13convertHSDataERKNSt6vectorINSt6stringEEERKNSt6stringERKNSt6stringE6size_t">
+<span id="_CPPv3NK4kiwi11KiwiBuilder13convertHSDataERKNSt6vectorINSt6stringEEERKNSt6stringERKNSt6stringE6size_t"></span><span id="_CPPv2NK4kiwi11KiwiBuilder13convertHSDataERKNSt6vectorINSt6stringEEERKNSt6stringERKNSt6stringE6size_t"></span><span id="kiwi::KiwiBuilder::convertHSData__std::vector:ss:CR.ssCR.ssCR.sC"></span><span class="target" id="classkiwi_1_1KiwiBuilder_1adf87edd8d7cf5b4dfdb103ceda3eabe0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">convertHSData</span></span></span><span class="sig-paren">(</span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">inputPathes</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">outputPath</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">morphemeDefPath</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">morphemeDefMinCnt</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK4kiwi11KiwiBuilder13convertHSDataERKNSt6vectorINSt6stringEEERKNSt6stringERKNSt6stringE6size_t" title="Link to this definition"></a><br /></dt>
+<dd></dd></dl>
+
+<dl class="cpp function">
+<dt class="sig sig-object cpp" id="_CPPv4NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tddRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset">
+<span id="_CPPv3NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tddRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset"></span><span id="_CPPv2NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tddRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset"></span><span id="kiwi::KiwiBuilder::makeHSDataset__std::vector:ss:CR.s.s.s.s.double.double.TokenFilterCR.TokenFilterCR.double.b.ssCR.s.HSDatasetPC"></span><span class="target" id="classkiwi_1_1KiwiBuilder_1af7329fcb1068a9d14e06f697e8aab319"></span><span class="n"><span class="pre">HSDataset</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">makeHSDataset</span></span></span><span class="sig-paren">(</span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">inputPathes</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">causalContextSize</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">windowSize</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">numWorkers</span></span>, <span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dropoutProb</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dropoutProbOnHistory</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N4kiwi11KiwiBuilder11TokenFilterE" title="kiwi::KiwiBuilder::TokenFilter"><span class="n"><span class="pre">TokenFilter</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tokenFilter</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N4kiwi11KiwiBuilder11TokenFilterE" title="kiwi::KiwiBuilder::TokenFilter"><span class="n"><span class="pre">TokenFilter</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">windowFilter</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">splitRatio</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">separateDefaultMorpheme</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span>, <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">morphemeDefPath</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span>, <span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">morphemeDefMinCnt</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span>, <span class="n"><span class="pre">HSDataset</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">splitDataset</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK4kiwi11KiwiBuilder13makeHSDatasetERKNSt6vectorINSt6stringEEE6size_t6size_t6size_t6size_tddRK11TokenFilterRK11TokenFilterdbRKNSt6stringE6size_tP9HSDataset" title="Link to this definition"></a><br /></dt>
 <dd></dd></dl>
 
 </div>