Refactor model path

.github/workflows/arm64_centos7.yml

@@ -34,15 +34,15 @@ jobs:
           - name: Test
             run: |
               ./build/test/kiwi-test
-              mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
+              mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
               cp -r build /artifacts/
               cp -r eval_results /artifacts/
           - name: Benchmark
             run: |
               curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
-              KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-              KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-              KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
+              KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+              KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+              KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
     - name: Archive binaries
       uses: actions/upload-artifact@v3
       with:

.github/workflows/centos7.yml

@@ -40,8 +40,8 @@ jobs:
           - name: Run Evaluator
             run: |
               mkdir eval_results
-              ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
-              ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
+              ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
+              ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
           - run: tar -zcvf arts.tgz build/*kiwi* build/test/*kiwi* eval_results/*.txt build/bindings/java/*.jar
     - name: Archive binaries
       uses: actions/upload-artifact@v3

.github/workflows/macos.yml

@@ -60,24 +60,24 @@ jobs:
     - name: Run Evaluator
       run: |
         mkdir eval_results
-        ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
-        ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
     - name: Run Benchmark
       run: |
         curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
-        ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
+        ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
         if [ "${{ matrix.arch }}" = "x86_64" ]; then
-          KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-          KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-          KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-          KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt
-          KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
+          KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+          KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+          KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+          KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt
+          KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt
         else
-          KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-          KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt
-          KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
+          KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+          KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt
+          KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt
         fi
     - name: Archive binaries
       uses: actions/upload-artifact@v3

.github/workflows/ppc64le_centos7.yml

@@ -28,7 +28,7 @@ jobs:
           mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 ..
           make -j2 && popd
           ./build/test/kiwi-test
-          mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
+          mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
           cp -r build /artifacts/
           cp -r eval_results /artifacts/
     - name: Archive binaries

.github/workflows/release.yml

@@ -114,7 +114,7 @@ jobs:
         asset_name: kiwi-java-${{ steps.get_release.outputs.tag_name }}-mac-${{ matrix.arch }}.jar
         asset_content_type: application/octet-stream
     - if: matrix.arch == 'arm64'
-      run: tar -zcvf model.tgz ModelGenerator/sj.* ModelGenerator/extract.mdl ModelGenerator/*.dict ModelGenerator/combiningRule.txt ModelGenerator/skipbigram.mdl
+      run: tar -zcvf model.tgz models/base/sj.* models/base/extract.mdl models/base/*.dict models/base/combiningRule.txt models/base/skipbigram.mdl
     - name: Upload release binary
       if: matrix.arch == 'arm64'
       uses: actions/[email protected]
@@ -214,7 +214,7 @@ jobs:
           mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 ..
           make -j2 && popd
           ./build/test/kiwi-test
-          mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
+          mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
           cd build
           mkdir include && mkdir lib && mkdir bin
           mv libkiwi* lib/

.github/workflows/ubuntu.yml

@@ -60,20 +60,20 @@ jobs:
     - name: Run Evaluator
       run: |
         mkdir eval_results
-        ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
-        ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
     - name: Run Benchmark
       run: |
         curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
-        ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
-        KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt
-        KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
+        ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
+        KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt
+        KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt
     - name: Archive binaries
       uses: actions/upload-artifact@v3
       with:

.github/workflows/windows.yml

@@ -35,8 +35,8 @@ jobs:
     - name: Run Evaluator
       run: |
         mkdir eval_results
-        .\build\Release\kiwi-evaluator.exe -m .\ModelGenerator (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
-        .\build\Release\kiwi-evaluator.exe -m .\ModelGenerator --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
+        .\build\Release\kiwi-evaluator.exe -m .\models\base (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
+        .\build\Release\kiwi-evaluator.exe -m .\models\base --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
     - name: Archive binaries
       uses: actions/upload-artifact@v3
       with:
@@ -59,21 +59,21 @@ jobs:
     - name: Run Benchmark
       run: |
         curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
-        .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+        .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
         $env:KIWI_ARCH_TYPE = 'none'
-        .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+        .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
         $env:KIWI_ARCH_TYPE = 'balanced'
-        .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+        .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
         $env:KIWI_ARCH_TYPE = 'sse2'
-        .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+        .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
         if ( '${{ matrix.arch }}' -eq 'x64' )
         {
           $env:KIWI_ARCH_TYPE = 'sse4_1'
-          .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+          .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
           $env:KIWI_ARCH_TYPE = 'avx2'
-          .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+          .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
           $env:KIWI_ARCH_TYPE = 'avx512bw'
-          .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
+          .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
         }
-        .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out --sbg kowiki1000.txt
-        .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
+        .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out --sbg kowiki1000.txt
+        .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out --typos 6 kowiki1000.txt

README.md

@@ -80,7 +80,7 @@ $ ldconfig
 
 설치가 잘 됐는지 확인하기 위해서는 `kiwi-evaluator`를 실행해봅니다.
 ```console
-$ ./kiwi-evaluator --model ../ModelGenerator ../eval_data/* --sbg
+$ ./kiwi-evaluator --model ../models/base ../eval_data/* --sbg
 Loading Time : 981.745 ms
 ArchType : avx2
 LM Size : 34.1853 MB
@@ -110,7 +110,7 @@ Avg Score
 
 0.13.0 버전부터 추가된 오타 교정 기능이 잘 작동하는지 확인하기 위해서는 다음과 같이 실행합니다.
 ```console
-$ ./kiwi-evaluator --model ../ModelGenerator ../eval_data/* --sbg --typo 6
+$ ./kiwi-evaluator --model ../models/base ../eval_data/* --sbg --typo 6
 Loading Time : 9414.45 ms
 ArchType : avx2
 LM Size : 34.1853 MB

bindings/java/README.md

@@ -12,7 +12,7 @@ KiwiJava의 jar 파일은 자체적으로 실행 가능한 main함수를 가지
 
 # 모델 파일 다운로드
 $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.0/kiwi_model_v0.16.0_base.tgz
-$ tar -zxvf kiwi_model_v0.16.0_base.tgz # 압축 해제. 모델을 포함한 ModelGenerator라는 폴더가 생성됨
+$ tar -zxvf kiwi_model_v0.16.0_base.tgz # 압축 해제. 모델을 포함한 models/base라는 폴더가 생성됨
 
 # KiwiJava 다운로드
 $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.1/kiwi-java-v0.16.1-lnx-x86-64.jar
@@ -21,7 +21,7 @@ $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.1/kiwi-java-v0.16
 # 즉, kiwi_model_v0.16.0은 kiwi-java-v0.16.* 전부에서 사용가능합니다.
 
 # jar 구동. 구동 인자로 모델 경로를 입력해주어야 함
-$ java -jar kiwi-java-v0.16.1-lnx-x86-64.jar ModelGenerator/
+$ java -jar kiwi-java-v0.16.1-lnx-x86-64.jar models/base/
 Kiwi 0.16.1 is loaded!
 >> 안녕하세요?
 [Token(form=안녕, tag=NNG, position=0, length=2),

bindings/java/kr/pe/bab2min/KiwiTest.java

@@ -11,7 +11,7 @@
 
 public class KiwiTest {
 
-	private static String modelPath = "../../ModelGenerator";
+	private static String modelPath = "../../models/base";
 	private static Kiwi reusableInst = null;
 
 	Kiwi getReusableKiwi() throws Exception {

bindings/wasm/package-demo/vite.config.ts

@@ -3,7 +3,7 @@ import { viteStaticCopy } from 'vite-plugin-static-copy';
 import { requiredModelFiles } from './src/modelFiles';
 
 const targets = requiredModelFiles.map((file) => ({
-    src: '../../../ModelGenerator/' + file,
+    src: '../../../models/base/' + file,
     dest: 'model',
 }));
 

include/kiwi/capi.h

@@ -442,7 +442,7 @@ DECL_DLL int kiwi_typo_close(kiwi_typo_h handle);
 /**
  * @brief KiwiBuilder를 거치지 않고 바로 Kiwi instance를 생성합니다.
  * 
- * @param model_path 모델이 들어있는 디렉토리 경로 (e.g., ./ModelGenerator).
+ * @param model_path 모델이 들어있는 디렉토리 경로 (e.g., ./models/base).
  * @param num_threads 사용할 쓰레드의 수 (0일 경우, 자동으로 설정).
  * @param options 생성 옵션. KIWI_BUILD_* 참조.
  * @return Kiwi의 핸들.

test/common.h

@@ -4,7 +4,7 @@
 #include <iostream>
 #include <fstream>
 
-#define MODEL_PATH "./ModelGenerator"
+#define MODEL_PATH "./models/base"
 #define KWORD u"킼윜"
 #define KWORD8 u8"킼윜"
 

test/test_cpp.cpp

@@ -365,7 +365,7 @@ TEST(KiwiCpp, HSDataset)
 {
 	KiwiBuilder kw{ MODEL_PATH, 0, BuildOption::default_, };
 	std::vector<std::string> data;
-	data.emplace_back(MODEL_PATH "/testHSDataset.txt");
+	data.emplace_back("./ModelGenerator/testHSDataset.txt");
 
 	static constexpr size_t batchSize = 32, windowSize = 8;
 

tools/evaluator_main.cpp

@@ -125,7 +125,7 @@ int main(int argc, const char* argv[])
 {
 	CmdLine cmd{ "Kiwi evaluator" };
 
-	ValueArg<string> model{ "m", "model", "Kiwi model path", false, "ModelGenerator", "string" };
+	ValueArg<string> model{ "m", "model", "Kiwi model path", false, "models/base", "string" };
 	ValueArg<string> output{ "o", "output", "output dir for evaluation errors", false, "", "string" };
 	SwitchArg noNormCoda{ "", "no-normcoda", "without normalizing coda", false };
 	SwitchArg noZCoda{ "", "no-zcoda", "without z-coda", false };