diff --git a/.github/workflows/arm64_centos7.yml b/.github/workflows/arm64_centos7.yml index 5380fdf9..40d28e9b 100644 --- a/.github/workflows/arm64_centos7.yml +++ b/.github/workflows/arm64_centos7.yml @@ -34,15 +34,15 @@ jobs: - name: Test run: | ./build/test/kiwi-test - mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/ + mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/ cp -r build /artifacts/ cp -r eval_results /artifacts/ - name: Benchmark run: | curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt - KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt - name: Archive binaries uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/centos7.yml b/.github/workflows/centos7.yml index 6a152373..f381dca1 100644 --- a/.github/workflows/centos7.yml +++ b/.github/workflows/centos7.yml @@ -40,8 +40,8 @@ jobs: - name: Run Evaluator run: | mkdir eval_results - ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ - ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/ + ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ + ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/ - run: tar -zcvf arts.tgz build/*kiwi* build/test/*kiwi* eval_results/*.txt build/bindings/java/*.jar - name: Archive binaries uses: actions/upload-artifact@v3 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0a8d55f0..28cf8b7e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -60,24 +60,24 @@ jobs: - name: Run Evaluator run: | mkdir eval_results - ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ - ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/ + ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ + ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/ - name: Run Benchmark run: | curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt - ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt + ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt if [ "${{ matrix.arch }}" = "x86_64" ]; then - KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt - KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt + KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt + KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt else - KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt - KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt + KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt + KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt fi - name: Archive binaries uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ppc64le_centos7.yml b/.github/workflows/ppc64le_centos7.yml index 2837048b..310915c2 100644 --- a/.github/workflows/ppc64le_centos7.yml +++ b/.github/workflows/ppc64le_centos7.yml @@ -28,7 +28,7 @@ jobs: mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 .. make -j2 && popd ./build/test/kiwi-test - mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/ + mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/ cp -r build /artifacts/ cp -r eval_results /artifacts/ - name: Archive binaries diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1070c9f3..e3c6e1a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -114,7 +114,7 @@ jobs: asset_name: kiwi-java-${{ steps.get_release.outputs.tag_name }}-mac-${{ matrix.arch }}.jar asset_content_type: application/octet-stream - if: matrix.arch == 'arm64' - run: tar -zcvf model.tgz ModelGenerator/sj.* ModelGenerator/extract.mdl ModelGenerator/*.dict ModelGenerator/combiningRule.txt ModelGenerator/skipbigram.mdl + run: tar -zcvf model.tgz models/base/sj.* models/base/extract.mdl models/base/*.dict models/base/combiningRule.txt models/base/skipbigram.mdl - name: Upload release binary if: matrix.arch == 'arm64' uses: actions/upload-release-asset@v1.0.2 @@ -214,7 +214,7 @@ jobs: mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 .. make -j2 && popd ./build/test/kiwi-test - mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ + mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ cd build mkdir include && mkdir lib && mkdir bin mv libkiwi* lib/ diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e7235c25..f6b9dc44 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -60,20 +60,20 @@ jobs: - name: Run Evaluator run: | mkdir eval_results - ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ - ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/ + ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ + ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/ - name: Run Benchmark run: | curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt - ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt - KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt - KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt + ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt + KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt + KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt - name: Archive binaries uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a1123b4e..d828a746 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -35,8 +35,8 @@ jobs: - name: Run Evaluator run: | mkdir eval_results - .\build\Release\kiwi-evaluator.exe -m .\ModelGenerator (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\ - .\build\Release\kiwi-evaluator.exe -m .\ModelGenerator --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\ + .\build\Release\kiwi-evaluator.exe -m .\models\base (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\ + .\build\Release\kiwi-evaluator.exe -m .\models\base --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\ - name: Archive binaries uses: actions/upload-artifact@v3 with: @@ -59,21 +59,21 @@ jobs: - name: Run Benchmark run: | curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt $env:KIWI_ARCH_TYPE = 'none' - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt $env:KIWI_ARCH_TYPE = 'balanced' - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt $env:KIWI_ARCH_TYPE = 'sse2' - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt if ( '${{ matrix.arch }}' -eq 'x64' ) { $env:KIWI_ARCH_TYPE = 'sse4_1' - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt $env:KIWI_ARCH_TYPE = 'avx2' - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt $env:KIWI_ARCH_TYPE = 'avx512bw' - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt } - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out --sbg kowiki1000.txt - .\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out --typos 6 kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out --sbg kowiki1000.txt + .\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out --typos 6 kowiki1000.txt diff --git a/README.md b/README.md index 885827b7..4dbafb22 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ $ ldconfig 설치가 잘 됐는지 확인하기 위해서는 `kiwi-evaluator`를 실행해봅니다. ```console -$ ./kiwi-evaluator --model ../ModelGenerator ../eval_data/* --sbg +$ ./kiwi-evaluator --model ../models/base ../eval_data/* --sbg Loading Time : 981.745 ms ArchType : avx2 LM Size : 34.1853 MB @@ -110,7 +110,7 @@ Avg Score 0.13.0 버전부터 추가된 오타 교정 기능이 잘 작동하는지 확인하기 위해서는 다음과 같이 실행합니다. ```console -$ ./kiwi-evaluator --model ../ModelGenerator ../eval_data/* --sbg --typo 6 +$ ./kiwi-evaluator --model ../models/base ../eval_data/* --sbg --typo 6 Loading Time : 9414.45 ms ArchType : avx2 LM Size : 34.1853 MB diff --git a/bindings/java/README.md b/bindings/java/README.md index 14f95f77..6d0e9d0f 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -12,7 +12,7 @@ KiwiJava의 jar 파일은 자체적으로 실행 가능한 main함수를 가지 # 모델 파일 다운로드 $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.0/kiwi_model_v0.16.0_base.tgz -$ tar -zxvf kiwi_model_v0.16.0_base.tgz # 압축 해제. 모델을 포함한 ModelGenerator라는 폴더가 생성됨 +$ tar -zxvf kiwi_model_v0.16.0_base.tgz # 압축 해제. 모델을 포함한 models/base라는 폴더가 생성됨 # KiwiJava 다운로드 $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.1/kiwi-java-v0.16.1-lnx-x86-64.jar @@ -21,7 +21,7 @@ $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.1/kiwi-java-v0.16 # 즉, kiwi_model_v0.16.0은 kiwi-java-v0.16.* 전부에서 사용가능합니다. # jar 구동. 구동 인자로 모델 경로를 입력해주어야 함 -$ java -jar kiwi-java-v0.16.1-lnx-x86-64.jar ModelGenerator/ +$ java -jar kiwi-java-v0.16.1-lnx-x86-64.jar models/base/ Kiwi 0.16.1 is loaded! >> 안녕하세요? [Token(form=안녕, tag=NNG, position=0, length=2), diff --git a/bindings/java/kr/pe/bab2min/KiwiTest.java b/bindings/java/kr/pe/bab2min/KiwiTest.java index 6ee7dfa0..cb9b1815 100644 --- a/bindings/java/kr/pe/bab2min/KiwiTest.java +++ b/bindings/java/kr/pe/bab2min/KiwiTest.java @@ -11,7 +11,7 @@ public class KiwiTest { - private static String modelPath = "../../ModelGenerator"; + private static String modelPath = "../../models/base"; private static Kiwi reusableInst = null; Kiwi getReusableKiwi() throws Exception { diff --git a/bindings/wasm/package-demo/vite.config.ts b/bindings/wasm/package-demo/vite.config.ts index 66cbb960..5071fa08 100644 --- a/bindings/wasm/package-demo/vite.config.ts +++ b/bindings/wasm/package-demo/vite.config.ts @@ -3,7 +3,7 @@ import { viteStaticCopy } from 'vite-plugin-static-copy'; import { requiredModelFiles } from './src/modelFiles'; const targets = requiredModelFiles.map((file) => ({ - src: '../../../ModelGenerator/' + file, + src: '../../../models/base/' + file, dest: 'model', })); diff --git a/include/kiwi/capi.h b/include/kiwi/capi.h index 76dd282e..03854dfa 100644 --- a/include/kiwi/capi.h +++ b/include/kiwi/capi.h @@ -442,7 +442,7 @@ DECL_DLL int kiwi_typo_close(kiwi_typo_h handle); /** * @brief KiwiBuilder를 거치지 않고 바로 Kiwi instance를 생성합니다. * - * @param model_path 모델이 들어있는 디렉토리 경로 (e.g., ./ModelGenerator). + * @param model_path 모델이 들어있는 디렉토리 경로 (e.g., ./models/base). * @param num_threads 사용할 쓰레드의 수 (0일 경우, 자동으로 설정). * @param options 생성 옵션. KIWI_BUILD_* 참조. * @return Kiwi의 핸들. diff --git a/test/common.h b/test/common.h index e36bb567..37136bf3 100644 --- a/test/common.h +++ b/test/common.h @@ -4,7 +4,7 @@ #include #include -#define MODEL_PATH "./ModelGenerator" +#define MODEL_PATH "./models/base" #define KWORD u"킼윜" #define KWORD8 u8"킼윜" diff --git a/tools/evaluator_main.cpp b/tools/evaluator_main.cpp index ea4ee1c3..4913add8 100644 --- a/tools/evaluator_main.cpp +++ b/tools/evaluator_main.cpp @@ -125,7 +125,7 @@ int main(int argc, const char* argv[]) { CmdLine cmd{ "Kiwi evaluator" }; - ValueArg model{ "m", "model", "Kiwi model path", false, "ModelGenerator", "string" }; + ValueArg model{ "m", "model", "Kiwi model path", false, "models/base", "string" }; ValueArg output{ "o", "output", "output dir for evaluation errors", false, "", "string" }; SwitchArg noNormCoda{ "", "no-normcoda", "without normalizing coda", false }; SwitchArg noZCoda{ "", "no-zcoda", "without z-coda", false };