根据总词频动态决定未登录词的默认词频

hankcs · Nov 5, 2021 · 6cff689 · 6cff689 · hanlpbot · Nov 5, 2021
1 parent 61cc753
commit 6cff689
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/src/main/java/com/hankcs/hanlp/seg/common/WordNet.java b/src/main/java/com/hankcs/hanlp/seg/common/WordNet.java
@@ -283,7 +283,7 @@ public void add(int line, List<AtomNode> atomSegment)
                     break;
             }
             // 这些通用符的量级都在10万左右
-            add(line + offset, new Vertex(sWord, atomNode.sWord, new CoreDictionary.Attribute(nature, 10000), id));
+            add(line + offset, new Vertex(sWord, atomNode.sWord, new CoreDictionary.Attribute(nature, Predefine.OOV_DEFAULT_FREQUENCY), id));
             offset += atomNode.sWord.length();
         }
     }

diff --git a/src/main/java/com/hankcs/hanlp/utility/Predefine.java b/src/main/java/com/hankcs/hanlp/utility/Predefine.java
@@ -56,6 +56,10 @@ public class Predefine
     public static int MAX_SEGMENT_NUM = 10;
 
     public static int TOTAL_FREQUENCY = 25146057; // 现在总词频25146057
+    /**
+     * 未登录词的默认词频
+     */
+    public static int OOV_DEFAULT_FREQUENCY = 10000;
     /**
      * Bigram 平滑因子
      */
@@ -141,5 +145,6 @@ public static void setTotalFrequency(int totalFrequency)
     {
         TOTAL_FREQUENCY = totalFrequency;
         myu = 1 - ((double) 1 / TOTAL_FREQUENCY + 0.00001);
+        OOV_DEFAULT_FREQUENCY = Math.max(1, Math.min(OOV_DEFAULT_FREQUENCY / 100, TOTAL_FREQUENCY)); // 默认百分之一
     }
 }