Merge remote-tracking branch 'upstream/master'

trholding · Aug 5, 2023 · 44f2e96 · 44f2e96
2 parents 6ebb062 + 0609eb6
commit 44f2e96
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/model.py b/model.py
@@ -215,8 +215,8 @@ def __init__(self, params: ModelArgs):
         # share the unembedding parameters with the embedding parameters
         self.tok_embeddings.weight = self.output.weight # https://paperswithcode.com/method/weight-tying
 
-        # some useful precompute for the RoPE relative positional embeddings. TODO why * 2 here? confuse
-        freqs_cos, freqs_sin = precompute_freqs_cis(self.params.dim // self.params.n_heads, self.params.max_seq_len * 2)
+        # some useful precompute for the RoPE relative positional embeddings
+        freqs_cos, freqs_sin = precompute_freqs_cis(self.params.dim // self.params.n_heads, self.params.max_seq_len)
         self.register_buffer("freqs_cos", freqs_cos, persistent=False)
         self.register_buffer("freqs_sin", freqs_sin, persistent=False)