add link to another relevant issue here

miknoj · Feb 19, 2024 · f7122fe · f7122fe
1 parent aac75c8
commit f7122fe
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/minbpe/gpt4.py b/minbpe/gpt4.py
@@ -31,6 +31,7 @@ def recover_merges(mergeable_ranks):
     # so we have to recover the original pairings. We can do this by doing
     # a small BPE training run on all the tokens, in their order.
     # also see https://github.com/openai/tiktoken/issues/60
+    # also see https://github.com/karpathy/minbpe/issues/11#issuecomment-1950805306
     merges = {}
     for token, rank in mergeable_ranks.items():
         if len(token) == 1: