File tree Expand file tree Collapse file tree 2 files changed +7
-7
lines changed Expand file tree Collapse file tree 2 files changed +7
-7
lines changed Original file line number Diff line number Diff line change @@ -111,7 +111,7 @@ Logging Arguments
111
111
112
112
- ** git_hash** : str
113
113
114
- Default = 7d682df
114
+ Default = 6dd0344
115
115
116
116
current git hash of repository
117
117
Original file line number Diff line number Diff line change @@ -353,13 +353,13 @@ def eod(self):
353
353
class TiktokenTokenizer (AbstractTokenizer ):
354
354
"""Tokenizer from OpenAI's tiktoken implementation"""
355
355
356
- try :
357
- import tiktoken
358
- except ModuleNotFoundError :
359
- print ("Please install tiktoken: (https://github.com/openai/tiktoken)" )
360
- raise Exception
361
-
362
356
def __init__ (self , vocab_file ):
357
+ try :
358
+ import tiktoken
359
+ except ModuleNotFoundError :
360
+ print ("Please install tiktoken: (https://github.com/openai/tiktoken)" )
361
+ raise Exception
362
+
363
363
name = "TiktokenTokenizer"
364
364
super ().__init__ (name )
365
365
You can’t perform that action at this time.
0 commit comments