Sha256: 09625d61e81f3444eccebdf8ecc32d096435ffca8839ea16a124c78c5b828142
Contents?: true
Size: 505 Bytes
Versions: 17
Compression:
Stored size: 505 Bytes
Contents
# frozen_string_literal: true
class Tiktoken::Encoding
def self.method_missing(method)
Tiktoken::Encoding.new(Tiktoken::BpeFactory.send(method))
end
def initialize(ext_base_bpe)
@ext_base_bpe = ext_base_bpe
end
def encode_ordinary(text)
@ext_base_bpe.encode_ordinary(text)
end
def encode(text, allowed_special: [])
@ext_base_bpe.encode(text, allowed_special)
end
def decode(tokens)
@ext_base_bpe.decode(tokens)
end
end
Version data entries
17 entries across 17 versions & 1 rubygems