lib/ai21/client.rb in ai21-0.2.0 vs lib/ai21/client.rb in ai21-0.2.1

@@ -46,9 +46,13 @@
 
     def answer(question, context)
       AI21::Client.post("/experimental/answer", {question: question, context: context})
     end
 
+    def tokenize(text)
+      AI21::Client.post("/tokenize", {text: text})
+    end
+
     def dataset
       @dataset ||= AI21::Dataset.new
     end
 
     def custom_model