tokenizer_pos.rb in opener-chained-daemon-3.1.3

- old
+ new

@@ -41,11 +41,13 @@
         prod     = params[:cache_keys][:environment] != 'staging'
         if prod and !LANGUAGES_CACHE.get.include?(kaf.language)
           raise Core::UnsupportedLanguageError.new kaf.language
         end
 
-        response = Faraday.post BASE_URL, {lang: kaf.language, input: kaf.raw}.to_query
+        input    = kaf.raw
+        input    = input.gsub(/\,[^\ ]/, ', ')
+        response = Faraday.post BASE_URL, {lang: kaf.language, input: input}.to_query
         raise Core::UnsupportedLanguageError, kaf.language if response.status == 406
         raise response.body if response.status >= 400
         tokens   = JSON.parse response.body
 
         w_index = 0
@@ -60,17 +62,19 @@
               word['id'].each { |id| miscs[id] = word['misc'] }
               next
             end
             misc = word['misc'] || miscs[word['id']]
 
+            Rollbar.scoped({ input: input, params: params, tokens: tokens, word: word }) do
+              raise 'Missing misc'
+            end if misc.nil?
+
             offset = misc.match(/start_char=(\d+)|/)[1].to_i
             length = misc.match(/end_char=(\d+)/)[1].to_i - offset
 
             u_pos  = word['upos']
             pos    = POS[u_pos]
-            Rollbar.scoped({ input: input, params: params, tokens: tokens }) do
-              raise "Didn't find a map for #{u_pos}"
-            end if pos.nil?
+            raise "Didn't find a map for #{u_pos}" if pos.nil?
             type   = if POS_OPEN.include? pos then 'open' else 'close' end
 
             params = {
               wid:        w_index,
               sid:        s_index + 1,