ext/rbs_extension/lexstate.c in rbs-3.6.0.dev.1 vs ext/rbs_extension/lexstate.c in rbs-3.6.0.pre.1

@@ -127,9 +127,25 @@
   }
 
   return t;
 }
 
+token next_eof_token(lexstate *state) {
+  if (state->current.byte_pos == RSTRING_LEN(state->string)+1) {
+    // End of String
+    token t;
+    t.type = pEOF;
+    t.range.start = state->start;
+    t.range.end = state->start;
+    state->start = state->current;
+
+    return t;
+  } else {
+    // NULL byte in the middle of the string
+    return next_token(state, pEOF);
+  }
+}
+
 void rbs_skip(lexstate *state) {
   if (!state->last_char) {
     peek(state);
   }
   int byte_len = rb_enc_codelen(state->last_char, rb_enc_get(state->string));