ext/include/iv/lexer.h in iv-phonic-0.1.0 vs ext/include/iv/lexer.h in iv-phonic-0.1.1

@@ -36,18 +36,20 @@
         pos_(0),
         end_(source_->size()),
         has_line_terminator_before_next_(false),
         has_shebang_(false),
         line_number_(1),
+        previous_location_(),
         location_() {
     Initialize();
   }
 
   template<typename LexType>
   typename Token::Type Next(bool strict) {
     typename Token::Type token;
     has_line_terminator_before_next_ = false;
+    StorePreviousLocation();
     do {
       while (Chars::IsWhiteSpace(c_)) {
         // white space
         Advance();
       }
@@ -354,11 +356,15 @@
             token = Token::ILLEGAL;
           }
           break;
       }
     } while (token == Token::NOT_FOUND);
-    location_.set_end_position(pos() - 1);
+    if (c_ == -1) {
+      location_.set_end_position(pos());
+    } else {
+      location_.set_end_position(pos() - 1);
+    }
     return token;
   }
 
   inline const std::vector<uc16>& Buffer() const {
     return buffer16_;
@@ -416,10 +422,18 @@
 
   inline std::size_t end_position() const {
     return location_.end_position();
   }
 
+  inline std::size_t previous_begin_position() const {
+    return previous_location_.begin_position();
+  }
+
+  inline std::size_t previous_end_position() const {
+    return previous_location_.end_position();
+  }
+
   bool ScanRegExpLiteral(bool contains_eq) {
     // location begin_position is the same with DIV
     // so, no need to set
     bool character = false;
     buffer16_.clear();
@@ -469,11 +483,15 @@
         Record16(uc);
       } else {
         Record16Advance();
       }
     }
-    location_.set_end_position(pos() - 1);
+    if (c_ == -1) {
+      location_.set_end_position(pos());
+    } else {
+      location_.set_end_position(pos() - 1);
+    }
     return true;
   }
 
  private:
   static const std::size_t kInitialReadBufferCapacity = 32;
@@ -491,29 +509,43 @@
     has_shebang_ = false;
     line_number_ = 1;
     swap(location_, Location());
   }
 
+  inline void StorePreviousLocation() {
+    previous_location_ = location_;
+  }
+
   inline void Advance() {
     if (pos_ == end_) {
       c_ = -1;
     } else {
       c_ = source_->Get(pos_++);
     }
   }
+
   inline void Record8() {
     buffer8_.push_back(static_cast<char>(c_));
   }
+
   inline void Record8(const int ch) {
     buffer8_.push_back(static_cast<char>(ch));
   }
-  inline void Record16() { buffer16_.push_back(c_); }
-  inline void Record16(const int ch) { buffer16_.push_back(ch); }
+
+  inline void Record16() {
+    buffer16_.push_back(c_);
+  }
+
+  inline void Record16(const int ch) {
+    buffer16_.push_back(ch);
+  }
+
   inline void Record8Advance() {
     Record8();
     Advance();
   }
+
   inline void Record16Advance() {
     Record16();
     Advance();
   }
 
@@ -877,9 +909,10 @@
   const std::size_t end_;
   bool has_line_terminator_before_next_;
   bool has_shebang_;
   int c_;
   std::size_t line_number_;
+  Location previous_location_;
   Location location_;
 };
 
 
 } }  // namespace iv::core