# encoding: utf-8 # frozen_string_literal: true require "strscan" require "carbon/compiler/scanner/main" require "carbon/compiler/scanner/token" module Carbon module Compiler # Scans over the file, taking a squence of characters and turning them into # tokens. class Scanner include Main include Enumerable # The file that the scanner will run over. # # @return [Project::File] attr_reader :file # Initialize the scanner. Takes a file, which contains information about # the file, and can read from the file. # # @param file [Project::File] def initialize(file) @file = file @scanner = StringScanner.new(@file.read) @line = 1 @last = 0 end # Pretty inspect. # # @return [void] def inspect "#<#{self.class} #{@file.name}>" end # Yields each token to the block, if one is given. Returns an # enumerator that enumerates over all generated tokens. # # @yield [token] # @yieldparam token [Token] # @return [Enumerator] def each return to_enum unless block_given? until @scanner.eos? token = scan yield token if token.is_a?(Token) end yield Token.new(:EOF, "", location) @scanner.reset end private def match(string, name = :"#{string}") if string.is_a?(::String) string = /#{::Regexp.escape(string.to_s)}/ elsif string.is_a?(::Symbol) string = /#{::Regexp.escape(string.to_s)}(?![\w])/ end if @scanner.scan(string) Token.new(name, @scanner[0], location(@scanner[0].length)) end end def location(length = 0) lines = @line..@line start = @scanner.charpos - @last columns = (start - length)..start Location.new(lines, columns, @file.short) end def error! @file.emit("Syntax/Token/Unknown", location, [@scanner.string[@scanner.charpos]]) end end end end