Contents

require "strscan"

class Lexer
  TOKENS = [
    /\(|\)/,          # parens
    /'/,              # quote
    /"(\\"|[^"])*"/,  # string literal
    /[^\s)]+/,        # any non-whitespace character excluding )
  ]

  def self.tokenize(program)
    new(program).tokenize
  end

  def initialize(program)
    @scanner = StringScanner.new(sanitize(program))
  end

  def tokenize
    tokens = []
    while (token = read_token)
      tokens << token
    end
    tokens
  end

  private

  attr_reader :scanner

  def read_token
    scanner.skip(/\s+/)
    return if scanner.eos?

    TOKENS.each do |token|
      return scanner.scan(token) if scanner.match?(token)
    end
  end

  def sanitize(string)
    # Remove comments
    string.gsub(/;.*$/, "")
  end
end

Version data entries

2 entries across 2 versions & 1 rubygems

Version	Path
lasp-0.10.1	lib/lasp/lexer.rb
lasp-0.10.0	lib/lasp/lexer.rb