# File lib/NADOLFile.rb, line 184
  def NADOLTokenisedFile.tokenise(buffer)
   
    s=""
    buffer.each("\n") do |line|
      #trim the trailing \n
      line.chomp!
      #first set the high bit in each char
      for i in 0..line.length-1
      line[i]=line[i]|0x80
      end

      #now replace token strings with token number
      for i in 1..0x64
        token=" "*NADOL_EDITOR_TOKENS[i].length
        for j in 0..token.length-1
          token[j]=NADOL_EDITOR_TOKENS[i][j]|0x80
        end
        line.gsub!(token,i.chr)
      end

      #replace 2 to 15 spaces with a run-length-encoding
      0x0f.downto(2) do |i|
      line.gsub!(" "*i,(0x70+i).chr)
      end

      #now add length to each line
      s+=((line.length+2).chr)+line+"\0"
    end
    
    if ! NADOLTokenisedFile.can_be_nadol_tokenised_file?(s)
      raise "tokenisation appears to have failed!"
  end
  s
  end