require 'external/base'

# for some reason this is sometimes not required by String 
# automatically, leading to a missing each_char method
require 'jcode'  

#--
# not implemented --
# dclone, flatten, flatten!, frozen?, pack, quote, to_yaml, transpose, yaml_initialize
#
# be careful accession io directly.  for peformance reasons there is no check to make
# sure io is in register (ie pos is at a frame boundary, ie io.length % frame_size == 0)
# In addition, note that length must be adjusted manually in most io operations (truncate is
# the exception).  Thus if you change the file length by any means, the file length must be
# reset.
#
# ExtInd allows array-like access to formatted binary data stored on disk.  
#   
# == Caching
#
# To improve peformance, ExtInd can be run in a cached mode where the data is loaded into
# memory and kept in memory until the ExtInd closes (or is flushed).  Cached mode is 
# recommended for all but the largest index files, which cannot or should not be loaded
# into memory.  
#++
class ExtInd < External::Base
  
  class << self
    def [](*args)
      options = args.last.kind_of?(Hash) ? args.pop : {}
      ab = self.new(nil, options)
      normalized_args = args.collect {|item| item.nil? ? ab.nil_value : item }.flatten
      ab.unframed_write(normalized_args)
      
      # reset the position of the IO under this initialize
      ab.pos = 0
      ab
    end
    
    def read(fd, options={})
      return [] if fd.nil?
      open(fd, "r", options) do |index|
        index.read(nil, 0)
      end
    end
    
    # Returns the number of bytes required to pack an item in an array
    # using the directive (see Array.pack for more details).  All directives
    # return a size except the positioning directives '@' and 'X'; these
    # and all other unknown directives return nil.
    #
    #   Directives          N bytes
    #   ------------------------------
    #   AaBbCcHhUwxZ     |   1
    #   nSsv             |   2
    #   M                |   3
    #   eFfgIiLlNPpV     |   4
    #   m                |   5
    #   u                |   6
    #   DdEGQq           |   8
    #   @X               |   nil
    def directive_size(directive)
      case directive
      when /^[eFfgIiLlNPpV]$/ then 4
      when /^[DdEGQq]$/ then 8
      when /^[AaBbCcHhUwxZ]$/ then 1
      when /^[nSsv]$/ then 2
      when 'M' then 3
      when 'm' then 5
      when 'u' then 6
      else
        nil
      end
    end
    
    attr_writer :default_nil_value
    def default_nil_value
      @default_nil_value ||= 0
    end
  end
  
  attr_reader :frame, :frame_size, :format, :cache, :process_in_bulk
  
  def initialize(io=nil, options={})
    super(io)
    
    options = {
      :format => "I",
      :nil_value => nil,
      :cached => false,
      :buffer_size => 8 * 2**20     # 8Mb
    }.merge(options)
  
    # set the format, frame, and frame size
    @format = options[:format]
    @frame = @format.length
    @frame_size = 0
    
    # TODO -- allow specification of numbers in the format
    @format.each_char do |directive|
      size = ExtInd.directive_size(directive)
      raise ArgumentError.new("cannot determine size of: '#{directive}'") if size == nil
      @frame_size += size
    end
    @process_in_bulk = false
    
    # set the buffer size
    self.buffer_size = options[:buffer_size]

    # Repetitive formats like "I", "II", "III", 
    # etc can be packed and unpacked in bulk
    directive = @format[0, 1]
    if @format == (directive * @frame)
      @format = "#{directive}*"
      @process_in_bulk = true
    end

    # set the nil value to an array of zeros, or
    # to the specified nil value.  If a nil value
    # was specified, ensure it is of the correct
    # frame size and can be packed
    nil_value = if options[:nil_value] == nil 
      Array.new(@frame, self.class.default_nil_value) 
    else
      options[:nil_value]
    end
    
    begin 
      @nil_value = nil_value.pack(@format)
      raise "" unless nil_value.length == @frame && @nil_value.unpack(@format) == nil_value
    rescue
      raise ArgumentError.new(
         "unacceptable nil value '#{nil_value}': the nil value must " +
         "be in frame and packable using the format '#{format}'")
    end
    
    self.cached = options[:cached]
  end
  
  def buffer_size
    self.io.default_blksize
  end
  
  def buffer_size=(buffer_size)
    raise ArgumentError.new("buffer size must be > 0") if buffer_size <= 0
    
    @default_blksize = (buffer_size/frame_size).ceil
    self.io.default_blksize = buffer_size
  end
  
  def default_blksize=(value)
    @default_blksize = value
    self.io.default_blksize = value * frame_size
  end
  
  def options
    {:format => (process_in_bulk ? format.chomp("*") * frame : format),
    :nil_value => nil_value,
    :cached => cached?,
    :buffer_size => buffer_size}
  end
  
  # Returns the string value used for nils.  Specify unpacked to 
  # show the unpacked array value.
  #
  #   i = ExtInd.new 
  #   i.nil_value            # => [0]
  #   i.nil_value(false)     # => "\000\000\000\000"
  def nil_value(unpacked=true)
    unpacked ? @nil_value.unpack(format) : @nil_value
  end
  
  # True if cached 
  def cached?
    cache != nil
  end
  
  # Sets the index to cache data or not.  When setting cached to 
  # false, currently cached data is flushed.
  def cached=(input)
    if input && !cache
      @cache_pos = self.pos
      @cache = read(nil, 0)
      
      # ensure the cache is an array of framed items...
      # if io has only one item, then read returns an 
      # array like [0] rather than [[0]]
      unless @cache.empty? || @cache.first.kind_of?(Array)
        @cache = [@cache] 
      end
    elsif !input && cache
      flush
      @cache = nil
      self.pos = @cache_pos
      @cache_pos = nil
    end
  end
  
  # Flushes the io, writing cached data if necessary.
  def flush
    if cached?
      io.truncate(0)
      cache.each {|item| io.write item.pack(format) }
    end
      
    io.flush
    io.reset_length
  end
   
  # Flushes cached data and closes the io.
  def close
    cached = false if cached?
    super
  end

  ###########################
  # Array methods
  ###########################

  # def &(another)
  #   not_implemented
  # end

  # def *(arg)
  #   not_implemented
  # end
  
  def dup
    self.flush
    ExtInd.new(nil, options).concat(self)
  end

  def +(another)
    dup.concat(another)
  end

  # def -(another)
  #   not_implemented
  # end

  def <<(array)
    # WRONG BEHAVIOR -- should be write, push in frame
    unframed_write(array, length)
    self
  end
  
  def <=>(another)
    return 0 if self.object_id == another.object_id
    
    # reverse comparison in case another is an ExtInd
    return -1 * (another <=> cache) if cached?

    case another
    when Array
      if another.length < self.length
        # if another is equal to the matching subset of self,
        # then self is obviously the longer array and wins.
        result = (self.to_a(another.length) <=> another)
        result == 0 ? 1 : result
      else
        self.to_a <=> another
      end
    when ExtInd
      self.io.sort_compare(another.io, (buffer_size/2).ceil)
    else
      raise TypeError.new("can't convert from #{another.class} to ExtInd or Array")
    end
  end

  def ==(another)
    return true if super
    
    case another
    when Array
      return false unless self.length == another.length
      self.to_a == another
    when ExtInd
      return false unless self.length == another.length
      
      unless self.cached? && another.cached?
        return false unless self.index_attrs == another.index_attrs
        if (self.io.sort_compare(another.io, (buffer_size/2).ceil)) == 0
          return true
        end
      end
      
      self.to_a == another.to_a
    else
      false
    end      
  end

  # Element Reference — Returns the entry at index, or returns an array starting 
  # at start and continuing for length entries, or returns an array specified 
  # by range. Negative indices count backward from the end of self (-1 is the last 
  # element). Returns nil if the index (or starting index) is out of range.
  #
  #   io = StringIO.new [1,2,3,4,5].pack("I*")
  #   i = ExtInd.new(io, :format => 'I')
  #   i[2]                   #=> [3]
  #   i[6]                   #=> nil
  #   i[1, 2]                #=> [ [2], [3] ]
  #   i[1..3]                #=> [ [2], [3], [4] ]
  #   i[4..7]                #=> [ [5] ]
  #   i[6..10]               #=> nil
  #   i[-3, 3]               #=> [ [3], [4], [5] ]
  #   # special cases
  #   i[5]                   #=> nil
  #   i[5, 1]                #=> []
  #   i[5..10]               #=> []
  #
  # Note that entries are returned in frame, as arrays.
  def [](index, length=nil)
    # return the cached value if cached
    return (length == nil ? cache[index] : cache[index,length]) if cached?
    
    case index
    when Fixnum
      index += self.length if index < 0
      return nil if index < 0

      unless length == nil
        raise TypeError.new("no implicit conversion from nil to integer") if length.nil?
        return [] if length == 0 || index >= self.length
        return nil if length < 0
        
        # ensure you don't try to read more entries than are available
        max_length = self.length - index
        length = max_length if length > max_length
      end
      
      case 
      when length == nil then read(1, index)    # read one, as index[0]
      when length == 1 then [read(1, index)]    # read one framed, as index[0,1]
      else
        read(length, index)                     # read length, automatic framing
      end 
      
    when Range
      raise TypeError.new("can't convert Range into Integer") unless length == nil
  
      offset, length = split_range(index)
  
      # for conformance with array range retrieval
      return nil if offset < 0 || offset > self.length
      return [] if length < 0
      
      self[offset, length + 1]
    when nil
      raise TypeError.new("no implicit conversion from nil to integer")
    else
      raise TypeError.new("can't convert #{index.class} into Integer") 
    end
  end

  # Element Assignment — Sets the entry at index, or replaces a subset starting at start
  # and continuing for length entries, or replaces a subset specified by range.
  # A negative indices will count backward from the end of self. Inserts elements if 
  # length is zero. If nil is used in the second and third form, deletes elements from 
  # self. An IndexError is raised if a negative index points past the beginning of self. 
  # See also push, and unshift.
  #
  #   io = StringIO.new ""
  #   i = ExtInd.new(io, :format => 'I')
  #   i.nil_value                  # => [0]
  #   i[4] = [4]                   # => [[0], [0], [0], [0], [4]]
  #   i[0, 3] = [ [1], [2], [3] ]  # => [[1], [2], [3], [0], [4]]
  #   i[1..2] = [ [5], [6] ]       # => [[1], [5], [6], [0], [4]]
  #   i[0, 2] = [ [7] ]            # => [[7], [6], [0], [4]]
  #   i[0..2] = [ [8] ]            # => [[8], [4]]
  #   i[-1]   = [9]                # => [[8], [9]]
  #   i[1..-1] = nil               # => [[8]]
  #
  # Note that []= must take entries in frame, or (in the case of [offset, length] and 
  # range insertions) another ExtInd with the same frame, format, and nil_value.
  #--
  # TODO -- cleanup error messages so they are more meaningful 
  # and helpful, esp for frame errors
  #++
  def []=(*args)
    raise ArgumentError.new("wrong number of arguments (1 for 2)") if args.length < 2
    index, length, value = args
    if args.length == 2
      value = length 
      length = nil
    end

    case index
    when Fixnum
      if index < 0
        index += self.length
        raise IndexError.new("index #{index} out of range") if index  < 0
      end
      
      if length == nil
        # simple insertion 
        value = nil_value if value.object_id == 4 # nil
        unframed_write(value, index)
      else
        raise IndexError.new("negative length (#{length})") if length < 0
        
        # arrayify value if needed
        unless value.kind_of?(ExtInd)
          value = [value] unless value.kind_of?(Array)
        end
        
        case
        when cached?
          # validation must occur here, because this cached insertion
          # bypasses the validations that normally occur in write
          case value
          when Array then validate_framed_array(value)
          when ExtInd then validate_index(value)
          end
          
          # must be done before padding in case value == self
          # WARN - could be expensive 
          # TODO - check the effect of cache.dup on speed if cached?
          value = value.to_a.collect {|item| item == nil ? nil_value : item }
          
          # pad as needed
          pad_to(index) if index > self.length
          
          # write the value to the cache
          cache[index, length] = value
        when self == value
          # special case when insertion is self (no validation needed)
          # A whole copy of self is required because the insertion 
          # can overwrite the tail of self.  As such this can be a
          # worst-case scenario-slow and expensive procedure.
          copy_beg = (index + length) * frame_size
          copy_end = io.length
          
          io.copy do |copy|
            # truncate io
            io.truncate(index * frame_size)
            io.pos = io.length
            
            # pad as needed
            pad_to(index) if index > self.length
            
            # write the copy of self
            io.insert(copy)
            
            # copy the tail of the insertion
            io.insert(copy, copy_beg..copy_end)
          end
        when value.length == length
          # optimized insertion, when insertion is the correct length
          write(value, index)
        else
          # range insertion: requires copy and rewrite of the tail 
          # of the ExtInd, after the insertion.
          # WARN - can be slow when the tail is large
          copy_beg = (index + length) * frame_size
          copy_end = io.length
          
          io.copy("r", copy_beg..copy_end) do |copy|
            # pad as needed
            pad_to(index) if index > self.length
            
            # write inserted value
            io.pos = index * frame_size
            write(value)
            
            # truncate io
            io.truncate(io.pos)

            # copy the tail of the insertion
            io.insert(copy)
          end
        end
      end
      
      value
    when Range
      raise TypeError.new("can't convert Range into Integer") if args.length == 3 
      
      # for conformance with setting a range with nil (truncates)
      value = [] if value.nil?
      offset, length = split_range(index)
      self[offset, length + 1] = value
    when nil
      raise TypeError.new("no implicit conversion from nil to integer")
    else
      raise TypeError.new("can't convert #{index.class} into Integer") 
    end
  end

  # def abbrev(pattern=nil)
  #   not_implemented
  # end

  # def assoc(obj)
  #   not_implemented
  # end

  # Returns entry at index
  def at(index)
    self[index]
  end

  # Removes all elements from _self_.
  def clear
    cached? ? cache.clear : io.truncate(0)
    self
  end

  # def compact
  #   not_implemented
  # end

  # def compact!
  #   not_implemented
  # end

  def concat(another)
    case another
    when Array
      write(another, length)
    when ExtInd 
      validate_index(another)
      
      if cached?
        # WARN - could be expensive
        cache.concat(another.to_a)
      else
        io.concat(another.io)
      end
    else 
      raise TypeError.new("can't convert #{another.class} into ExtInd or Array")
    end
    self
  end

  # def delete(obj)
  #   not_implemented
  # end

  # def delete_at(index)
  #   not_implemented
  # end

  # def delete_if # :yield: item
  #   not_implemented
  # end

  def each(&block) # :yield: item
    self.pos = 0
    chunk do |offset, length|
      # special treatment for 1, because then read(1) => [...] rather
      # than [[...]].  when frame > 1, each will iterate over the 
      # element rather than pass it to the block directly
      if length == 1
        yield read(1)
      else
        read(length).each(&block)
      end
    end
    self
  end

  # Passes the index of each char
  def each_index(&block) # :yield: index
    0.upto(length-1, &block)
    self
  end

  # Returns true if _self_ contains no elements
  def empty?
    length == 0
  end

  def eql?(another)
    self == another
  end 

  # def fetch(index, default=nil, &block)
  #   index += index_length if index < 0 
  #   val = (index >= length ? default : self[index])
  #   block_given? ? yield(val) : val
  # end

  # def fill(*args)
  #   not_implemented
  # end

  # Returns the first n entries (default 1)
  def first(n=nil)
    n.nil? ? self[0] : self[0,n]
  end

  # def hash
  #   not_implemented
  # end

  # def include?(obj)
  #   not_implemented
  # end

  # def index(obj)
  #   not_implemented
  # end

  # def indexes(*args)
  #   values_at(*args)
  # end
  # 
  # def indicies(*args)
  #   values_at(*args)
  # end

  # def replace(other)
  #   not_implemented
  # end

  # def insert(index, *obj)
  #   self[index] = obj
  # end

  # def inspect
  #   not_implemented
  # end

  # def join(sep=$,)
  #   not_implemented
  # end

  # Returns the last n entries (default 1)
  def last(n=nil)
    return self[-1] if n.nil?
  
    start = length-n
    start = 0 if start < 0
    self[start, n]
  end

  # Returns the number of entries in self
  def length
    cached? ? cache.length : io.length/frame_size
  end

  # def nitems
  #   not_implemented
  # end

  # def pop
  #   not_implemented
  # end

  # def pretty_print(q)
  #   not_implemented
  # end

  # def pretty_print_cycle(q)
  #   not_implemented
  # end

  # def push(*obj)
  #   not_implemented
  # end

  # def rassoc(key)
  #   not_implemented
  # end

  # def replace(another)
  #   not_implemented
  # end

  # def reverse
  #   not_implemented
  # end

  # def reverse!
  #   not_implemented
  # end

  def reverse_each(&block)
    reverse_chunk do |offset, length|
      # special treatment for 1, because then read(1) => [...] rather
      # than [[...]].  when frame > 1, each will iterate over the 
      # element rather than pass it to the block directly
      if length == 1
        yield read(1)
      else
        read(length, offset).reverse_each(&block)
      end
    end
    self
  end

  # def rindex(obj)
  #   not_implemented
  # end

  # def select # :yield: item
  #   not_implemented
  # end

  # def shift
  #   not_implemented
  # end

  # Alias for length
  def size
    length
  end

  # def slice(*args)
  #   self.call(:[], *args)
  # end

  # def slice!(*args)
  #   not_implemented
  # end

  def to_a(length=self.length)
    case
    when cached? then cache.dup
    when length == 0 then []
    when length == 1 then [read(length, 0)]
    else
      read(length, 0)
    end
  end

  # def to_ary
  #   not_implemented
  # end

  # Returns _self_.join.
  # def to_s
  #   self.join
  # end

  # def uniq
  #   not_implemented
  # end

  # def uniq!
  #   not_implemented
  # end

  # def unshift(*obj)
  #   not_implemented
  # end

  # Returns an array containing the chars in io corresponding to the given
  # selector(s). The selectors may be either integer indices or ranges
  # def values_at(*selectors)
  #   selectors.collect {|s| self[s]}.flatten
  # end

  # def |(another)
  #   not_implemented
  # end
  
  #################
  # IO-like methods
  ##################
  
  # Sets the current position of the index.  Negative positions
  # are counted from the end of the index (just as they are in
  # an array).  Positions can be set beyond the actual length
  # of the index (similar to an IO).
  #
  #   i = ExtInd[[1],[2],[3]]
  #   i.length                      # => 3
  #   i.pos = 2; i.pos              # => 2
  #   i.pos = -1; i.pos             # => 2
  #   i.pos = 10; i.pos             # => 40
  def pos=(pos)
    if pos < 0
      raise ArgumentError.new("position out of bounds: #{pos}") if pos < -length
      pos += length 
    end
    
    # do something fake for caching so that 
    # the position need not be set (this 
    # works either way)
    if cached? 
      self.cache_pos = pos
    else
      io.pos = (pos * frame_size)
    end
  end
  
  # Returns the current position of the index
  def pos
    cached? ? cache_pos : io.pos/frame_size
  end

  # Reads the packed byte string for n entries from the specified 
  # position. By default reads the string for all remaining entries
  # from the current position.
  # 
  #   i = ExtInd[[1],[2],[3]]
  #   i.pos                              # => 0
  #   i.readbytes.unpack("I*")           # => [1,2,3]
  #   i.readbytes(1,0).unpack("I*")      # => [1]
  #   i.readbytes(10,1).unpack("I*")     # => [2,3]
  #
  # Like an IO, when n is nil and no entries can be read, an empty
  # string is returned.  When n is specified, nil will be returned
  # when no entries can be read.
  #
  #   i.pos = 3
  #   i.readbytes                        # => ""
  #   i.readbytes(1)                     # => nil
  def readbytes(n=nil, pos=nil)
    if cached?
      ary = read(n, pos)  
      return (ary == nil ? nil : ary.flatten.pack(format))
    end
  
    # set the io position to the specified index
    self.pos = pos unless pos == nil

    # read until the end if no n is given
    n == nil ? io.read : io.read(n * frame_size)
  end
  
  # Unpacks the given string into an array of index values.
  # Single entries are returned in frame, multiple entries 
  # are returned in an array.
  #
  #   i.format                          # => 'I*'
  #   i.unpack( [1].pack('I*') )        # => [1] 
  #   i.unpack( [1,2,3].pack('I*') )    # => [[1],[2],[3]]
  #   i.unpack("")                      # => []
  #   
  def unpack(str)
    case
    when str.empty? then []
    when str.length == frame_size 
      str.unpack(format)
    when process_in_bulk
      results = []
      str.unpack(format).each_slice(frame) {|s| results << s}
      results
    else
      Array.new(str.length/frame_size) do |i|
        str[i*frame_size, frame_size].unpack(format)
      end
    end
  end

  # Reads n entries from the specified position. By default 
  # reads all remaining entries from the current position.
  # Single entries are returned in frame, multiple entries 
  # are returned in an array.
  #
  #   i = ExtInd[[1],[2],[3]]
  #   i.pos                       # => 0
  #   i.read                      # => [[1],[2],[3]]
  #   i.read(1,0)                 # => [1]
  #   i.read(10,1)                # => [[2],[3]]
  # 
  # When n is nil and no entries can be read, an empty array
  # is returned.  When n is specified, nil will be returned
  # when no entries can be read.
  #
  #   i.pos = 3
  #   i.read                      # => []
  #   i.read(1)                   # => nil       
  def read(n=nil, pos=nil)
    if cached?
      self.pos = pos unless pos == nil
      m = (n == nil || n > (length - cache_pos)) ? (length - cache_pos) : n
      
      return case 
      when n == nil && m == 0 then []
      when m <= 1 then cache[cache_pos] 
      else
        cache[cache_pos, m]
      end
    end
    
    str = readbytes(n, pos)
    str == nil ? nil : unpack(str)
  end
  
  def write(array, pos=nil)
    case array
    when Array
      validate_framed_array(array)
      prepare_write_to_pos(pos)
      write_framed_array(array)
    when ExtInd
      validate_index(array)
      prepare_write_to_pos(pos)
      write_index(array)
    else  
      raise ArgumentError.new("could not convert #{array.class} to Array or ExtInd")
    end
  end
  
  # Writes the array as an entry (or set of entries) into
  # self starting at the specified position.  By default
  # write begins at the current position.  The array can
  # have multiple entries in sequence, but MUST be in the 
  # correct frame.
  #
  #   i = ExtInd[]
  #   i.unframed_write([2,3], 1)
  #   i.pos = 0; 
  #   i.unframed_write([1])
  #   i.read(3, 0)                # => [[1],[2],[3]]
  #
  # Note -- no range checking when cached
  def unframed_write(array, pos=nil)
    case array
    when Array
      validate_unframed_array(array)
      prepare_write_to_pos(pos)
      write_unframed_array(array)
    when ExtInd
      validate_index(array)
      prepare_write_to_pos(pos)
      write_index(array)
    else  
      raise ArgumentError.new("could not convert #{array.class} to Array or ExtInd")
    end
  end
  
  protected
  
  attr_accessor :cache_pos
  
  # An array of the core index attributes: frame, format, nil_value
  def index_attrs # :nodoc:
    [frame, format, nil_value]
  end
   
  def prepare_write_to_pos(pos)
    unless pos == nil
      # pad to the starting position if necessary
      pad_to(pos) if pos > length
    
      # set the io position to the specified index
      self.pos = pos
    end
  end
  
  def pad_to(pos)
    n = (pos-length)/frame
      
    if cached?
      cache.concat(Array.new(n, nil_value))
    else
      io.pos = io.length
      io.length += io.write(nil_value(false) * n) 
        
      # in this case position doesn't 
      # need to be set.  set pos to nil
      # to skip the set statement below
      pos = nil
    end
  end

  def validate_index(index)
    unless index.index_attrs == index_attrs
      raise ArgumentError.new("incompatible index attributes [#{index.index_attrs.join(',')}]") 
    end
  end
  
  def validate_framed_array(array)
    array.each do |item| 
      case item
      when Array
        unless item.length == frame
          raise ArgumentError.new("expected array in frame '#{frame}' but was '#{item.length}'") 
        end
      when nil 
        # framed arrays can contain nils
        next
      else
        raise ArgumentError.new("expected array in frame '#{frame}', was #{item.class}")
      end
    end
  end
  
  def validate_unframed_array(array)
    unless array.length % frame == 0
      raise ArgumentError.new("expected array in frame '#{frame}' but was '#{array.length}'") 
    end
  end
  
  def write_index(index)
    if cached?
      if index.cached?
        cache[cache_pos, index.length] = index.cache
        self.cache_pos += index.length
      else
        index.each do |item|
          cache[cache_pos] = item
          self.cache_pos += 1
        end
      end
    else
      end_pos = io.pos
      if index.cached?
        end_pos += io.write( index.cache.pack(format) )
      else
        end_pos += io.insert(index.io)
      end
      
      io.length = end_pos if end_pos > io.length
    end
  end
  
  def write_framed_array(array)
    # framed arrays may contain nils, and must
    # be resolved before writing the data
    
    if cached?
      cache[cache_pos, array.length] = array.collect {|item| item == nil ? nil_value : item }
      self.cache_pos += array.length
    else
      start_pos = io.pos
      length_written = 0
      
      if process_in_bulk
        arr = []
        array.each {|item| arr.concat(item == nil ? nil_value : item) }
        length_written += io.write(arr.pack(format))
      else
        array.each do |item|
          str = (item == nil ? nil_value(false) : item.pack(format))
          length_written += io.write(str)
        end
      end
    
      end_pos = start_pos + length_written
      io.length = end_pos if end_pos > io.length
    end
  end
  
  def write_unframed_array(array)
    # unframed arrays cannot contain nils
    
    if cached?
      array.each_slice(frame) do |item|
        cache[cache_pos] = item
        self.cache_pos += 1
      end
    else
      start_pos = io.pos
      length_written = 0
      
      if process_in_bulk
        length_written += io.write(array.pack(format))
      else
        array.each_slice(frame) do |arr|
          length_written += io.write(arr.pack(format))
        end
      end
    
      end_pos = start_pos + length_written
      io.length = end_pos if end_pos > io.length
    end
  end
end


# # Include the inline enhancements for ExtInd
# if RUBY_PLATFORM.index('mswin').nil?
#   require 'inline'
#   inline do |builder|
#     #builder.include "<rubyio.h>"
#     # Array.new(str.length/frame_size) do |i|
#     #     str[i*frame_size, frame_size].unpack(format)
#     #   end
#     builder.c %Q{
#       static VALUE unpack(VALUE str)
#       {
#         char *p = RSTRING(str)->ptr;
#         int str_len = RSTRING(str)->len; 
#         int frame_size = NUM2INT(rb_iv_get(self, "@frame_size")); 
#         int frame = NUM2INT(rb_iv_get(self, "@frame")); 
#         int i, j, times = str_len/frame_size;
#         VALUE fmt = rb_iv_get(self, "@format"); 
#         VALUE results, arr;
# 
#         if(times <= 1)
#           return rb_funcall(str, rb_intern("unpack"), 1, fmt);
# 
#         results = rb_ary_new();
#         i = 0;
#         while(i < times)
#         {
#           j = 0;
#           arr = rb_ary_new();
#           while(j < frame)
#           {
#             // no need to copy the data at *p,
#                 // apparently the conversion can 
#                 // happen directly from the pointer
#                 rb_ary_push(arr, UINT2NUM(*p));
#                 p += 4;
#        
#             
#             ++j;
#           }
# 
#           rb_ary_push(results, arr);
#           ++i;
#         }
#         return results;
#       }    
#       
#     }#File.read(File.dirname(__FILE__) + "/../../src/inline.c")
#   end
# else
#   # on windows when it's not likely that the user has
#   # a compiler, include the precompiled binaries
#   # require ...
# end