# 
# = bio/io/flatfile/index.rb - OBDA flatfile index 
# 
# Copyright:: Copyright (C) 2002
#             GOTO Naohisa <ng@bioruby.org> 
# License:: The Ruby License
#
#  $Id: index.rb,v 1.19 2007/04/05 23:35:41 trevor Exp $ 
#
# = About Bio::FlatFileIndex
#
# Please refer documents of following classes.
# Classes/modules marked '#' are internal use only.
#
# == Classes/modules in index.rb
# * class  Bio::FlatFileIndex
# * class  Bio::FlatFileIndex::Results
# * module Bio::FlatFileIndex::DEBUG
# * #module Bio::FlatFileIndex::Template
# * #class  Bio::FlatFileIndex::Template::NameSpace
# * #class  Bio::FlatFileIndex::FileID
# * #class  Bio::FlatFileIndex::FileIDs
# * #module Bio::FlatFileIndex::Flat_1
# * #class  Bio::FlatFileIndex::Flat_1::Record
# * #class  Bio::FlatFileIndex::Flat_1::FlatMappingFile
# * #class  Bio::FlatFileIndex::Flat_1::PrimaryNameSpace
# * #class  Bio::FlatFileIndex::Flat_1::SecondaryNameSpace
# * #class  Bio::FlatFileIndex::NameSpaces
# * #class  Bio::FlatFileIndex::DataBank
#
# == Classes/modules in indexer.rb
# * module Bio::FlatFileIndex::Indexer
# * #class  Bio::FlatFileIndex::Indexer::NameSpace
# * #class  Bio::FlatFileIndex::Indexer::NameSpaces
# * #module Bio::FlatFileIndex::Indexer::Parser
# * #class  Bio::FlatFileIndex::Indexer::Parser::TemplateParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::GenBankParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::GenPeptParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::EMBLParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::SPTRParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::FastaFormatParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::MaXMLSequenceParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::MaXMLClusterParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::BlastDefaultParser
# * #class  Bio::FlatFileIndex::Indexer::Parser::PDBChemicalComponentParser
#
# == Classes/modules in bdb.rb
# * #module Bio::FlatFileIndex::BDBDefault
# * #class  Bio::FlatFileIndex::BDBWrapper
# * #module Bio::FlatFileIndex::BDB_1
# * #class  Bio::FlatFileIndex::BDB_1::BDBMappingFile
# * #class  Bio::FlatFileIndex::BDB_1::PrimaryNameSpace
# * #class  Bio::FlatFileIndex::BDB_1::SecondaryNameSpace
#
# = References
# * ((<URL:http://obda.open-bio.org/>))
# * ((<URL:http://cvs.open-bio.org/cgi-bin/viewcvs/viewcvs.cgi/obda-specs/?cvsroot=obf-common>))
#

require 'bio/io/flatfile/indexer'

module Bio


  # Bio::FlatFileIndex is a class for OBDA flatfile index.
  class FlatFileIndex

    autoload :Indexer,    'bio/io/flatfile/indexer'
    autoload :BDBdefault, 'bio/io/flatfile/bdb'
    autoload :BDBwrapper, 'bio/io/flatfile/bdb'
    autoload :BDB_1,      'bio/io/flatfile/bdb'

    # magic string for flat/1 index
    MAGIC_FLAT = 'flat/1'

    # magic string for BerkeleyDB/1 index
    MAGIC_BDB  = 'BerkeleyDB/1'

    #########################################################

    # Opens existing databank. Databank is a directory which contains
    # indexed files and configuration files. The type of the databank
    # (flat or BerkeleyDB) are determined automatically.
    #
    # If block is given, the databank object is passed to the block.
    # The databank will be automatically closed when the block terminates.
    #
    def self.open(name)
      if block_given? then
        begin
          i = self.new(name)
          r = yield i
        ensure
          if i then
            begin
              i.close
            rescue IOError
            end
          end
        end
      else
        r = self.new(name)
      end
      r
    end

    # Opens existing databank. Databank is a directory which contains
    # indexed files and configuration files. The type of the databank
    # (flat or BerkeleyDB) are determined automatically.
    #
    # Unlike +FlatFileIndex.open+, block is not allowed.
    #
    def initialize(name)
      @db = DataBank.open(name)
    end

    # common interface defined in registry.rb
    # Searching databank and returns entry (or entries) as a string.
    # Multiple entries (contatinated to one string) may be returned.
    # Returns empty string if not found.
    #
    def get_by_id(key)
      search(key).to_s
    end

    #--
    # original methods
    #++

    # Closes the databank.
    # Returns nil.
    def close
      check_closed?
      @db.close
      @db = nil
    end

    # Returns true if already closed. Otherwise, returns false.
    def closed?
      if @db then
        false
      else
        true
      end
    end

    # Set default namespaces.
    # <code>default_namespaces = nil</code>
    # means all namespaces in the databank.
    #
    # <code>default_namespaces= [ str1, str2, ... ]</code>
    # means set default namespeces to str1, str2, ...
    #
    # Default namespaces specified in this method only affect 
    # #get_by_id, #search, and #include? methods.
    #
    # Default of default namespaces is nil (that is, all namespaces
    # are search destinations by default).
    #
    def default_namespaces=(names)
      if names then
        @names = []
        names.each { |x| @names.push(x.dup) }
      else
        @names = nil
      end
    end

    # Returns default namespaces.
    # Returns an array of strings or nil.
    # nil means all namespaces.
    def default_namespaces
      @names
    end

    # Searching databank and returns a Bio::FlatFileIndex::Results object.
    def search(key)
      check_closed?
      if @names then
        @db.search_namespaces(key, *@names)
      else
        @db.search_all(key)
      end
    end

    # Searching only specified namespeces.
    # Returns a Bio::FlatFileIndex::Results object.
    #
    def search_namespaces(key, *names)
      check_closed?
      @db.search_namespaces(key, *names)
    end

    # Searching only primary namespece.
    # Returns a Bio::FlatFileIndex::Results object.
    #
    def search_primary(key)
      check_closed?
      @db.search_primary(key)
    end

    # Searching databank.
    # If some entries are found, returns an array of
    # unique IDs (primary identifiers).
    # If not found anything, returns nil.
    #
    # This method is useful when search result is very large and
    # #search method is very slow.
    #
    def include?(key)
      check_closed?
      if @names then
        r = @db.search_namespaces_get_unique_id(key, *@names)
      else
        r = @db.search_all_get_unique_id(key)
      end
      if r.empty? then
        nil
      else
        r
      end
    end

    # Same as #include?, but serching only specified namespaces.
    #
    def include_in_namespaces?(key, *names)
      check_closed?
      r = @db.search_namespaces_get_unique_id(key, *names)
      if r.empty? then
        nil
      else
        r
      end
    end

    # Same as #include?, but serching only primary namespace.
    #
    def include_in_primary?(key)
      check_closed?
      r = @db.search_primary_get_unique_id(key)
      if r.empty? then
        nil
      else
        r
      end
    end

    # Returns names of namespaces defined in the databank.
    # (example: [ 'LOCUS', 'ACCESSION', 'VERSION' ] )
    #
    def namespaces
      check_closed?
      r = secondary_namespaces
      r.unshift primary_namespace
      r
    end

    # Returns name of primary namespace as a string.
    def primary_namespace
      check_closed?
      @db.primary.name
    end

    # Returns names of secondary namespaces as an array of strings.
    def secondary_namespaces
      check_closed?
      @db.secondary.names
    end

    # Check consistency between the databank(index) and original flat files.
    #
    # If the original flat files are changed after creating
    # the databank, raises RuntimeError.
    #
    # Note that this check only compares file sizes as
    # described in the OBDA specification.
    #
    def check_consistency
      check_closed?
      @db.check_consistency
    end

    # If true is given, consistency checks will be performed every time
    # accessing flatfiles. If nil/false, no checks are performed.
    #
    # By default, always_check_consistency is true.
    #
    def always_check_consistency=(bool)
      @db.always_check=(bool)
    end

    # If true, consistency checks will be performed every time
    # accessing flatfiles. If nil/false, no checks are performed.
    #
    # By default, always_check_consistency is true.
    #
    def always_check_consistency(bool)
      @db.always_check
    end

    #--
    # private methods
    #++
    
    # If the databank is closed, raises IOError.
    def check_closed?
      @db or raise IOError, 'closed databank'
    end
    private :check_closed?

    #--
    #########################################################
    #++

    # <code>Results</code> stores search results created by
    # <code>Bio::FlatFileIndex</code> methods.
    #
    # Currently, this class inherits Hash, but internal
    # structure of this class may be changed anytime.
    # Only using methods described below are strongly recomended.
    #
    class Results < Hash

      # Add search results.
      # "a + b" means "a OR b".
      # * Example
      #    # I want to search 'ADH_IRON_1' OR 'ADH_IRON_2'
      #    db = Bio::FlatFIleIndex.new(location)
      #    a1 = db.search('ADH_IRON_1')
      #    a2 = db.search('ADH_IRON_2')
      #    # a1 and a2 are Bio::FlatFileIndex::Results objects.
      #    print a1 + a2
      #
      def +(a)
        raise 'argument must be Results class' unless a.is_a?(self.class)
        res = self.dup
        res.update(a)
        res
      end

      # Returns set intersection of results.
      # "a * b" means "a AND b".
      # * Example
      #    # I want to search 'HIS_KIN' AND 'human'
      #    db = Bio::FlatFIleIndex.new(location)
      #    hk = db.search('HIS_KIN')
      #    hu = db.search('human')
      #    # hk and hu are Bio::FlatFileIndex::Results objects.
      #    print hk * hu
      #
      def *(a)
        raise 'argument must be Results class' unless a.is_a?(self.class)
        res = self.class.new
        a.each_key { |x| res.store(x, a[x]) if self[x] }
        res
      end

      # Returns a string. (concatinated if multiple results exists).
      # Same as <code>to_a.join('')</code>.
      #
      def to_s
        self.values.join
      end

      #--
      #alias each_orig each
      #++

      # alias for each_value.
      alias each each_value

      # Iterates over each result (string).
      # Same as to_a.each.
      def each(&x) #:yields: str
        each_value(&x)
      end if false #dummy for RDoc

      #--
      #alias to_a_orig to_a
      #++

      # alias for to_a.
      alias to_a values

      # Returns an array of strings.
      # If no search results are exist, returns an empty array.
      #
      def to_a; values; end if false #dummy for RDoc

      # Returns number of results.
      # Same as to_a.size.
      def size; end if false #dummy for RDoc

    end #class Results

    #########################################################

    # Module for output debug messages.
    # Default setting: If $DEBUG or $VERBOSE is true, output debug
    # messages to $stderr; Otherwise, don't output messages.
    #
    module DEBUG
      @@out = $stderr
      @@flag = nil

      # Set debug messages output destination.
      # If true is given, outputs to $stderr.
      # If nil is given, outputs nothing.
      # This method affects ALL of FlatFileIndex related objects/methods.
      #
      def self.out=(io)
        if io then
          @@out = io
          @@out = $stderr if io == true
          @@flag = true
        else
          @@out = nil
          @@flag = nil
        end
        @@out
      end

      # get current debug messeages output destination
      def self.out
        @@out
      end

      # prints debug messages
      def self.print(*arg)
        @@flag = true if $DEBUG or $VERBOSE
        @@out.print(*arg) if @@out and @@flag
      end
    end #module DEBUG

    #########################################################

    # Templates
    #
    # Internal use only.
    module Template

      # templates of namespace
      #
      # Internal use only.
      class NameSpace
        def filename
          # should be redifined in child class
          raise NotImplementedError, "should be redefined in child class"
        end

        def mapping(filename)
          # should be redifined in child class
          raise NotImplementedError, "should be redefined in child class"
          #Flat_1::FlatMappingFile.new(filename)
        end

        def initialize(dbname, name)
          @dbname = dbname
          @name = name.dup
          @name.freeze
          @file = mapping(filename)
        end
        attr_reader :dbname, :name, :file

        def search(key)
          @file.open
          @file.search(key)
        end

        def close
          @file.close
        end

        def include?(key)
          r = search(key)
          unless r.empty? then
            key
          else
            nil
          end
        end
      end #class NameSpace
    end #module Template

    # FileID class.
    #
    # Internal use only.
    class FileID
      def self.new_from_string(str)
        a = str.split("\t", 2)
        a[1] = a[1].to_i if a[1]
        self.new(a[0], a[1])
      end

      def initialize(filename, filesize = nil)
        @filename = filename
        @filesize = filesize
        @io = nil
      end
      attr_reader :filename, :filesize

      def check
        begin
          fsize = File.size(@filename)
          r = ( fsize == @filesize)
        rescue Errno::ENOENT
          fsize = -1
          r = nil
        end
        DEBUG.print "FileID: File.size(#{@filename.inspect}) = ",
          fsize, (r ? ' == ' : ' != ') , @filesize,
          (r ? '' : ' bad!'), "\n"
        r
      end

      def recalc
        @filesize = File.size(@filename)
      end

      def to_s(i = nil)
        if i then
          str = "fileid_#{i}\t"
        else
          str = ''
        end
        str << "#{@filename}\t#{@filesize}"
        str
      end

      def open
        unless @io then
          DEBUG.print "FileID: open #{@filename}\n"
          @io = File.open(@filename, 'rb')
          true
        else
          nil
        end
      end

      def close
        if @io then
          DEBUG.print "FileID: close #{@filename}\n"
          @io.close
          @io = nil
          nil
        else
          true
        end
      end

      def seek(*arg)
        @io.seek(*arg)
      end

      def read(size)
        @io.read(size)
      end

      def get(pos, length)
        open
        seek(pos, IO::SEEK_SET)
        data = read(length)
        close
        data
      end
    end #class FileID

    # FileIDs class.
    #
    # Internal use only.
    class FileIDs < Array
      def initialize(prefix, hash)
        @hash = hash
        @prefix = prefix
      end

      def [](n)
        r = super(n)
        if r then
          r
        else
          data = @hash["#{@prefix}#{n}"]
          if data then
            self[n] = data
          end
          super(n)
        end
      end

      def []=(n, data)
        if data.is_a?(FileID) then
          super(n, data)
        elsif data then
          super(n, FileID.new_from_string(data))
        else
          # data is nil
          super(n, nil)
        end
        self[n]
      end

      def add(*arg)
        arg.each do |filename|
          self << FileID.new(filename)
        end
      end

      def cache_all
        a = @hash.keys.collect do |k|
          if k =~ /\A#{Regexp.escape(@prefix)}(\d+)/ then
            $1.to_i
          else
            nil
          end
        end
        a.compact!
        a.each do |i|
          self[i]
        end
        a
      end

      def each
        (0...self.size).each do |i|
          x = self[i]
          yield(x) if x
        end
        self
      end

      def each_with_index
        (0...self.size).each do |i|
          x = self[i]
          yield(x, i) if x
        end
        self
      end

      def keys
        self.cache_all
        a = []
        (0...self.size).each do |i|
          a << i if self[i]
        end
        a
      end

      def filenames
        self.cache_all
        a = []
        self.each do |x|
          a << x.filename
        end
        a
      end

      def check_all
        self.cache_all
        r = true
        self.each do |x|
          r = x.check
          break unless r
        end
        r
      end
      alias check check_all

      def close_all
        self.each do |x|
          x.close
        end
        nil
      end
      alias close close_all

      def recalc_all
        self.cache_all
        self.each do |x|
          x.recalc
        end
        true
      end
      alias recalc recalc_all

    end #class FileIDs

    # module for flat/1 databank
    #
    # Internal use only.
    module Flat_1

      # Record class.
      #
      # Internal use only.
      class Record
        def initialize(str, size = nil)
          a = str.split("\t")
          a.each { |x| x.to_s.gsub!(/[\000 ]+\z/, '') }
          @key = a.shift.to_s
          @val = a
          @size = (size or str.length)
          #DEBUG.print "key=#{@key.inspect},val=#{@val.inspect},size=#{@size}\n"
        end
        attr_reader :key, :val, :size

        def to_s
          self.class.to_string(@size, @key, @val)
        end

        def self.to_string(size, key, val)
          sprintf("%-*s", size, key + "\t" + val.join("\t"))
        end

        def self.create(size, key, val)
          self.new(self.to_string(size, key, val))
        end

        def ==(x)
          self.to_s == x.to_s
        end
      end #class Record

      # FlatMappingFile class.
      #
      # Internal use only.
      class FlatMappingFile
        @@recsize_width = 4
        @@recsize_regex = /\A\d{4}\z/

        def self.open(*arg)
          self.new(*arg)
        end

        def initialize(filename, mode = 'rb')
          @filename = filename
          @mode = mode
          @file = nil
          #@file = File.open(filename, mode)
          @record_size = nil
          @records = nil
        end
        attr_accessor :mode
        attr_reader :filename
        
        def open
          unless @file then
            DEBUG.print "FlatMappingFile: open #{@filename}\n"
            @file = File.open(@filename, @mode)
            true
          else
            nil
          end
        end

        def close
          if @file then
            DEBUG.print "FlatMappingFile: close #{@filename}\n"
            @file.close
            @file = nil
          end
          nil
        end

        def record_size
          unless @record_size then
            open
            @file.seek(0, IO::SEEK_SET)
            s = @file.read(@@recsize_width)
            raise 'strange record size' unless s =~ @@recsize_regex
            @record_size = s.to_i
            DEBUG.print "FlatMappingFile: record_size: #{@record_size}\n"
          end
          @record_size
        end

        def get_record(i)
          rs = record_size
          seek(i)
          str = @file.read(rs)
          #DEBUG.print "get_record(#{i})=#{str.inspect}\n"
          str
        end

        def seek(i)
          rs = record_size
          @file.seek(@@recsize_width + rs * i)
        end

        def records
          unless @records then
            rs = record_size
            @records = (@file.stat.size - @@recsize_width) / rs
            DEBUG.print "FlatMappingFile: records: #{@records}\n"
          end
          @records
        end
        alias size records

        # methods for writing file
        def write_record(str)
          rs = record_size
          rec = sprintf("%-*s", rs, str)[0..rs]
          @file.write(rec)
        end

        def add_record(str)
          n = records
          rs = record_size
          @file.seek(0, IO::SEEK_END)
          write_record(str)
          @records += 1
        end

        def put_record(i, str)
          n = records
          rs = record_size
          if i >= n then
            @file.seek(0, IO::SEEK_END)
            @file.write(sprintf("%-*s", rs, '') * (i - n))
            @records = i + 1
          else
            seek(i)
          end
          write_record(str)
        end

        def init(rs)
          unless 0 < rs and rs < 10 ** @@recsize_width then
            raise 'record size out of range'
          end
          open
          @record_size = rs
          str = sprintf("%0*d", @@recsize_width, rs)
          @file.truncate(0)
          @file.seek(0, IO::SEEK_SET)
          @file.write(str)
          @records = 0
        end

        # export/import/edit data
        def each
          n = records
          seek(0)
          (0...n).each do |i|
            yield Record.new(get_record(i))
          end
          self
        end

        def export_tsv(stream)
          self.each do |x|
            stream << "#{x.to_s}\n"
          end
          stream
        end

        def init_with_sorted_tsv_file(filename, flag_primary = false)
          rec_size = 1
          f = File.open(filename)
          f.each do |y|
            rec_size = y.chomp.length if rec_size < y.chomp.length
          end
          self.init(rec_size)

          prev = nil
          f.rewind
          if flag_primary then
            f.each do |y|
              x = Record.new(y.chomp, rec_size)
              if prev then
                if x.key == prev.key
                  DEBUG.print "Warining: overwrote unique id #{x.key.inspect}\n"
                else
                  self.add_record(prev.to_s)
                end
              end
              prev = x
            end
            self.add_record(prev.to_s) if prev
          else
            f.each do |y|
              x = Record.new(y.chomp, rec_size)
              self.add_record(x.to_s) if x != prev
              prev = x
            end
          end
          f.close
          self
        end

        def self.external_sort_proc(sort_program = [ '/usr/bin/env', 
                                                     'LC_ALL=C',
                                                     '/usr/bin/sort' ])
          Proc.new do |out, in1, *files|
            cmd = sort_program + [ '-o', out, in1, *files ]
            system(*cmd)
          end
        end

        def self.external_merge_sort_proc(sort_program = [ '/usr/bin/env', 
                                                           'LC_ALL=C',
                                                           '/usr/bin/sort' ])
          Proc.new do |out, in1, *files|
            # (in1 may be sorted)
            tf_all = []
            tfn_all = []
            files.each do |fn|
              tf = Tempfile.open('sort')
              tf.close(false)
              cmd = sort_program + [ '-o', tf.path, fn ]
              system(*cmd)
              tf_all << tf
              tfn_all << tf.path
            end
            cmd_fin = sort_program + [ '-m', '-o', out, in1, *tfn_all ]
            system(*cmd_fin)
            tf_all.each do |tf|
              tf.close(true)
            end
          end
        end

        def self.external_merge_proc(sort_program =  [ '/usr/bin/env', 
                                                       'LC_ALL=C',
                                                       '/usr/bin/sort' ])
          Proc.new do |out, in1, *files|
            # files (and in1) must be sorted
            cmd = sort_program + [ '-m', '-o', out, in1, *files ]
            system(*cmd)
          end
        end

        def self.internal_sort_proc
          Proc.new do |out, in1, *files|
            a = IO.readlines(in1)
            files.each do |fn|
              IO.foreach(fn) do |x|
                a << x
              end
            end
            a.sort!
            of = File.open(out, 'w')
            a.each { |x| of << x }
            of.close
          end
        end

        def import_tsv_files(flag_primary, mode, sort_proc, *files)
          require 'tempfile'

          tmpfile1 = Tempfile.open('flat')
          self.export_tsv(tmpfile1) unless mode == :new
          tmpfile1.close(false)

          tmpfile0 = Tempfile.open('sorted')
          tmpfile0.close(false)

          sort_proc.call(tmpfile0.path, tmpfile1.path, *files)

          tmpmap = self.class.new(self.filename + ".#{$$}.tmp~", 'wb+')
          tmpmap.init_with_sorted_tsv_file(tmpfile0.path, flag_primary)
          tmpmap.close
          self.close

          begin
            File.rename(self.filename, self.filename + ".#{$$}.bak~")
          rescue Errno::ENOENT
          end
          File.rename(tmpmap.filename, self.filename)
          begin
            File.delete(self.filename + ".#{$$}.bak~")
          rescue Errno::ENOENT
          end

          tmpfile0.close(true)
          tmpfile1.close(true)
          self
        end


        # methods for searching
        def search(key)
          n = records
          return [] if n <= 0
          i = n / 2
          i_prev = nil
          DEBUG.print "binary search starts...\n"
          begin
            rec = Record.new(get_record(i))
            i_prev = i
            if key < rec.key then
              n = i
              i = i / 2
            elsif key > rec.key then
              i = (i + n) / 2
            else # key == rec.key
              result = [ rec.val ]
              j = i - 1
              while j >= 0 and
                  (rec = Record.new(get_record(j))).key == key
                result << rec.val
                j = j - 1
              end
              result.reverse!
              j = i + 1
              while j < n and
                  (rec = Record.new(get_record(j))).key == key
                result << rec.val
                j = j + 1
              end
              DEBUG.print "#{result.size} hits found!!\n"
              return result
            end
          end until i_prev == i
          DEBUG.print "no hits found\n"
          #nil
          []
        end
      end #class FlatMappingFile

      # primary name space
      #
      # Internal use only.
      class PrimaryNameSpace < Template::NameSpace
        def mapping(filename)
          FlatMappingFile.new(filename)
        end
        def filename
          File.join(dbname, "key_#{name}.key")
        end
      end #class PrimaryNameSpace

      # secondary name space
      #
      # Internal use only.
      class SecondaryNameSpace < Template::NameSpace
        def mapping(filename)
          FlatMappingFile.new(filename)
        end
        def filename
          File.join(dbname, "id_#{name}.index")
        end
        def search(key)
          r = super(key)
          file.close
          r.flatten!
          r
        end
      end #class SecondaryNameSpace
    end #module Flat_1

    # namespaces
    #
    # Internal use only.
    class NameSpaces < Hash
      def initialize(dbname, nsclass, arg)
        @dbname = dbname
        @nsclass = nsclass
        if arg.is_a?(String) then
          a = arg.split("\t")
        else
          a = arg
        end
        a.each do |x|
          self[x] = @nsclass.new(@dbname, x)
        end
        self
      end

      def each_names
        self.names.each do |x|
          yield x
        end
      end

      def each_files
        self.values.each do |x|
          yield x
        end
      end

      def names
        keys
      end

      def close_all
        values.each { |x| x.file.close }
      end
      alias close close_all

      def search(key)
        r = []
        values.each do |ns|
          r.concat ns.search(key)
        end
        r.sort!
        r.uniq!
        r
      end

      def search_names(key, *names)
        r = []
        names.each do |x|
          ns = self[x]
          raise "undefined namespace #{x.inspect}" unless ns
          r.concat ns.search(key)
        end
        r
      end

      def to_s
        names.join("\t")
      end
    end #class NameSpaces

    # databank
    #
    # Internal use only.
    class DataBank
      def self.file2hash(fileobj)
        hash = {}
        fileobj.each do |line|
          line.chomp!
          a = line.split("\t", 2)
          hash[a[0]] = a[1]
        end
        hash
      end
      private_class_method :file2hash

      def self.filename(dbname)
        File.join(dbname, 'config.dat')
      end

      def self.read(name, mode = 'rb', *bdbarg)
        f = File.open(filename(name), mode)
        hash = file2hash(f)
        f.close
        db = self.new(name, nil, hash)
        db.bdb_open(*bdbarg)
        db
      end

      def self.open(*arg)
        self.read(*arg)
      end

      def initialize(name, idx_type = nil, hash = {})
        @dbname = name.dup
        @dbname.freeze
        @bdb = nil

        @always_check = true
        self.index_type = (hash['index'] or idx_type)

        if @bdb then
          @config = BDBwrapper.new(@dbname, 'config')
          @bdb_fileids = BDBwrapper.new(@dbname, 'fileids')
          @nsclass_pri = BDB_1::PrimaryNameSpace
          @nsclass_sec = BDB_1::SecondaryNameSpace
        else
          @config = hash
          @nsclass_pri = Flat_1::PrimaryNameSpace
          @nsclass_sec = Flat_1::SecondaryNameSpace
        end
        true
      end

      attr_reader :dbname, :index_type

      def index_type=(str)
        case str
        when MAGIC_BDB
          @index_type = MAGIC_BDB
          @bdb = true
          unless defined?(BDB)
            raise RuntimeError, "Berkeley DB support not found"
          end
        when MAGIC_FLAT, '', nil, false
          @index_type = MAGIC_FLAT
          @bdb = false
        else
          raise 'unknown or unsupported index type'
        end
      end

      def to_s
        a = ""
        a << "index\t#{@index_type}\n"

        unless @bdb then
          a << "format\t#{@format}\n"
          @fileids.each_with_index do |x, i|
            a << "#{x.to_s(i)}\n"
          end
          a << "primary_namespace\t#{@primary.name}\n"
          a << "secondary_namespaces\t"
          a << @secondary.names.join("\t")
          a << "\n"
        end
        a
      end

      def bdb_open(*bdbarg)
        if @bdb then
          @config.close
          @config.open(*bdbarg)
          @bdb_fileids.close
          @bdb_fileids.open(*bdbarg)
          true
        else
          nil
        end
      end

      def write(mode = 'wb', *bdbarg)
        unless FileTest.directory?(@dbname) then
          Dir.mkdir(@dbname)
        end
        f = File.open(self.class.filename(@dbname), mode)
        f.write self.to_s
        f.close

        if @bdb then
          bdb_open(*bdbarg)
          @config['format'] = format
          @config['primary_namespace'] = @primary.name
          @config['secondary_namespaces'] = @secondary.names.join("\t")
          @bdb_fileids.writeback_array('', fileids, *bdbarg)
        end
        true
      end

      def close
        DEBUG.print "DataBank: close #{@dbname}\n"
        primary.close
        secondary.close
        fileids.close
        if @bdb then
          @config.close
          @bdb_fileids.close
        end
        nil
      end

      ##parameters
      def primary
        unless @primary then
          self.primary = @config['primary_namespace']
        end
        @primary
      end

      def primary=(pri_name)
        if !pri_name or pri_name.empty? then
          pri_name = 'UNIQUE'
        end
        @primary = @nsclass_pri.new(@dbname, pri_name)
        @primary
      end

      def secondary
        unless @secondary then
          self.secondary = @config['secondary_namespaces']
        end
        @secondary
      end

      def secondary=(sec_names)
        if !sec_names then
          sec_names = []
        end
        @secondary = NameSpaces.new(@dbname, @nsclass_sec, sec_names)
        @secondary
      end

      def format=(str)
        @format = str.to_s.dup
      end

      def format
        unless @format then
          self.format = @config['format']
        end
        @format
      end

      def fileids
        unless @fileids then
          init_fileids
        end
        @fileids
      end

      def init_fileids
        if @bdb then
          @fileids = FileIDs.new('', @bdb_fileids)
        else
          @fileids = FileIDs.new('fileid_', @config)
        end
        @fileids
      end

      # high level methods
      def always_check=(bool)
        if bool then
          @always_check = true
        else
          @always_check = false
        end
      end
      attr_reader :always_check

      def get_flatfile_data(f, pos, length)
        fi = fileids[f.to_i]
        if @always_check then
          raise "flatfile #{fi.filename.inspect} may be modified" unless fi.check
        end
        fi.get(pos.to_i, length.to_i)
      end

      def search_all_get_unique_id(key)
        s = secondary.search(key)
        p = primary.include?(key)
        s.push p if p
        s.sort!
        s.uniq!
        s
      end

      def search_primary(*arg)
        r = Results.new
        arg.each do |x|
          a = primary.search(x)
          # a is empty or a.size==1 because primary key must be unique
          r.store(x, get_flatfile_data(*a[0])) unless a.empty?
        end
        r
      end

      def search_all(key)
        s = search_all_get_unique_id(key)
        search_primary(*s)
      end

      def search_primary_get_unique_id(key)
        s = []
        p = primary.include?(key)
        s.push p if p
        s
      end

      def search_namespaces_get_unique_id(key, *names)
        if names.include?(primary.name) then
          n2 = names.dup
          n2.delete(primary.name)
          p = primary.include?(key)
        else
          n2 = names
          p = nil
        end
        s = secondary.search_names(key, *n2)
        s.push p if p
        s.sort!
        s.uniq!
        s
      end

      def search_namespaces(key, *names)
        s = search_namespaces_get_unique_id(key, *names)
        search_primary(*s)
      end

      def check_consistency
        fileids.check_all
      end
    end #class DataBank

  end #class FlatFileIndex
end #module Bio