ldap.rb in whimsy-asf-0.0.76

- old
+ new

@@ -31,10 +31,11 @@
 require 'wunderbar'
 require 'ldap'
 require 'weakref'
 require 'net/http'
 require 'base64'
+require 'thread'
 
 module ASF
   module LDAP
      # https://www.pingmybox.com/dashboard?location=304
      # https://github.com/apache/infrastructure-puppet/blob/deployment/data/common.yaml (ldapserver::slapd_peers)
@@ -46,10 +47,13 @@
       ldaps://snappy5.apache.org:636
       ldaps://ldap2-lw-us.apache.org:636
       ldaps://ldap2-lw-eu.apache.org:636
     )
 
+    CONNECT_LOCK = Mutex.new
+    HOST_QUEUE = Queue.new
+
     # fetch configuration from apache/infrastructure-puppet
     def self.puppet_config
       return @puppet if @puppet
       file = '/apache/infrastructure-puppet/deployment/data/common.yaml'
       http = Net::HTTP.new('raw.githubusercontent.com', 443)
@@ -69,12 +73,17 @@
     rescue
       nil
     end
 
     # connect to LDAP
-    def self.connect
-      hosts.shuffle.each do |host|
+    def self.connect(test = true)
+      # Try each host at most once
+      hosts.length.times do
+        # Ensure we use each host in turn
+        hosts.each {|host| HOST_QUEUE.push host} if HOST_QUEUE.empty?
+        host = HOST_QUEUE.shift
+
         Wunderbar.info "Connecting to LDAP server: #{host}"
 
         begin
           # request connection
           uri = URI.parse(host)
@@ -83,11 +92,11 @@
           else
             ldap = ::LDAP::Conn.new(uri.host, uri.port)
           end
 
           # test the connection
-          ldap.bind
+          ldap.bind if test
 
           # save the host
           @host = host
 
           return ldap
@@ -95,18 +104,22 @@
           Wunderbar.warn "Error connecting to LDAP server #{host}: " +
             re.message + " (continuing)"
         end
 
       end
+
       Wunderbar.error "Failed to connect to any LDAP host"
       return nil
     end
   end
 
-  # backwards compatibility for tools that called this interface
-  def self.init_ldap
-    @ldap ||= ASF::LDAP.connect
+  # public entry point for establishing a connection safely
+  def self.init_ldap(reset = false)
+    ASF::LDAP::CONNECT_LOCK.synchronize do
+      @ldap = nil if reset
+      @ldap ||= ASF::LDAP.connect(!reset)
+    end
   end
 
   # determine where ldap.conf resides
   if Dir.exist? '/etc/openldap'
     ETCLDAP = '/etc/openldap'
@@ -116,27 +129,43 @@
 
   def self.ldap
     @ldap || self.init_ldap
   end
 
-  # search with a scope of one
+  # search with a scope of one, with automatic retry/failover
   def self.search_one(base, filter, attrs=nil)
-    init_ldap unless defined? @ldap
-    return [] unless @ldap
 
-    Wunderbar.info "ldapsearch -x -LLL -b #{base} -s one #{filter} " +
+    cmd = "ldapsearch -x -LLL -b #{base} -s one #{filter} " +
       "#{[attrs].flatten.join(' ')}"
-    
+
+    # try once per host, with a minimum of two tries
+    attempts_left = [ASF::LDAP.hosts.length, 2].max
     begin
+      attempts_left -= 1
+      init_ldap unless @ldap
+      return [] unless @ldap
+
+      target = @ldap.get_option(::LDAP::LDAP_OPT_HOST_NAME) rescue '?'
+      Wunderbar.info "[#{target}] #{cmd}"
+
       result = @ldap.search2(base, ::LDAP::LDAP_SCOPE_ONELEVEL, filter, attrs)
-    rescue
-      result = []
+    rescue Exception => re
+      if attempts_left <= 0
+        Wunderbar.error "[#{target}] => #{re.inspect} for #{cmd}"
+        raise
+      else
+        Wunderbar.warn "[#{target}] => #{re.inspect} for #{cmd}, retrying ..."
+        @ldap.unbind if @ldap.bound? rescue nil
+        @ldap = nil # force new connection
+        sleep 1
+        retry
+      end
     end
 
     result.map! {|hash| hash[attrs]} if String === attrs
 
-    result
+    result.compact
   end
 
   # safely dereference a weakref array attribute.  Block provided is
   # used when reference is not set or has been reclaimed.
   def self.dereference_weakref(object, attr, &block)
@@ -144,11 +173,13 @@
     value = object.instance_variable_get(attr) || block.call
     value[0..-1]
   rescue WeakRef::RefError
     value = block.call
   ensure
-    if value and not value.instance_of? WeakRef
+    if not value or RUBY_VERSION.start_with? '1'
+      object.instance_variable_set(attr, value)
+    elsif value and not value.instance_of? WeakRef
       object.instance_variable_set(attr, WeakRef.new(value))
     end
   end
 
   def self.weakref(attr, &block)
@@ -311,11 +342,11 @@
     def alt_email
       attrs['asf-altEmail'] || []
     end
 
     def pgp_key_fingerprints
-      attrs['asf-pgpKeyFingerprint']
+      attrs['asf-pgpKeyFingerprint'] || []
     end
 
     def urls
       attrs['asf-personalURL'] || []
     end
@@ -381,21 +412,22 @@
         return true
       end
     end
 
     def self.preload
-      Hash[ASF.search_one(base, "cn=*", %w(dn memberUid modifyTimestamp)).map do |results|
+      Hash[ASF.search_one(base, "cn=*", %w(dn memberUid modifyTimestamp createTimestamp)).map do |results|
         cn = results['dn'].first[/^cn=(.*?),/, 1]
         group = ASF::Group.find(cn)
         group.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
-        members = results['memberUid']
-        group.members = members || []
+        group.createTimestamp = results['createTimestamp'].first # it is returned as an array of 1 entry
+        members = results['memberUid']  || []
+        group.members = members
         [group, members]
       end]
     end
 
-    attr_accessor :modifyTimestamp
+    attr_accessor :modifyTimestamp, :createTimestamp
 
     def members=(members)
       @members = WeakRef.new(members)
     end
 
@@ -404,31 +436,50 @@
         ASF.search_one(base, "cn=#{name}", 'memberUid').flatten
       end
 
       members.map {|uid| Person.find(uid)}
     end
+
+    def dn
+      @dn ||= ASF.search_one(base, "cn=#{name}", 'dn').first.first
+    end
+
+    def remove(people)
+      people = Array(people).map(&:id)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_DELETE, 'memberUid', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
+
+    def add(people)
+      people = Array(people).map(&:dn)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_ADD, 'memberUid', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
   end
 
   class Committee < Base
     @base = 'ou=pmc,ou=committees,ou=groups,dc=apache,dc=org'
 
     def self.list(filter='cn=*')
       ASF.search_one(base, filter, 'cn').flatten.map {|cn| Committee.find(cn)}
     end
 
     def self.preload
-      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp)).map do |results|
+      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp createTimestamp)).map do |results|
         cn = results['dn'].first[/^cn=(.*?),/, 1]
         committee = ASF::Committee.find(cn)
         committee.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
-        members = results['member']
+        committee.createTimestamp = results['createTimestamp'].first # it is returned as an array of 1 entry
+        members = results['member'] || []
         committee.members = members
         [committee, members]
       end]
     end
 
-    attr_accessor :modifyTimestamp
+    attr_accessor :modifyTimestamp, :createTimestamp
 
     def members=(members)
       @members = WeakRef.new(members)
     end
 
@@ -441,10 +492,24 @@
     end
 
     def dn
       @dn ||= ASF.search_one(base, "cn=#{name}", 'dn').first.first
     end
+
+    def remove(people)
+      people = Array(people).map(&:dn)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_DELETE, 'member', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
+
+    def add(people)
+      people = Array(people).map(&:dn)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_ADD, 'member', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
   end
 
   class Service < Base
     @base = 'ou=groups,ou=services,dc=apache,dc=org'
 
@@ -454,39 +519,63 @@
 
     def dn
       "cn=#{id},#{self.class.base}"
     end
 
+    def self.preload
+      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp createTimestamp)).map do |results|
+        cn = results['dn'].first[/^cn=(.*?),/, 1]
+        service = ASF::Service.find(cn)
+        service.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
+        service.createTimestamp = results['createTimestamp'].first # it is returned as an array of 1 entry
+        members = results['member'] || []
+        service.members = members
+        [service, members]
+      end]
+    end
+
+    attr_accessor :modifyTimestamp, :createTimestamp
+
+    def members=(members)
+      @members = WeakRef.new(members)
+    end
+
     def members
-      ASF.search_one(base, "cn=#{name}", 'member').flatten.
-        map {|uid| Person.find uid[/uid=(.*?),/,1]}
+      members = weakref(:members) do
+        ASF.search_one(base, "cn=#{name}", 'member').flatten
+      end
+
+      members.map {|uid| Person.find uid[/uid=(.*?),/,1]}
     end
 
     def remove(people)
       people = Array(people).map(&:dn)
       mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_DELETE, 'member', people)
       ASF.ldap.modify(self.dn, [mod])
+      @members = nil
     end
 
     def add(people)
       people = Array(people).map(&:dn)
       mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_ADD, 'member', people)
       ASF.ldap.modify(self.dn, [mod])
+      @members = nil
     end
   end
 
   module LDAP
     def self.bind(user, password, &block)
       dn = ASF::Person.new(user).dn
       raise ::LDAP::ResultError.new('Unknown user') unless dn
 
-      ASF.ldap.unbind rescue nil
+      ASF.ldap.unbind if ASF.ldap.bound? rescue nil
+      ldap = ASF.init_ldap(true)
       if block
-        ASF.ldap.bind(dn, password, &block)
-        ASF.init_ldap
+        ldap.bind(dn, password, &block)
+        ASF.init_ldap(true)
       else
-        ASF.ldap.bind(dn, password)
+        ldap.bind(dn, password)
       end
     end
 
     # validate HTTP authorization, and optionally invoke a block bound to
     # that user.
@@ -507,35 +596,37 @@
       end
     end
 
     # determine what LDAP hosts are available
     def self.hosts
+      return @hosts if @hosts # cache the hosts list
       # try whimsy config
       hosts = Array(ASF::Config.get(:ldap))
 
       # check system configuration
       if hosts.empty?
         conf = "#{ETCLDAP}/ldap.conf"
         if File.exist? conf
           uris = File.read(conf)[/^uri\s+(.*)/i, 1].to_s
           hosts = uris.scan(/ldaps?:\/\/\S+?:\d+/)
+          Wunderbar.debug "Using hosts from LDAP config"
         end
+      else
+        Wunderbar.debug "Using hosts from Whimsy config"
       end
 
       # if all else fails, use default list
+      Wunderbar.debug "Using default host list" if hosts.empty?
       hosts = ASF::LDAP::HOSTS if hosts.empty?
 
-      hosts
+      hosts.shuffle!
+      #Wunderbar.debug "Hosts:\n#{hosts.join(' ')}"
+      @hosts = hosts
     end
 
-    # select LDAP host
-    def self.host
-      @host ||= hosts.sample
-    end
-
     # query and extract cert from openssl output
     def self.extract_cert
-      host = LDAP.host[%r{//(.*?)(/|$)}, 1]
+      host = hosts.sample[%r{//(.*?)(/|$)}, 1]
       puts ['openssl', 's_client', '-connect', host, '-showcerts'].join(' ')
       out, err, rc = Open3.capture3 'openssl', 's_client',
         '-connect', host, '-showcerts'
       out[/^-+BEGIN.*?\n-+END[^\n]+\n/m]
     end