ldap.rb in whimsy-asf-0.0.75

- old
+ new

@@ -1,50 +1,121 @@
+#
+# Encapsulate access to LDAP, caching results for performance.  For best
+# performance in applications that access large number of objects, make use
+# of the preload methods to pre-fetch multiple objects in a single LDAP
+# call, and rely on the cache to find the objects later.
+#
+# The cache makes heavy use of Weak References internally to enable garbage
+# collection to reclaim objects; among other things, this ensures that
+# LDAP results don't become too stale.
+#
+# Until garbage collection reclaims an object, calls to find methods for the
+# same name is guaranteed to return the same object.  Holding on to the
+# results of find or preload calls (by assigning it to a variable) is
+# sufficient to prevent reclaiming of objects.
+#
+# To illustrate, the following is likely to return the same id twice, followed
+# by a new id:
+#   puts ASF::Person.find('rubys').__id__
+#   puts ASF::Person.find('rubys').__id__
+#   GC.start
+#   puts ASF::Person.find('rubys').__id__
+#
+# By contrast, the following is guaranteed to produce the same id three times:
+#   rubys1 = ASF::Person.find('rubys')
+#   rubys2 = ASF::Person.find('rubys')
+#   GC.start
+#   rubys3 = ASF::Person.find('rubys')
+#   puts [rubys1.__id__, rubys2.__id__, rubys3.__id__]
+#
+
 require 'wunderbar'
 require 'ldap'
 require 'weakref'
+require 'net/http'
+require 'base64'
 
 module ASF
   module LDAP
      # https://www.pingmybox.com/dashboard?location=304
      # https://github.com/apache/infrastructure-puppet/blob/deployment/data/common.yaml (ldapserver::slapd_peers)
     HOSTS = %w(
       ldaps://ldap1-us-west.apache.org:636
-      ldaps://ldap1-eu-central.apache.org:636
+      ldaps://ldap1-lw-us.apache.org:636
       ldaps://ldap2-us-west.apache.org:636
-      ldaps://ldap1-us-east.apache.org:636
+      ldaps://ldap1-lw-eu.apache.org:636
       ldaps://snappy5.apache.org:636
+      ldaps://ldap2-lw-us.apache.org:636
+      ldaps://ldap2-lw-eu.apache.org:636
     )
-  end
 
-  # determine where ldap.conf resides
-  if Dir.exist? '/etc/openldap'
-    ETCLDAP = '/etc/openldap'
-  else
-    ETCLDAP = '/etc/ldap'
-  end
+    # fetch configuration from apache/infrastructure-puppet
+    def self.puppet_config
+      return @puppet if @puppet
+      file = '/apache/infrastructure-puppet/deployment/data/common.yaml'
+      http = Net::HTTP.new('raw.githubusercontent.com', 443)
+      http.use_ssl = true
+      @puppet = YAML.load(http.request(Net::HTTP::Get.new(file)).body)
+    end
 
-  # determine whether or not the LDAP API can be used
-  def self.init_ldap
-    @ldap = nil
-    @mtime = Time.now
+    # extract the ldapcert from the puppet configuration
+    def self.puppet_cert
+      puppet_config['ldapclient::ldapcert']
+    end
 
-    host = ASF::LDAP.host
+    # extract the ldap servers from the puppet configuration
+    def self.puppet_ldapservers
+      puppet_config['ldapserver::slapd_peers'].values.
+        map {|host| "ldaps://#{host}:636"}
+    rescue
+      nil
+    end
 
-    Wunderbar.info "Connecting to LDAP server: #{host}"
+    # connect to LDAP
+    def self.connect
+      hosts.shuffle.each do |host|
+        Wunderbar.info "Connecting to LDAP server: #{host}"
 
-    begin
-      uri = URI.parse(host)
-      if uri.scheme == 'ldaps'
-        @ldap = ::LDAP::SSLConn.new(uri.host, uri.port)
-      else
-        @ldap = ::LDAP::Conn.new(uri.host, uri.port)
+        begin
+          # request connection
+          uri = URI.parse(host)
+          if uri.scheme == 'ldaps'
+            ldap = ::LDAP::SSLConn.new(uri.host, uri.port)
+          else
+            ldap = ::LDAP::Conn.new(uri.host, uri.port)
+          end
+
+          # test the connection
+          ldap.bind
+
+          # save the host
+          @host = host
+
+          return ldap
+        rescue ::LDAP::ResultError => re
+          Wunderbar.warn "Error connecting to LDAP server #{host}: " +
+            re.message + " (continuing)"
+        end
+
       end
-    rescue ::LDAP::ResultError=>re
-      Wunderbar.error "Error binding to LDAP server: message: ["+ re.message + "]"
+      Wunderbar.error "Failed to connect to any LDAP host"
+      return nil
     end
   end
 
+  # backwards compatibility for tools that called this interface
+  def self.init_ldap
+    @ldap ||= ASF::LDAP.connect
+  end
+
+  # determine where ldap.conf resides
+  if Dir.exist? '/etc/openldap'
+    ETCLDAP = '/etc/openldap'
+  else
+    ETCLDAP = '/etc/ldap'
+  end
+
   def self.ldap
     @ldap || self.init_ldap
   end
 
   # search with a scope of one
@@ -64,34 +135,38 @@
     result.map! {|hash| hash[attrs]} if String === attrs
 
     result
   end
 
-  def self.refresh(symbol)
-    if not @mtime or Time.now - @mtime > 300.0
-      @mtime = Time.now
+  # safely dereference a weakref array attribute.  Block provided is
+  # used when reference is not set or has been reclaimed.
+  def self.dereference_weakref(object, attr, &block)
+    attr = "@#{attr}"
+    value = object.instance_variable_get(attr) || block.call
+    value[0..-1]
+  rescue WeakRef::RefError
+    value = block.call
+  ensure
+    if value and not value.instance_of? WeakRef
+      object.instance_variable_set(attr, WeakRef.new(value))
     end
+  end
 
-    if instance_variable_get("#{symbol}_mtime") != @mtime
-      instance_variable_set("#{symbol}_mtime", @mtime)
-      instance_variable_set(symbol, nil)
-    end
+  def self.weakref(attr, &block)
+    self.dereference_weakref(self, attr, &block)
   end
 
   def self.pmc_chairs
-    refresh(:@pmc_chairs)
-    @pmc_chairs ||= Service.find('pmc-chairs').members
+    weakref(:pmc_chairs) {Service.find('pmc-chairs').members}
   end
 
   def self.committers
-    refresh(:@committers)
-    @committers ||= Group.find('committers').members
+    weakref(:committers) {Group.find('committers').members}
   end
 
   def self.members
-    refresh(:@members)
-    @members ||= Group.find('member').members
+    weakref(:members) {Group.find('member').members}
   end
 
   class Base
     attr_reader :name
 
@@ -132,10 +207,14 @@
 
     def reference
       self
     end
 
+    def weakref(attr, &block)
+      ASF.dereference_weakref(self, attr, &block)
+    end
+
     unless Object.respond_to? :id
       def id
         @name
       end
     end
@@ -211,16 +290,20 @@
 
     def asf_member?
       ASF::Member.status[name] or ASF.members.include? self
     end
 
+    def asf_officer_or_member?
+      asf_member? or ASF.pmc_chairs.include? self
+    end
+
     def asf_committer?
        ASF::Group.new('committers').include? self
     end
 
     def banned?
-      not attrs['loginShell'] or attrs['loginShell'].include? "/usr/bin/false"
+      not attrs['loginShell'] or %w(/usr/bin/false bin/nologin bin/no-cla).any? {|a| attrs['loginShell'].first.include? a}
     end
 
     def mail
       attrs['mail'] || []
     end
@@ -297,26 +380,66 @@
       else
         return true
       end
     end
 
+    def self.preload
+      Hash[ASF.search_one(base, "cn=*", %w(dn memberUid modifyTimestamp)).map do |results|
+        cn = results['dn'].first[/^cn=(.*?),/, 1]
+        group = ASF::Group.find(cn)
+        group.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
+        members = results['memberUid']
+        group.members = members || []
+        [group, members]
+      end]
+    end
+
+    attr_accessor :modifyTimestamp
+
+    def members=(members)
+      @members = WeakRef.new(members)
+    end
+
     def members
-      ASF.search_one(base, "cn=#{name}", 'memberUid').flatten.
-        map {|uid| Person.find(uid)}
+      members = weakref(:members) do
+        ASF.search_one(base, "cn=#{name}", 'memberUid').flatten
+      end
+
+      members.map {|uid| Person.find(uid)}
     end
   end
 
   class Committee < Base
     @base = 'ou=pmc,ou=committees,ou=groups,dc=apache,dc=org'
 
     def self.list(filter='cn=*')
       ASF.search_one(base, filter, 'cn').flatten.map {|cn| Committee.find(cn)}
     end
 
+    def self.preload
+      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp)).map do |results|
+        cn = results['dn'].first[/^cn=(.*?),/, 1]
+        committee = ASF::Committee.find(cn)
+        committee.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
+        members = results['member']
+        committee.members = members
+        [committee, members]
+      end]
+    end
+
+    attr_accessor :modifyTimestamp
+
+    def members=(members)
+      @members = WeakRef.new(members)
+    end
+
     def members
-      ASF.search_one(base, "cn=#{name}", 'member').flatten.
-        map {|uid| Person.find uid[/uid=(.*?),/,1]}
+      members = weakref(:members) do
+        ASF.search_one(base, "cn=#{name}", 'member').flatten
+      end
+
+      members.map {|uid| Person.find uid[/uid=(.*?),/,1]}
     end
 
     def dn
       @dn ||= ASF.search_one(base, "cn=#{name}", 'dn').first.first
     end
@@ -352,61 +475,114 @@
   end
 
   module LDAP
     def self.bind(user, password, &block)
       dn = ASF::Person.new(user).dn
+      raise ::LDAP::ResultError.new('Unknown user') unless dn
+
+      ASF.ldap.unbind rescue nil
       if block
         ASF.ldap.bind(dn, password, &block)
+        ASF.init_ldap
       else
         ASF.ldap.bind(dn, password)
       end
-      ASF.init_ldap
     end
 
-    # select LDAP host
-    def self.host
+    # validate HTTP authorization, and optionally invoke a block bound to
+    # that user.
+    def self.http_auth(string, &block)
+      auth = Base64.decode64(string.to_s[/Basic (.*)/, 1] || '')
+      user, password = auth.split(':', 2)
+      return unless password
+
+      if block
+        self.bind(user, password, &block)
+      else
+        begin
+          ASF::LDAP.bind(user, password) {}
+          return ASF::Person.new(user)
+        rescue ::LDAP::ResultError
+          return nil
+        end
+      end
+    end
+
+    # determine what LDAP hosts are available
+    def self.hosts
       # try whimsy config
-      host = ASF::Config.get(:ldap)
+      hosts = Array(ASF::Config.get(:ldap))
 
       # check system configuration
-      unless host
+      if hosts.empty?
         conf = "#{ETCLDAP}/ldap.conf"
         if File.exist? conf
-          host = File.read(conf)[/^uri\s+(ldaps?:\/\/\S+?:\d+)/i, 1]
+          uris = File.read(conf)[/^uri\s+(.*)/i, 1].to_s
+          hosts = uris.scan(/ldaps?:\/\/\S+?:\d+/)
         end
       end
 
-      # if all else fails, pick one at random
-      host = ASF::LDAP::HOSTS.sample unless host
+      # if all else fails, use default list
+      hosts = ASF::LDAP::HOSTS if hosts.empty?
 
-      host
+      hosts
     end
 
+    # select LDAP host
+    def self.host
+      @host ||= hosts.sample
+    end
+
     # query and extract cert from openssl output
-    def self.cert
+    def self.extract_cert
       host = LDAP.host[%r{//(.*?)(/|$)}, 1]
-      query = "openssl s_client -connect #{host} -showcerts"
-      output = `#{query} < /dev/null 2> /dev/null`
-      output[/^-+BEGIN.*?\n-+END[^\n]+\n/m]
+      puts ['openssl', 's_client', '-connect', host, '-showcerts'].join(' ')
+      out, err, rc = Open3.capture3 'openssl', 's_client',
+        '-connect', host, '-showcerts'
+      out[/^-+BEGIN.*?\n-+END[^\n]+\n/m]
     end
 
     # update /etc/ldap.conf. Usage:
+    #
     #   sudo ruby -r whimsy/asf -e "ASF::LDAP.configure"
+    #
     def self.configure
-      if not File.exist? "#{ETCLDAP}/asf-ldap-client.pem"
-        File.write "#{ETCLDAP}/asf-ldap-client.pem", self.cert
+      cert = Dir["#{ETCLDAP}/asf*-ldap-client.pem"].first
+
+      # verify/obtain/write the cert
+      if not cert
+        cert = "#{ETCLDAP}/asf-ldap-client.pem"
+        File.write cert, ASF::LDAP.puppet_cert || self.extract_cert
       end
 
+      # read the current configuration file
       ldap_conf = "#{ETCLDAP}/ldap.conf"
       content = File.read(ldap_conf)
-      unless content.include? 'asf-ldap-client.pem'
-        content.gsub!(/^TLS_CACERT/, '# TLS_CACERT')
-        content.gsub!(/^TLS_REQCERT/, '# TLS_REQCERT')
+
+      # ensure that the right cert is used
+      unless content =~ /asf.*-ldap-client\.pem/
+        content.gsub!(/^TLS_CACERT/i, '# TLS_CACERT')
         content += "TLS_CACERT #{ETCLDAP}/asf-ldap-client.pem\n"
-        content += "uri #{LDAP.host}\n"
+      end
+
+      # provide the URIs of the ldap hosts
+      content.gsub!(/^URI/, '# URI')
+      content += "uri \n" unless content =~ /^uri /
+      content[/uri (.*)\n/, 1] = hosts.join(' ')
+
+      # verify/set the base
+      unless content.include? 'base dc=apache'
+        content.gsub!(/^BASE/i, '# BASE')
         content += "base dc=apache,dc=org\n"
-        content += "TLS_REQCERT allow\n" if ETCLDAP.include? 'openldap'
-        File.write(ldap_conf, content)
       end
+
+      # ensure TLS_REQCERT is allow (Mac OS/X only)
+      if ETCLDAP.include? 'openldap' and not content.include? 'REQCERT allow'
+        content.gsub!(/^TLS_REQCERT/i, '# TLS_REQCERT')
+        content += "TLS_REQCERT allow\n"
+      end
+
+      # write the configuration if there were any changes
+      File.write(ldap_conf, content) unless content == File.read(ldap_conf)
     end
   end
 end