lib/miam/exporter.rb in miam-0.1.1 vs lib/miam/exporter.rb in miam-0.1.2

@@ -1,67 +1,69 @@
 class Miam::Exporter
-  def self.export(iam, options = {}, &block)
-    self.new(iam, options).export(&block)
+  def self.export(iam, options = {})
+    self.new(iam, options).export
   end
 
   def initialize(iam, options = {})
     @iam = iam
     @options = options
+    @mutex = Mutex.new
+    @concurrency = options[:export_concurrency] || 16
   end
 
-  def export(&block)
+  def export
     users = list_users
     groups = list_groups
     roles = list_roles
     instance_profiles = list_instance_profiles
     group_users = {}
     instance_profile_roles = {}
 
-    export_options = {
-      :progress_total => (users.length + groups.length + roles.length + instance_profiles.length),
-      :progress => 0,
-    }
+    unless @options[:no_progress]
+      progress_total = users.length + groups.length + roles.length + instance_profiles.length
+      @progressbar = ProgressBar.create(:title => "Loading", :total => progress_total, :output => $stderr)
+    end
 
     expected = {
-      :users => export_users(users, group_users, export_options, &block),
-      :groups => export_groups(groups, export_options, &block),
-      :roles => export_roles(roles, instance_profile_roles, export_options, &block),
-      :instance_profiles => export_instance_profiles(instance_profiles, export_options, &block),
+      :users => export_users(users, group_users),
+      :groups => export_groups(groups),
+      :roles => export_roles(roles, instance_profile_roles),
+      :instance_profiles => export_instance_profiles(instance_profiles),
     }
 
     [expected, group_users, instance_profile_roles]
   end
 
   private
 
-  def export_users(users, group_users, export_options = {})
+  def export_users(users, group_users)
     result = {}
 
-    users.each do |user|
+    Parallel.each(users, :in_threads => @concurrency) do |user|
       user_name = user.user_name
-
       groups = export_user_groups(user_name)
+      policies = export_user_policies(user_name)
+      login_profile = export_login_profile(user_name)
 
-      groups.each do |group_name|
-        group_users[group_name] ||= []
-        group_users[group_name] << user_name
-      end
+      @mutex.synchronize do
+        groups.each do |group_name|
+          group_users[group_name] ||= []
+          group_users[group_name] << user_name
+        end
 
-      result[user_name] = {
-        :path => user.path,
-        :groups => groups,
-        :policies => export_user_policies(user_name),
-      }
+        result[user_name] = {
+          :path => user.path,
+          :groups => groups,
+          :policies => policies,
+        }
 
-      login_profile = export_login_profile(user_name)
+        if login_profile
+          result[user_name][:login_profile] = login_profile
+        end
 
-      if login_profile
-        result[user_name][:login_profile] = login_profile
+        progress
       end
-
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
     end
 
     result
   end
 
@@ -94,23 +96,25 @@
     rescue Aws::IAM::Errors::NoSuchEntity
       nil
     end
   end
 
-  def export_groups(groups, export_options = {})
+  def export_groups(groups)
     result = {}
 
-    groups.each do |group|
+    Parallel.each(groups, :in_threads => @concurrency) do |group|
       group_name = group.group_name
+      policies = export_group_policies(group_name)
 
-      result[group_name] = {
-        :path => group.path,
-        :policies => export_group_policies(group_name),
-      }
+      @mutex.synchronize do
+        result[group_name] = {
+          :path => group.path,
+          :policies => policies,
+        }
 
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
+        progress
+      end
     end
 
     result
   end
 
@@ -126,34 +130,35 @@
     end
 
     result
   end
 
-  def export_roles(roles, instance_profile_roles, export_options = {})
+  def export_roles(roles, instance_profile_roles)
     result = {}
 
-    roles.each do |role|
+    Parallel.each(roles, :in_threads => @concurrency) do |role|
       role_name = role.role_name
-
       instance_profiles = export_role_instance_profiles(role_name)
+      policies = export_role_policies(role_name)
 
-      instance_profiles.each do |instance_profile_name|
-        instance_profile_roles[instance_profile_name] ||= []
-        instance_profile_roles[instance_profile_name] << role_name
-      end
+      @mutex.synchronize do
+        instance_profiles.each do |instance_profile_name|
+          instance_profile_roles[instance_profile_name] ||= []
+          instance_profile_roles[instance_profile_name] << role_name
+        end
 
-      document = CGI.unescape(role.assume_role_policy_document)
+        document = CGI.unescape(role.assume_role_policy_document)
 
-      result[role_name] = {
-        :path => role.path,
-        :assume_role_policy_document => JSON.parse(document),
-        :instance_profiles => instance_profiles,
-        :policies => export_role_policies(role_name),
-      }
+        result[role_name] = {
+          :path => role.path,
+          :assume_role_policy_document => JSON.parse(document),
+          :instance_profiles => instance_profiles,
+          :policies => policies,
+        }
 
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
+        progress
+      end
     end
 
     result
   end
 
@@ -177,22 +182,23 @@
     end
 
     result
   end
 
-  def export_instance_profiles(instance_profiles, export_options = {})
+  def export_instance_profiles(instance_profiles)
     result = {}
 
-    instance_profiles.each do |instance_profile|
+    Parallel.each(instance_profiles, :in_threads => @concurrency) do |instance_profile|
       instance_profile_name = instance_profile.instance_profile_name
 
-      result[instance_profile_name] = {
-        :path => instance_profile.path,
-      }
+      @mutex.synchronize do
+        result[instance_profile_name] = {
+          :path => instance_profile.path,
+        }
 
-      export_options[:progress] += 1
-      yield(export_options) if block_given?
+        progress
+      end
     end
 
     result
   end
 
@@ -224,7 +230,11 @@
 
   def list_instance_profiles
     @iam.list_instance_profiles.map {|resp|
       resp.instance_profiles.to_a
     }.flatten
+  end
+
+  def progress
+    @progressbar.increment if @progressbar
   end
 end