# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
#
# Licensed under the BSD-3 license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the root of the project or at
#
#     http://egt-labs.com/mu/LICENSE.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module MU

  # Scrape cloud providers for existing resources, and reverse-engineer them
  # into runnable {MU::Config} descriptors and/or {MU::MommaCat} deploy objects.
  class Adoption

    attr_reader :found

    # Error class for objects which fail to fully resolve (e.g. references to 
    # other objects which are not found)
    class Incomplete < MU::MuNonFatal; end

    # Presets methods we use to clump discovered resources into discrete deploys
    GROUPMODES = {
      :logical => "Group resources in logical layers (folders and habitats together, users/roles/groups together, network resources together, etc)",
      :omnibus => "Jam everything into one monolothic configuration"
    }


    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false)
      @scraped = {}
      @clouds = clouds
      @types = types
      @parent = parent
      @boks = {}
      @billing = billing
      @reference_map = {}
      @sources = sources
      @target_creds = credentials
      @group_by = group_by
      @savedeploys = savedeploys
      @diff = diff
      @habitats = habitats
      @regions = regions
      @habitats ||= []
      @scrub_mu_isms = scrub_mu_isms
      @merge = merge
    end

    # Walk cloud providers with available credentials to discover resources
    def scrapeClouds()
      @default_parent = nil

      @clouds.each { |cloud|
        cloudclass = MU::Cloud.cloudClass(cloud)
        next if cloudclass.listCredentials.nil?

        if cloud == "Google" and !@parent and @target_creds
          dest_org = MU::Cloud::Google.getOrg(@target_creds)
          if dest_org
            @default_parent = dest_org.name
          end
        end

        cloudclass.listCredentials.each { |credset|
          next if @sources and !@sources.include?(credset)
          cfg = cloudclass.credConfig(credset)
          if cfg and cfg['restrict_to_habitats']
            cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
          end

          if @parent
# TODO handle different inputs (cloud_id, etc)
# TODO do something about vague matches
            found = MU::MommaCat.findStray(
              cloud,
              "folders",
              flags: { "display_name" => @parent },
              credentials: credset,
              allow_multi: false,
              dummy_ok: true,
              debug: false
            )
            if found and found.size == 1
              @default_parent = found.first
            end
          end

          @types.each { |type|
            begin
              resclass = MU::Cloud.resourceClass(cloud, type)
            rescue ::MU::Cloud::MuCloudResourceNotImplemented
              next
            end
            if !resclass.instance_methods.include?(:toKitten)
              MU.log "Skipping MU::Cloud::#{cloud}::#{type} (resource has not implemented #toKitten)", MU::WARN
              next
            end
            MU.log "Scraping #{cloud}/#{credset} for #{resclass.cfg_plural}"

            found = MU::MommaCat.findStray(
              cloud,
              type,
              credentials: credset,
              allow_multi: true,
              habitats: @habitats.dup,
              region: @regions,
              dummy_ok: true,
              skip_provider_owned: true,
#              debug: false#,
            )


            if found and found.size > 0
              if resclass.cfg_plural == "habitats"
                found.reject! { |h|
                  !cloudclass.listHabitats(credset).include?(h.cloud_id)
                }
              end
              MU.log "Found #{found.size.to_s} raw #{resclass.cfg_plural} in #{cloud}"
              @scraped[type] ||= {}
              found.each { |obj|
                if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat)
                  next
                end
                # XXX apply any filters (e.g. MU-ID tags)
                if obj.cloud_id.nil?
                  MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj
                  exit
                end
                @scraped[type][obj.cloud_id] = obj
              }
            end

          }
        }
      }

      if @parent and !@default_parent
        MU.log "Failed to locate a folder that resembles #{@parent}", MU::ERR
      end
      MU.log "Scraping complete"

      @scraped
    end

    # Given a list of BoK style tags, try to reverse-engineer the correct
    # Basket of Kittens shorthand name of the resource that owns them. Mostly
    # this infers from Mu-style tagging, but we'll add a couple cases for
    # special cloud provider cases.
    # @param tags [Array<Hash>]
    # @param basename [String]
    # return [String]
    def self.tagsToName(tags = [], basename: nil)
      tags.each { |tag|
        if tag['key'] == "aws:cloudformation:logical-id"
          return tag['value']
        end
      }
      muid = nil
      tags.each { |tag|
        if tag['key'] == "MU-ID" or tag['key'] == "mu-id"
          muid = tag['value']
          break
        end
      }

      tags.each { |tag|
        if tag['key'] == "Name"
          if muid and tag['value'].match(/^#{Regexp.quote(muid)}-(.*)/)
            return Regexp.last_match[1].downcase
          else
            return tag['value'].downcase
          end
        end
      }

      if basename and muid and basename.match(/^#{Regexp.quote(muid)}-(.*)/)
        return Regexp.last_match[1].downcase
      end

      nil
    end

    # Generate a {MU::Config} (Basket of Kittens) hash using our discovered
    # cloud objects.
    # @return [Hash]
    def generateBaskets(prefix: "")
      groupings = {
        "" =>  MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
      }

      # XXX as soon as we come up with a method that isn't about what resource
      # type you are, this code will stop making sense
      if @group_by == :logical
        groupings = {
          "spaces" => ["folders", "habitats"],
          "people" => ["users", "groups", "roles"],
          "network" => ["vpcs", "firewall_rules", "dnszones"],
          "storage" => ["storage_pools", "buckets"],
        }
        # "the movie star/and the rest"
        groupings["services"] = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] } - groupings.values.flatten
      elsif @group_by == :omnibus
        prefix = "mu" if prefix.empty? # so that appnames aren't ever empty
      end

      # Find any previous deploys with this particular profile, which we'll use
      # later for --diff.
      @existing_deploys = {}
      @existing_deploys_by_id = {}
      @origins = {}
      @types_found_in = {}
      groupings.each_pair { |appname, types|
        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
        next if (types & allowed_types).size == 0
        origin = {
          "appname" => prefix+appname,
          "types" => (types & allowed_types).sort,
          "habitats" => @habitats.sort,
          "group_by" => @group_by.to_s
        }

        @existing_deploys[appname] = MU::MommaCat.findMatchingDeploy(origin)
        if @existing_deploys[appname]
          @existing_deploys_by_id[@existing_deploys[appname].deploy_id] = @existing_deploys[appname]
          @origins[appname] = origin
          origin['types'].each { |t|
            @types_found_in[t] = @existing_deploys[appname]
          }
        end
      }

      groupings.each_pair { |appname, types|
        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
        next if (types & allowed_types).size == 0

        bok = { "appname" => prefix+appname }
        if @scrub_mu_isms
          bok["scrub_mu_isms"] = true
        end
        if @target_creds
          bok["credentials"] = @target_creds
        end

        count = 0
        if @diff
          if !@existing_deploys[appname]
            MU.log "--diff was set but I failed to find a deploy like '#{appname}' to compare to (have #{@existing_deploys.keys.join(", ")})", MU::ERR, details: @origins[appname]
            exit 1
          else
            MU.log "Will diff current live resources against #{@existing_deploys[appname].deploy_id}", MU::NOTICE, details: @origins[appname]
          end
        end

        threads = []
        timers = {}
        walltimers = {}
        @clouds.each { |cloud|
          @scraped.each_pair { |type, resources|
            typestart = Time.now
            res_class = begin
              MU::Cloud.resourceClass(cloud, type)
            rescue MU::Cloud::MuCloudResourceNotImplemented
              # XXX I don't think this can actually happen
              next
            end
            next if !types.include?(res_class.cfg_plural)

            bok[res_class.cfg_plural] ||= []
            timers[type] ||= {}

            class_semaphore = Mutex.new

            Thread.abort_on_exception = true
            resources.values.each { |obj_thr|
              obj_desc = nil
              begin
                obj_desc = obj_thr.cloud_desc
              rescue StandardError
              ensure
                if !obj_desc
                  MU.log cloud+" "+type.to_s+" "+obj_thr.cloud_id+" did not return a cloud descriptor, skipping", MU::WARN
                  next
                end
              end
              threads << Thread.new(obj_thr) { |obj|
                start = Time.now

                kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types)
                if kitten_cfg
                  print "."
                  kitten_cfg.delete("credentials") if @target_creds
                  class_semaphore.synchronize {
                    bok[res_class.cfg_plural] << kitten_cfg
                    if !kitten_cfg['cloud_id']
                      MU.log "No cloud id in this #{res_class.cfg_name} kitten!", MU::ERR, details: kitten_cfg
                    end
                    timers[type][kitten_cfg['cloud_id']] = (Time.now - start)
                  }
                  count += 1
                end
              }

            }

            threads.each { |t|
              t.join
            }

            puts ""
            bok[res_class.cfg_plural].sort! { |a, b|
              strs = [a, b].map { |x|
                if x['cloud_id']
                  x['cloud_id']
                elsif x['parent'] and ['parent'].respond_to?(:id) and kitten_cfg['parent'].id
                  x['name']+x['parent'].id
                elsif x['project']
                  x['name']+x['project']
                else
                  x['name']
                end
              }
              strs[0] <=> strs[1]
            }

            # If we've got duplicate names in here, try to deal with it
            bok[res_class.cfg_plural].each { |kitten_cfg|
              bok[res_class.cfg_plural].each { |sibling|
                next if kitten_cfg == sibling
                if sibling['name'] == kitten_cfg['name']
                  MU::Adoption.deDuplicateName(kitten_cfg, res_class)
                  MU.log "De-duplication: Renamed #{res_class.cfg_name} name '#{sibling['name']}' => '#{kitten_cfg['name']}'", MU::NOTICE
                  break
                end
              }
            }
            walltimers[type] ||= 0
            walltimers[type] += (Time.now - typestart)
          }
        }

        timers.each_pair { |type, resources|
          next if resources.empty?
          total = resources.values.sum
          top_5 =  resources.keys.sort { |a, b|
            resources[b] <=> resources[a]
          }.slice(0, 5).map { |k|
            k.to_s+": "+sprintf("%.2fs", resources[k])
          }
          if walltimers[type] < 45
            MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])}"
          else
            MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])} (CPU time #{sprintf("%.2fs", total)}, avg #{sprintf("%.2fs", total/resources.size)}). Top 5:", MU::NOTICE, details: top_5
          end
        }

        # No matching resources isn't necessarily an error
        next if count == 0 or bok.nil?

# Now walk through all of the Refs in these objects, resolve them, and minimize
# their config footprint
        MU.log "Minimizing footprint of #{count.to_s} found resources", MU::DEBUG

        generated_deploy = generateStubDeploy(bok)
        @boks[bok['appname']] = vacuum(bok, origin: @origins[appname], deploy: generated_deploy, save: @savedeploys)

        if @diff and !@existing_deploys[appname]
          MU.log "diff flag set, but no comparable deploy provided for #{bok['appname']}", MU::ERR
          exit 1
        end

        if @diff
          prev_vacuumed = vacuum(@existing_deploys[appname].original_config, deploy: @existing_deploys[appname], keep_missing: true, copy_from: generated_deploy)
          prevcfg = MU::Config.manxify(prev_vacuumed)
          if !prevcfg
            MU.log "#{@existing_deploys[appname].deploy_id} didn't have a working original config for me to compare", MU::ERR
            exit 1
          end
          newcfg = MU::Config.manxify(@boks[bok['appname']])
          report = prevcfg.diff(newcfg)

          if report

            if MU.muCfg['adopt_change_notify']
              notifyChanges(@existing_deploys[appname], report.freeze)
            end
            if @merge
              MU.log "Saving changes to #{@existing_deploys[appname].deploy_id}"
              @existing_deploys[appname].updateBasketofKittens(newcfg, save_now: true)
            end
          end

        end
      }
      @boks
    end

    private

    # @param tier [Hash]
    # @param parent_key [String]
    def crawlChangeReport(tier, parent_key = nil, indent: "")
      report = []
      if tier.is_a?(Array)
        tier.each { |a|
          sub_report = crawlChangeReport(a, parent_key)
          report.concat(sub_report) if sub_report and !sub_report.empty?
        }
      elsif tier.is_a?(Hash)
        if tier[:action]
          preposition = if tier[:action] == :added
            "to"
          elsif tier[:action] == :removed
            "from"
          else
            "in"
          end

          name = ""
          type_of = parent_key.sub(/s$|\[.*/, '') if parent_key
          loc = tier[:habitat]

          if tier[:value] and tier[:value].is_a?(Hash)
            name, loc = MU::MommaCat.getChunkName(tier[:value], type_of)
          elsif parent_key
            name = parent_key
          end

          path_str = []
          slack_path_str = ""
          if tier[:parents] and tier[:parents].size > 2
            path = tier[:parents].clone
            slack_path_str += "#{preposition} \*"+path.join(" ⇨ ")+"\*" if path.size > 0
            path.shift
            path.shift
            path.pop if path.last == name
            for c in (0..(path.size-1)) do
              path_str << ("  " * (c+2)) + (path[c] || "<nil>")
            end
          end
          path_str << "" if !path_str.empty?

          plain = (name ? name : type_of) if name or type_of
          plain ||= "" # XXX but this is a problem
          slack = "`"+plain+"`"

          plain += " ("+loc+")" if loc and !loc.empty?
          color = plain

          if tier[:action] == :added
            color = "+ ".green + plain
            plain = "+ " + plain
            slack += " added"
          elsif tier[:action] == :removed
            color = "- ".red + plain
            plain = "- " + plain
            slack += " removed"
          end

          slack += " #{tier[:action]} #{preposition} \*#{loc}\*" if loc and !loc.empty? and [Array, Hash].include?(tier[:value].class)

          plain = path_str.join(" => \n") + indent + plain
          color = path_str.join(" => \n") + indent + color

          slack += " "+slack_path_str if !slack_path_str.empty?
          myreport = {
            "slack" => slack,
            "plain" => plain,
            "color" => color
          }

          append = ""
          if tier[:value] and (tier[:value].is_a?(Array) or tier[:value].is_a?(Hash))
            if tier[:value].is_a?(Hash)
              if name
                tier[:value].delete("entity")
                tier[:value].delete(name.sub(/\[.*/, '')) if name
              end
              if (tier[:value].keys - ["id", "name", "type"]).size > 0
                myreport["details"] = tier[:value].clone
                append = PP.pp(tier[:value], '').gsub(/(^|\n)/, '\1'+indent)
              end
            else
              append = indent+"["+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s.light_blue }.join(", ")+"]"
              slack += " #{tier[:action].to_s}: "+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s }.join(", ")
            end
          else
            tier[:value] ||= "<nil>"
            if ![:removed].include?(tier[:action])
              myreport["slack"] += ". New #{tier[:field] ? "`"+tier[:field]+"`" : :value}: \*#{tier[:value]}\*"
            else
              myreport["slack"] += " (was \*#{tier[:value]}\*)"
            end
            append = tier[:value].to_s.bold
          end

          if append and !append.empty?
            myreport["plain"] += " =>\n  "+indent+append
            myreport["color"] += " =>\n  "+indent+append
          end

          report << myreport if tier[:action]
        end

        # Just because we've got changes at this level doesn't mean there aren't
        # more further down.
        tier.each_pair { |k, v|
          next if !(v.is_a?(Hash) or v.is_a?(Array))
          sub_report = crawlChangeReport(v, k, indent: indent+"  ")
          report.concat(sub_report) if sub_report and !sub_report.empty?
        }
      end

      report
    end


    def notifyChanges(deploy, report)
      snippet_threshold = (MU.muCfg['adopt_change_notify'] && MU.muCfg['adopt_change_notify']['slack_snippet_threshold']) || 5

      report.each_pair { |res_type, resources|
        shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(res_type, false)
        next if !shortclass # we don't really care about Mu metadata changes
        resources.each_pair { |name, data|
          if MU::MommaCat.getChunkName(data[:value], res_type).first.nil?
            symbol = if data[:action] == :added
              "+".green
            elsif data[:action] == :removed
              "-".red
            else
              "~".yellow
            end
            puts (symbol+" "+res_type+"["+name+"]")
          end

          noun = shortclass ? shortclass.to_s : res_type.capitalize
          verb = if data[:action]
            data[:action].to_s
          else
            "modified"
          end

          changes = crawlChangeReport(data.freeze, res_type)

          slacktext = "#{noun} \*#{name}\* was #{verb}"
          if data[:habitat]
            slacktext += " in \*#{data[:habitat]}\*"
          end
          snippets = []

          if [:added, :removed].include?(data[:action]) and data[:value]
            snippets << { text: "```"+JSON.pretty_generate(data[:value])+"```" }
          else
            changes.each { |c|
              slacktext += "\n • "+c["slack"]
              if c["details"]
                details = JSON.pretty_generate(c["details"])
                snippets << { text: "```"+JSON.pretty_generate(c["details"])+"```" }
              end
            }
          end

          changes.each { |c|
            puts c["color"]
          }
          puts ""

          if MU.muCfg['adopt_change_notify'] and MU.muCfg['adopt_change_notify']['slack']
            deploy.sendAdminSlack(slacktext, scrub_mu_isms: MU.muCfg['adopt_scrub_mu_isms'], snippets: snippets, noop: false)
          end

        }
      }

    end

    def scrubSchemaDefaults(conf_chunk, schema_chunk, depth = 0, type: nil)
      return if schema_chunk.nil?

      if !conf_chunk.nil? and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash)
        deletia = []
        schema_chunk["properties"].each_pair { |key, subschema|
          next if !conf_chunk[key]
          shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key, false)

          if subschema["default_if"]
            subschema["default_if"].each { |cond|
              if conf_chunk[cond["key_is"]] == cond["value_is"]
                subschema["default"] = cond["set"]
                break
              end
            }
          end

          if subschema["default"] and conf_chunk[key] == subschema["default"]
            deletia << key
          elsif ["array", "object"].include?(subschema["type"])
            scrubSchemaDefaults(conf_chunk[key], subschema, depth+1, type: shortclass)
          end
        }
        deletia.each { |key| conf_chunk.delete(key) }
      elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
        conf_chunk.each { |item|
          # this bit only happens at the top-level key for a resource type, in
          # theory
          realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])

            _toplevel_required, cloudschema = MU::Cloud.resourceClass(item['cloud'], type).schema(self)

            newschema = schema_chunk["items"].dup
            newschema["properties"].merge!(cloudschema)
            newschema
          else
            schema_chunk["items"].dup
          end
          next if ["array", "object"].include?(realschema["type"])

          scrubSchemaDefaults(item, realschema, depth+1, type: type)
        }
      end

      conf_chunk
    end

    # Recursively walk through a BoK hash, validate all {MU::Config::Ref}
    # objects, convert them to hashes, and pare them down to the minimal
    # representation (remove extraneous attributes that match the parent
    # object).
    # Do the same for our main objects: if they all use the same credentials,
    # for example, remove the explicit +credentials+ attributes and set that
    # value globally, once.
    def vacuum(bok, origin: nil, save: false, deploy: nil, copy_from: nil, keep_missing: false)

      globals = {
        'cloud' => {},
        'credentials' => {},
        'region' => {},
        'billing_acct' => {},
        'us_only' => {},
      }
      MU::Cloud.resource_types.values.each { |attrs|
        if bok[attrs[:cfg_plural]]
          processed = []
          bok[attrs[:cfg_plural]].each { |resource|
            globals.each_pair { |field, counts|
              if resource[field]
                counts[resource[field]] ||= 0
                counts[resource[field]] += 1
              end
            }
            obj = deploy.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
            inject_metadata = save
            if obj.nil? and copy_from
              obj = copy_from.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
              if obj
                inject_metadata = true
                obj.intoDeploy(deploy, force: true)
              end
            end

            begin
              raise Incomplete if obj.nil?
              if inject_metadata
                deploydata = obj.notify
                deploy.notify(attrs[:cfg_plural], resource['name'], deploydata, triggering_node: obj)
              end
              new_cfg = resolveReferences(resource, deploy, obj)
              new_cfg.delete("cloud_id")
              cred_cfg = MU::Cloud.cloudClass(obj.cloud).credConfig(obj.credentials)
              if cred_cfg['region'] == new_cfg['region']
                new_cfg.delete('region')
              end
              if cred_cfg['default']
                new_cfg.delete('credentials')
                new_cfg.delete('habitat')
              end
              processed << new_cfg
            rescue Incomplete
              if keep_missing
                processed << resource
              else
                MU.log "#{attrs[:cfg_name]} #{resource['name']} didn't show up from findLitterMate", MU::WARN, details: deploy.original_config[attrs[:cfg_plural]].reject { |r| r['name'] != "" }
              end
            end
          }

          deploy.original_config[attrs[:cfg_plural]] = processed
          bok[attrs[:cfg_plural]] = processed
        end
      }

      # Pare out global values like +cloud+ or +region+ that appear to be
      # universal in the deploy we're creating.
      scrub_globals = Proc.new { |h, field|
        if h.is_a?(Hash)
          newhash = {}
          h.each_pair { |k, v|
            next if k == field
            newhash[k] = scrub_globals.call(v, field)
          }
          h = newhash
        elsif h.is_a?(Array)
          newarr = []
          h.each { |v|
            newarr << scrub_globals.call(v, field)
          }
          h = newarr.uniq
        end
        h
      }

      globals.each_pair { |field, counts|
        next if counts.size != 1
        bok[field] = counts.keys.first
        MU.log "Setting global default #{field} to #{bok[field]} (#{deploy.deploy_id})", MU::DEBUG
        MU::Cloud.resource_types.values.each { |attrs|
          if bok[attrs[:cfg_plural]]
            new_resources = []
            bok[attrs[:cfg_plural]].each { |resource|
              new_resources << scrub_globals.call(resource, field)
            }
            bok[attrs[:cfg_plural]] = new_resources
          end
        }
      }

      scrubSchemaDefaults(bok, MU::Config.schema)

      if save
        MU.log "Committing adopted deployment to #{MU.dataDir}/deployments/#{deploy.deploy_id}", MU::NOTICE, details: origin
        deploy.save!(force: true, origin: origin)
      end

      bok
    end

    def resolveReferences(cfg, deploy, parent)
      mask_deploy_id = false

      check_deploy_id = Proc.new { |cfgblob|
        (deploy and
         (cfgblob.is_a?(MU::Config::Ref) or cfgblob.is_a?(Hash)) and
         cfgblob['deploy_id'] and
         cfgblob['deploy_id'] != deploy.deploy_id and
         @diff and
         @types_found_in[cfgblob['type']] and
         @types_found_in[cfgblob['type']].deploy_id == cfgblob['deploy_id']
        )
      }

      mask_deploy_id = check_deploy_id.call(cfg)

      if cfg.is_a?(MU::Config::Ref)
        if mask_deploy_id
          cfg.delete("deploy_id")
          cfg.delete("mommacat")
          cfg.kitten(deploy)
        else
          cfg.kitten(deploy) || cfg.kitten
        end

        hashcfg = cfg.to_h

        if cfg.kitten
          littermate = deploy.findLitterMate(type: cfg.type, name: cfg.name, cloud_id: cfg.id, habitat: cfg.habitat)

          if littermate and littermate.config['name']
            hashcfg['name'] = littermate.config['name']
            hashcfg.delete("id") if hashcfg["name"]
            hashcfg
          elsif cfg.deploy_id and cfg.name and @savedeploys
            hashcfg.delete("id") if hashcfg["name"]
            hashcfg
          elsif cfg.id
            littermate = deploy.findLitterMate(type: cfg.type, cloud_id: cfg.id, habitat: cfg.habitat)
            if littermate and littermate.config['name']
              hashcfg['name'] = littermate.config['name']
              hashcfg.delete("id") if hashcfg["name"]
            elsif !@savedeploys
              hashcfg.delete("deploy_id")
              hashcfg.delete("name")
            else
              hashcfg.delete("name") if cfg.id and !cfg.deploy_id
            end
          end
        elsif hashcfg["id"] and !hashcfg["name"]
          hashcfg.delete("deploy_id")
        else
          pp parent.cloud_desc
          raise Incomplete, "Failed to resolve reference on behalf of #{parent}"
        end
        hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id

        if parent and parent.config
          cred_cfg = MU::Cloud.cloudClass(parent.cloud).credConfig(parent.credentials)

          if parent.config['region'] == hashcfg['region'] or
             cred_cfg['region'] == hashcfg['region']
            hashcfg.delete("region")
          end

          habitat_id = if cfg.habitat
            if cfg.habitat.is_a?(MU::Config::Ref)
              cfg.habitat.id
            else
              cfg.habitat['id']
            end
          else
            nil
          end

          if habitat_id
            if (parent.config['habitat'] and parent.config['habitat']['id'] == habitat_id) or
               cred_cfg['account_number'] == habitat_id or # AWS
               cred_cfg['project'] == habitat_id or # GCP
               cred_cfg['subscription'] == habitat_id # Azure
              hashcfg.delete('habitat') 
            end
          end

          if parent.config['credentials'] == hashcfg['credentials']
            hashcfg.delete("credentials")
          end
        end
        cfg = hashcfg
      elsif cfg.is_a?(Hash)
        deletia = []
        cfg.each_pair { |key, value|
          begin
            cfg[key] = resolveReferences(value, deploy, parent)
          rescue Incomplete
            MU.log "Dropping unresolved key #{key}", MU::WARN, details: cfg
            deletia << key
          end
        }
        deletia.each { |key|
          cfg.delete(key)
        }
        cfg = nil if cfg.empty? and deletia.size > 0
      elsif cfg.is_a?(Array)
        new_array = []
        cfg.each { |value|
          begin
            new_item = resolveReferences(value, deploy, parent)
            if !new_item
              MU.log "Dropping unresolved value", MU::WARN, details: value
            else
              new_array << new_item
            end
          rescue Incomplete
            MU.log "Dropping unresolved value", MU::WARN, details: value
          end
        }
        cfg = new_array.uniq
      end

      if mask_deploy_id or check_deploy_id.call(cfg)
        cfg.delete("deploy_id")
        MU.log "#{parent} in #{deploy.deploy_id} references something in #{@types_found_in[cfg['type']].deploy_id}, ditching extraneous deploy_id", MU::DEBUG, details: cfg.to_h
      end

      cfg
    end

    # @return [MU::MommaCat]
    def generateStubDeploy(bok)
#      hashify Ref objects before passing into here... or do we...?

      time = Time.new
      timestamp = time.strftime("%Y%m%d%H").to_s;
      timestamp.freeze

      retries = 0
      deploy_id = nil
      seed = nil
      begin
        raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
        seedsize = 1 + (retries/10).abs
        seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
        deploy_id = bok['appname'].upcase + "-ADOPT-" + timestamp + "-" + seed.upcase
      end while MU::MommaCat.deploy_exists?(deploy_id) or seed == "mu" or seed[0] == seed[1]

      MU.setVar("deploy_id", deploy_id)
      MU.setVar("appname", bok['appname'].upcase)
      MU.setVar("environment", "ADOPT")
      MU.setVar("timestamp", timestamp)
      MU.setVar("seed", seed)
      MU.setVar("handle", MU::MommaCat.generateHandle(seed))

      deploy = MU::MommaCat.new(
        deploy_id,
        create: true,
        config: bok,
        environment: "adopt",
        appname: bok['appname'].upcase,
        timestamp: timestamp,
        nocleanup: true,
        no_artifacts: !(@savedeploys),
        set_context_to_me: true,
        mu_user: MU.mu_user
      )

      MU::Cloud.resource_types.each_pair { |typename, attrs|
        if bok[attrs[:cfg_plural]]
          bok[attrs[:cfg_plural]].each { |kitten|

            if !@scraped[typename][kitten['cloud_id']]
              MU.log "No object in scraped tree for #{attrs[:cfg_name]} #{kitten['cloud_id']} (#{kitten['name']})", MU::ERR, details: kitten
              if kitten['cloud_id'].nil?
                pp caller
                exit
              end
              next
            end

            MU.log "Inserting #{attrs[:cfg_name]} #{kitten['name']} (#{kitten['cloud_id']}) into stub deploy", MU::DEBUG, details: @scraped[typename][kitten['cloud_id']]

            @scraped[typename][kitten['cloud_id']].config!(kitten)

            deploy.addKitten(
              attrs[:cfg_plural],
              kitten['name'],
              @scraped[typename][kitten['cloud_id']],
              do_notify: true
            )
          }
        end
      }

      deploy
    end

    def self.deDuplicateName(kitten_cfg, res_class)
      orig_name = kitten_cfg['name'].dup
      if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['parent'].id
      elsif kitten_cfg['project']
        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['project']
      elsif kitten_cfg['region']
        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['region']
      elsif kitten_cfg['cloud_id']
        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
      else
        raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{orig_name} and couldn't come up with a good way to differentiate them"
      end
    end

    # Go through everything we've scraped and update our mappings of cloud ids
    # and bare name fields, so that resources can reference one another
    # portably by name.
    def catalogResources
    end

  end
end