lib/gizzmo.rb in gizzmo-0.12.1 vs lib/gizzmo.rb in gizzmo-0.13.0

@@ -2,37 +2,60 @@
 $: << File.dirname(__FILE__)
 class HelpNeededError < RuntimeError; end
 require "optparse"
 require "ostruct"
 require "gizzard"
+require "shellwords"
 require "yaml"
 
 DOC_STRINGS = {
+  "add-host" => "Add a remote cluster host to replicate to. Format: cluster:host:port.",
   "addforwarding" => "Add a forwarding from a graph_id / base_source_id to a given shard.",
   "addlink" => "Add a relationship link between two shards.",
+  "add-partition" => "Rebalance the cluster by appending new partitions to the current topology.",
+  "busy" => "List any shards with a busy flag set.",
+  "copy" => "Copy between the given list of shards. Given a set of shards, it will copy and repair to ensure that all shards have the latest data.",
   "create" => "Create shard(s) of a given Java/Scala class. If you don't know the list of available classes, you can just try a bogus class, and the exception will include a list of valid classes.",
+  "create-table" => "Create tables in an existing cluster.",
+  "delete" => "", # TODO: Undocumented
+  "deleteforwarding" => "", # TODO: Undocumented
+  "diff-shards" => "Log differences between n shards",
   "drill" => "Show shard trees for replicas of a given structure signature (from 'report').",
   "dump" => "Show shard trees for given table ids.",
   "find" => "Show all shards with a given hostname.",
+  "finish-migrate" => "", # TODO: Undocumented
   "finish-replica" => "Remove the write-only barrier in front of a shard that's finished being copied after 'setup-replica'.",
   "flush" => "Flush error queue for a given priority.",
   "forwardings" => "Get a list of all forwardings.",
   "hosts" => "List hosts used in shard names in the forwarding table and replicas.",
   "info" => "Show id/class/busy for shards.",
   "inject" => "Inject jobs (as literal json) into the server. Jobs can be linefeed-terminated from stdin, or passed as arguments. Priority is server-defined, but typically lower numbers (like 1) are lower priority.",
-  "links" => "List parent & child links for shards.",
+  "links" => "List parent and child links for shards.",
+  "list-hosts" => "List remote cluster hosts being replicated to.",
   "lookup" => "Lookup the shard id that holds the record for a given table / source_id.",
-  "markbusy" => "Mark a shard as busy.",
+  "markbusy" => "Mark a list of shards as busy.",
+  "markunbusy" => "Mark a list of shards as not busy.",
   "pair" => "Report the replica pairing structure for a list of hosts.",
+  "rebalance" => "Restructure and move shards to reflect a new list of tree structures.",
   "reload" => "Instruct application servers to reload the nameserver state.",
-  "repair-shards" => "Reconcile n shards by detecting differences and rescheduling them",
-  "diff-shards" => "Log differences between n shards",
+  "remove-host" => "Remove a remote cluster host being replicate to.",
+  "remove-partition" => "Rebalance the cluster by removing the provided partitions from the current topology.",
+  "repair-tables" => "Reconcile all the shards in the given tables (supplied with -T) by detecting differences and writing them back to shards as needed.",
   "report" => "Show each unique replica structure for a given list of shards. Usually this shard list comes from << gizzmo forwardings | awk '{ print $3 }' >>.",
+  "setup-migrate" => "", # TODO: Undocumented
   "setup-replica" => "Add a replica to be parallel to an existing replica, in write-only mode, ready to be copied to.",
+  "subtree" => "Show the subtree of replicas given a shard id.",
+  "tables" => "List the table IDs known by this nameserver.",
+  "topology" => "List the full topologies known for the table IDs provided.",
+  "transform" => "Transform from one topology to another.",
+  "transform-tree" => "Transforms given forwardings to the corresponding given tree structure.",
+  "unlink" => "Remove a link from one shard to another.",
+  "unwrap" => "Remove a wrapper created with wrap.",
   "wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
 }
 
+
 ORIGINAL_ARGV = ARGV.dup
 zero = File.basename($0)
 
 # Container for parsed options
 global_options = OpenStruct.new
@@ -44,15 +67,16 @@
 subcommand_options = OpenStruct.new
 
 # Leftover arguments
 argv = nil
 
+
 GIZZMO_VERSION = File.read(File.dirname(__FILE__) + "/../VERSION") rescue "unable to read version file"
 
 begin
   YAML.load_file(File.join(ENV["HOME"], ".gizzmorc")).each do |k, v|
-    global_options.send("#{k}=", v)
+    #global_options.send("#{k}=", v)
   end
 rescue Errno::ENOENT
   # Do nothing...
 rescue => e
   abort "Unknown error loading ~/.gizzmorc: #{e.message}"
@@ -84,10 +108,17 @@
 
 def load_config(options, filename)
   YAML.load(File.open(filename)).each do |k, v|
     k = "hosts" if k == "host"
     v = v.split(",").map {|h| h.strip } if k == "hosts"
+    if k == "template_options"
+      opts = {}
+      v.each do |k1, v1|
+        opts[k1.to_sym] = v1
+      end
+      v = opts
+    end
     options.send("#{k}=", v)
   end
 end
 
 def add_scheduler_opts(subcommand_options, opts)
@@ -98,18 +129,42 @@
     (subcommand_options.scheduler_options ||= {})[:copies_per_host] = c.to_i
   end
   opts.on("--poll-interval=SECONDS", "Sleep SECONDS between polling for copy status") do |c|
     (subcommand_options.scheduler_options ||= {})[:poll_interval] = c.to_i
   end
-  opts.on("--copy-wrapper=TYPE", "Wrap copy destination shards with TYPE. default WriteOnlyShard") do |t|
+  opts.on("--copy-wrapper=SHARD_TYPE", "Wrap copy destination shards with SHARD_TYPE. default BlockedShard") do |t|
     (subcommand_options.scheduler_options ||= {})[:copy_wrapper] = t
   end
+  opts.on("--skip-copies", "Do transformation without copying. WARNING: This is VERY DANGEROUS if you don't know what you're doing!") do
+    (subcommand_options.scheduler_options ||= {})[:skip_copies] = true
+  end
   opts.on("--no-progress", "Do not show progress bar at bottom.") do
     (subcommand_options.scheduler_options ||= {})[:no_progress] = true
   end
+  opts.on("--batch-finish", "Wait until all copies are complete before cleaning up unneeded links and shards") do
+    (subcommand_options.scheduler_options ||= {})[:batch_finish] = true
+  end
 end
 
+def add_template_opts(subcommand_options, opts)
+  opts.on("--virtual=SHARD_TYPE", "Concrete shards will exist behind a virtual shard of this SHARD_TYPE (default ReplicatingShard)") do |t|
+    (subcommand_options.template_options ||= {})[:replicating] = t
+  end
+
+  opts.on("-c", "--concrete=SHARD_TYPE", "Concrete shards will be this SHARD_TYPE (REQUIRED when using --simple)") do |t|
+    (subcommand_options.template_options ||= {})[:concrete] = t
+  end
+
+  opts.on("--source-type=DATA_TYPE", "The data type for the source column. (REQUIRED when using --simple)") do |t|
+    (subcommand_options.template_options ||= {})[:source_type] = t
+  end
+
+  opts.on("--dest-type=DATA_TYPE", "The data type for the destination column. (REQUIRED when using --simple)") do |t|
+    (subcommand_options.template_options ||= {})[:dest_type] = t
+  end
+end
+
 subcommands = {
   'create' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} create [options] CLASS_NAME SHARD_ID [MORE SHARD_IDS...]"
     separators(opts, DOC_STRINGS["create"])
 
@@ -243,25 +298,20 @@
     opts.on("--fnv", "Use FNV1A_64 hash on source") do
       subcommand_options.hash_function = :fnv
     end
   end,
   'copy' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} copy SOURCE_SHARD_ID DESTINATION_SHARD_ID"
+    opts.banner = "Usage: #{zero} copy SHARD_IDS..."
     separators(opts, DOC_STRINGS["copy"])
   end,
-  'repair-shards' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} repair-shards SHARD_IDS..."
-    separators(opts, DOC_STRINGS["repair-shards"])
+  'repair-tables' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} -T TABLE,... repair-tables [options]"
+    separators(opts, DOC_STRINGS["repair-tables"])
+    opts.on("--max-copies=COUNT", "Limit max simultaneous copies to COUNT.") do |c|
+      subcommand_options.num_copies = c.to_i
+    end
   end,
-  'diff-shards' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} diff-shards SHARD_IDS..."
-    separators(opts, DOC_STRINGS["diff-shards"])
-  end,
-  'diff-shards' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} diff-shards SOURCE_SHARD_ID DESTINATION_SHARD_ID"
-    separators(opts, DOC_STRINGS["diff-shards"])
-  end,
   'busy' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} busy"
     separators(opts, DOC_STRINGS["busy"])
   end,
   'setup-replica' => OptionParser.new do |opts|
@@ -315,52 +365,77 @@
     opts.on("--shards", "Show topology by root shard ids instead of counts") do
       subcommand_options.root_shards = true
     end
   end,
   'transform-tree' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} transform-tree [options] TEMPLATE ROOT_SHARD_ID"
+    opts.banner = "Usage: #{zero} transform-tree [options] TEMPLATE ROOT_SHARD_ID ..."
     separators(opts, DOC_STRINGS['transform-tree'])
 
     add_scheduler_opts subcommand_options, opts
+    add_template_opts subcommand_options, opts
 
     opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
       subcommand_options.quiet = true
     end
   end,
   'transform' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} transform [options] FROM_TEMPLATE TO_TEMPLATE ..."
     separators(opts, DOC_STRINGS['transform'])
 
     add_scheduler_opts subcommand_options, opts
+    add_template_opts subcommand_options, opts
 
     opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
       subcommand_options.quiet = true
     end
   end,
   'rebalance' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} rebalance [options] WEIGHT TO_TEMPLATE ..."
     separators(opts, DOC_STRINGS["rebalance"])
 
     add_scheduler_opts subcommand_options, opts
+    add_template_opts subcommand_options, opts
 
     opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
       subcommand_options.quiet = true
     end
   end,
+  'add-partition' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} add-partition [options] TEMPLATE ..."
+    separators(opts, DOC_STRINGS["add-partition"])
+
+    add_scheduler_opts subcommand_options, opts
+
+    opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
+      subcommand_options.quiet = true
+    end
+  end,
+  'remove-partition' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} remove-partition [options] TEMPLATE ..."
+    separators(opts, DOC_STRINGS["remove-partition"])
+
+    add_scheduler_opts subcommand_options, opts
+
+    opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
+      subcommand_options.quiet = true
+    end
+  end,
   'create-table' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} create-table [options] WEIGHT TEMPLATE ..."
     separators(opts, DOC_STRINGS["create-table"])
 
+    add_template_opts subcommand_options, opts
+
     opts.on("--shards=COUNT", "Create COUNT shards for each table.") do |count|
       subcommand_options.shards = count.to_i
     end
 
-    opts.on("--min-id=NUM", "Set lower bound on the id space to NUM (default min signed long: -1 * 2^63)") do |min_id|
+    opts.on("--min-id=NUM", "Set lower bound on the id space to NUM (default 0)") do |min_id|
       subcommand_options.min_id = min_id.to_i
     end
 
-    opts.on("--max-id=NUM", "Set upper bound on the id space to NUM (default max signed long: 2^63 - 1)") do |max_id|
+    opts.on("--max-id=NUM", "Set upper bound on the id space to NUM (default 2^60 - 1)") do |max_id|
       subcommand_options.max_id = max_id.to_i
     end
 
     opts.on("--base-name=NAME", "Use NAME as the base prefix for each shard's table prefix (default 'shard')") do |base_name|
       subcommand_options.base_name = base_name
@@ -385,10 +460,12 @@
   opts.separator ""
   opts.separator "You can type `#{zero} help SUBCOMMAND` for help on a specific subcommand. It's"
   opts.separator "also useful to remember that global options come *before* the subcommand, while"
   opts.separator "subcommand options come *after* the subcommand."
   opts.separator ""
+  opts.separator "You can find explanations and example usage on the wiki (go/gizzmo)."
+  opts.separator ""
   opts.separator "You may find it useful to create a ~/.gizzmorc file, which is simply YAML"
   opts.separator "key/value pairs corresponding to options you want by default. A common .gizzmorc"
   opts.separator "simply contains:"
   opts.separator ""
   opts.separator "    hosts: localhost"
@@ -406,27 +483,27 @@
     opts.separator base
   end
   opts.separator ""
   opts.separator ""
   opts.separator "Global options:"
-  opts.on("-H", "--hosts=HOST[,HOST,...]", "HOSTS of application servers") do |hosts|
+  opts.on("-H", "--hosts=HOST[,HOST,...]", "Comma-delimited list of application servers") do |hosts|
     global_options.hosts = hosts.split(",").map {|h| h.strip }
   end
 
-  opts.on("-P", "--port=PORT", "PORT of remote manager service. default 7920") do |port|
+  opts.on("-P", "--port=PORT", "PORT of remote manager service (default 7920)") do |port|
     global_options.port = port.to_i
   end
 
-  opts.on("-I", "--injector=PORT", "PORT of remote job injector service. default 7921") do |port|
+  opts.on("-I", "--injector=PORT", "PORT of remote job injector service (default 7921)") do |port|
     global_options.injector_port = port.to_i
   end
 
   opts.on("-T", "--tables=TABLE[,TABLE,...]", "TABLE ids of forwardings to affect") do |tables|
     global_options.tables = tables.split(",").map {|t| t.to_i }
   end
 
-  opts.on("-F", "--framed", "use the thrift framed transport") do |framed|
+  opts.on("-F", "--framed", "Use the thrift framed transport") do |framed|
     global_options.framed = true
   end
 
   opts.on("-r", "--retry=TIMES", "TIMES to retry the command") do |r|
     global_options.retry = r.to_i
@@ -446,10 +523,14 @@
 
   opts.on("-D", "--dry-run", "") do
     global_options.dry = true
   end
 
+  opts.on("-s", "--simple", "Represent shard templates in a simple format") do
+    (global_options.template_options ||= {})[:simple] = true #This is a temporary setting until the nameserver design changes match the simpler format
+  end
+
   opts.on("-C", "--config=YAML_FILE", "YAML_FILE of option key/values") do |filename|
     load_config(global_options, filename)
   end
 
   opts.on("-L", "--log=LOG_FILE", "Path to LOG_FILE") do |file|
@@ -457,10 +538,14 @@
   end
 
   opts.on("-f", "--force", "Don't display confirmation dialogs") do |force|
     global_options.force = force
   end
+ 
+  opts.on("--argv=FILE", "Put the contents of FILE onto the command line") do |f|
+    ARGV.push *Shellwords.shellwords(File.read(f))
+  end
 
   opts.on_tail("-v", "--version", "Show version") do
     puts GIZZMO_VERSION
     exit
   end
@@ -470,11 +555,10 @@
 if ARGV.length == 0
   STDERR.puts global
   exit 1
 end
 
-# This
 def process_nested_parsers(global, subcommands)
   begin
     global.order!(ARGV) do |subcommand_name|
       # puts args.inspect
       subcommand = subcommands[subcommand_name]
@@ -526,9 +610,10 @@
   end
 end
 
 begin
   custom_timeout(global_options.timeout) do
+    Gizzard::ShardTemplate.configure((global_options.template_options || {}).merge(subcommand_options.template_options || {}))
     Gizzard::Command.run(subcommand_name, global_options, argv, subcommand_options, log)
   end
 rescue HelpNeededError => e
   if e.class.name != e.message
     STDERR.puts("=" * 80)