require 'yaml' require 'json' require 'pp' require 'net/http' module GrafanaRb class Cli def initialize(argv) @argv = argv @panel_id = 0 end def run if @argv[0] == "run" || @argv[0] == "r" || @argv[0] == "q" apply else usage end end private CONFIG_FILE = "grafana.yml" REQUIRE_DEFAULT_TEMPLATE = ["*.yml"] DEFAULT_GRAFANA_USER = "admin" DEFAULT_GRAFANA_PASSWORD = "admin" TAG = "grafana-rb" def request(method, path, params = {}) uri = URI("http://" + config["grafana_url"] + path) http = Net::HTTP.new(uri.host, uri.port) # http.set_debug_output $stderr if method == :get req = Net::HTTP::Get.new(uri) else if method == :post req = Net::HTTP::Post.new(uri) elsif method == :patch req = Net::HTTP::Patch.new(uri) elsif method == :put req = Net::HTTP::Put.new(uri) elsif method == :delete req = Net::HTTP::Delete.new(uri) else raise "unknown method #{method}" end req["Content-Type"] = "application/json" req.body = params.to_json end req.basic_auth grafana_user, grafana_password JSON.load(http.request(req).body) end def die(msg = nil) if msg puts "FATAL: #{msg}" else puts "exit with status code -1" end exit -1 end def check_config_existance unless File.exists?(CONFIG_FILE) puts "FATAL: missed #{CONFIG_FILE} in current directory" puts "#{CONFIG_FILE} example:" puts "" puts "---" puts "grafana_url: monitoring.your-domain.com:3000 # required" puts "prometheus_url: monitoring.your-domain.com:9090 # required" puts "grafana_user: user # 'admin' if missed" puts "grafana_password: secret # 'admin' if missed" puts "slack_channel: #alerts # no slack notifications if missed" puts "slack_url: https://hooks.slack.com/services/... # no slack notifications if missed" puts "slack_mention: @john, @michelle # optional" puts "require: [main.yml, dash*.yml] # optional, [*.yml] by default" puts "" die end end def config @config ||= YAML.load(File.read(CONFIG_FILE)) || {} end def grafana_user config["grafana_user"] || DEFAULT_GRAFANA_USER end def grafana_password config["grafana_password"] || DEFAULT_GRAFANA_PASSWORD end def require_files @require_files ||= (config["require"] || REQUIRE_DEFAULT_TEMPLATE).flat_map { |template| Dir[template] }.map { |path| File.expand_path(path) } - [File.expand_path(CONFIG_FILE)] end def dashboards @dashboards ||= require_files.map { |path| yaml = YAML.load(File.read(path)) yaml["name"] ||= File.basename(path, File.extname(path)) yaml } end def create_or_update_prometheus_datasource payload = { name: "prometheus", type: "prometheus", url: "http://" + config["prometheus_url"], access: "proxy" } if @datasource_id = request(:get, "/api/datasources/id/prometheus")["id"] puts "update datasource" request(:put, "/api/datasources/#{@datasource_id}", payload) else puts "create datasource" @datasource_id = request(:post, "/api/datasources", payload)["id"] end end def create_or_update_slack_notifications notif = request(:get, "/api/alert-notifications").find{ |s| s["name"] == "slack" } if notif method = :put url = "/api/alert-notifications/#{notif["id"]}" opts = {id: notif["id"]} else method = :post url = "/api/alert-notifications" opts = {} end puts "create/update slack notifications" request(method, url, opts.merge(type: "slack", name: "slack", isDefault: true, settings: { url: config["slack_url"], recipient: config["slack_channel"], mention: config["slack_mention"] })) end def gen_panel_custom(id, datasource_id, desc) queries = ("A".."Z").to_a i = -1 exprs = desc.key?("exprs") ? desc["exprs"] : [{"expr" => desc["expr"]}] { datasource: "prometheus", id: id, span: desc["span"] || 2, stack: desc["stack"], targets: exprs.map { |e| q = queries[i += 1] { expr: e["expr"].gsub("$target", desc["target"] || "missed_target"), refId: q, intervalFactor: 2, step: 4, legendFormat: e.key?("title") ? e["title"] : nil } }, linewidth: desc["linewidth"] || 2, fill: desc["fill"] || 0, title: desc["title"] || "title", type: "graph", yaxes: [{format: desc["format"] || "short", show: true, min: desc["min"] || 0, max: desc["max"]}, {format: "short", show: false}], legend: {show: (desc["legend"].nil? ? false : desc["legend"])}, aliasColors: exprs.select { |e| e["title"] && e["color"] }.map { |e| [e["title"], e["color"]] }.to_h, }.merge(desc.key?("alert") ? {alert: desc["alert"], thresholds: desc["thresholds"]} : {}) end def gen_panel(id, datasource_id, desc) if desc["type"] == "memory" title = desc.delete("title") gen_panel_custom(id, datasource_id, desc.merge(JSON.load(JSON.dump({ title: desc["target"] + " - mem", stack: true, fill: 8, format: "decbytes", exprs: [ {expr: 'node_memory_MemTotal{instance="$target"} - node_memory_Cached{instance="$target"} - node_memory_Buffers{instance="$target"} - node_memory_MemFree{instance="$target"}', title: "Used", color: "#890F02"}, {expr: 'node_memory_Cached{instance="$target"}', title: "Cached", color: "#0A437C"}, {expr: 'node_memory_Buffers{instance="$target"}', title: "Buffers", color: "#1F78C1"}, {expr: 'node_memory_MemFree{instance="$target"}', title: "Free", color: "#BADFF4"} ] })))) elsif desc["type"] == "disk" raise "missed 'disk' field for disk panel" unless desc["disk"] gen_panel_custom(id, datasource_id, desc.merge(JSON.load(JSON.dump({ title: desc["target"] + " - #{desc["disk"]}", fill: 0, format: "ops", exprs: [ {expr: 'rate(node_disk_writes_completed{device="'+desc["disk"]+'",instance="$target"}[1m])', title: "Write", color: "#9F1B00"}, {expr: 'rate(node_disk_reads_completed{device="'+desc["disk"]+'",instance="$target"}[1m])', title: "Read", color: "#7EB26D"}, ] })))) elsif desc["type"] == "cpu" gen_panel_custom(id, datasource_id, desc.merge(JSON.load(JSON.dump({ title: desc["target"] + " - cpu", fill: 0, max: 100, exprs: [ {expr: '100 - (avg by (instance)(rate(node_cpu{instance="$target",mode="idle"}[1m]))*100)', title: "Usage", color: "#967302"}, ] })))) elsif desc["type"] == "network" raise "missed 'interface' field for network panel" unless desc["interface"] gen_panel_custom(id, datasource_id, desc.merge(JSON.load(JSON.dump({ title: desc["target"] + " - #{desc["interface"]}", fill: 8, format: "Bps", stack: true, exprs: [ {expr: 'rate(node_network_receive_bytes{device="'+desc["interface"]+'",instance="$target"}[1m])', title: "In", color: "#0A437C"}, {expr: 'rate(node_network_transmit_bytes{device="'+desc["interface"]+'",instance="$target"}[1m])', title: "Out", color: "#629E51"}, ] })))) elsif desc["type"] == "custom" gen_panel_custom(id, datasource_id, desc) elsif desc["type"] == "alert" if desc["expr"].index(">") expr = desc["expr"].split(">", 2)[0].gsub("$target", desc["target"] || "missed_target") limit = desc["expr"].split(">", 2)[1].to_f operator = "gt" elsif desc["expr"].index("<") expr = desc["expr"].split("<", 2)[0].gsub("$target", desc["target"] || "missed_target") limit = desc["expr"].split("<", 2)[1].to_f operator = "lt" else raise "wrong expr" end gen_panel_custom(id, datasource_id, desc.merge(JSON.load(JSON.dump({ exprs: [{expr: expr, title: nil}], fill: 0, thresholds: [{value: limit, op: operator, fill: true, line: true, colorMode: "critical"}], legend: false, alert: { conditions: [ { evaluator: {params: [limit], type: operator}, operator: {type: "and"}, query: { datasourceId: datasource_id, model: {expr: expr, refId: "A", intervalFactor: 2, step: 4 }, params: ["A", "1m", "now"] }, reducer: {params: [], type: "avg"}, type: "query" } ], executionErrorState: "alerting", frequency: "30s", name: desc["expr"] } })))) else raise "NIY. unknown panel type `#{desc["type"]}`" end end def create_or_update_dashboard(config) editable = config.delete("editable") { true } name = config.delete("name") puts "create/update dashboard '#{name}'" rows = config.map { |name, group| group = {"panels" => group} if group.is_a?(Array) { title: group["title"] || name, showTitle: group.delete("showTitle") { true }, height: group["height"] || 200, panels: group["panels"].map { |panel| gen_panel((@panel_id += 1), @datasource_id, panel) } } } hr = request(:post, "/api/dashboards/db", { dashboard: { title: name, tags: [TAG], editable: editable, time: {from: "now-30m", to: "now"}, refresh: "10s", rows: rows }, overwrite: true }) die(hr.inspect) unless hr["slug"] end def remove_obsole_dashboards names = dashboards.map { |config| config["name"] } request(:get, "/api/search/?tag=#{TAG}").reject { |s| names.index(s["title"]) }.each do |s| puts "remove dashboard '#{s["title"]}'" request(:delete, "/api/dashboards/#{s["uri"]}") end end def deep_clone(hash) JSON.load(JSON.dump(hash)) end def apply check_config_existance die("missed grafana_url variable") unless config["grafana_url"] die("missed prometheus_url variable") unless config["prometheus_url"] create_or_update_prometheus_datasource create_or_update_slack_notifications if config["slack_url"] && config["slack_channel"] dashboards.each do |dashboard| create_or_update_dashboard(deep_clone(dashboard)) end remove_obsole_dashboards puts "done." end def usage puts "Grafana-rb: utility to configure grafana dashboards as a code" puts "Version: #{VERSION}" puts "" puts "Usage:" puts " grafana-rb " puts "" puts "Commands:" puts " run - update dashboards" puts "" end end end