bin/hva in wakame-vdc-agents-10.11.0 vs bin/hva in wakame-vdc-agents-10.12.0

- old
+ new

@@ -3,17 +3,18 @@ begin require 'rubygems' require 'bundler' Bundler.setup(:default) -rescue Exception +rescue Exception end require File.expand_path('../../config/path_resolver', __FILE__) include Isono::Runner::RpcServer require 'fileutils' +require 'ipaddress' class ServiceNetfilter < Isono::NodeModules::Base include Dcmgr::Logger initialize_hook do @@ -27,108 +28,166 @@ event.subscribe('hva/instance_started', '#') do |args| @worker_thread.pass { logger.info("refresh on instance_started: #{args.inspect}") inst_id = args[0] - logger.info("refresh_netfilter_by_friend_instance_id: #{inst_id}") - myinstance.refresh_netfilter_by_friend_instance_id(inst_id) + #logger.info("refresh_netfilter_by_friend_instance_id: #{inst_id}") + #myinstance.refresh_netfilter_by_friend_instance_id(inst_id, 'started') + myinstance.init_netfilter } end event.subscribe('hva/instance_terminated', '#') do |args| @worker_thread.pass { logger.info("refresh on instance_terminated: #{args.inspect}") inst_id = args[0] - logger.info("refresh_netfilter_by_friend_instance_id: #{inst_id}") - myinstance.refresh_netfilter_by_friend_instance_id(inst_id) + #logger.info("refresh_netfilter_by_friend_instance_id: #{inst_id}") + #myinstance.refresh_netfilter_by_friend_instance_id(inst_id, 'terminated') + myinstance.init_netfilter } end event.subscribe('hva/netfilter_updated', '#') do |args| @worker_thread.pass { logger.info("refresh on netfilter_updated: #{args.inspect}") netfilter_group_id = args[0] - myinstance.refresh_netfilter_by_joined_netfilter_group_id(netfilter_group_id) + #myinstance.refresh_netfilter_by_joined_netfilter_group_id(netfilter_group_id) + myinstance.init_netfilter } end end def init_netfilter begin inst_maps = rpc.request('hva-collector', 'get_alive_instances', node.node_id) - init_ebtables(inst_maps) if @node.manifest.config.enable_ebtables + viftable_map = {} + inst_maps = inst_maps.map { |inst_map| + viftable_map[ inst_map[:ips].first ] = inst_map[:instance_nics].first[:uuid] + + # Does the hva have instance? + unless inst_map[:host_pool][:node_id] == node.node_id + logger.warn("no match for the instance: #{inst_map[:uuid]}") + next + end + # Does host have vif? + next unless valid_vif?(inst_map[:instance_nics].first[:uuid]) + inst_maps + }.flatten.uniq.compact + init_iptables(inst_maps) if @node.manifest.config.enable_iptables + init_ebtables(inst_maps, viftable_map) if @node.manifest.config.enable_ebtables logger.info("initialize netfilter") rescue Exception => e p e end end # from event_subscriber - def refresh_netfilter_by_friend_instance_id(inst_id) - raise "UnknownInstanceID" if inst_id.nil? +# def refresh_netfilter_by_friend_instance_id(inst_id, state = 'started') +# raise "UnknownInstanceID" if inst_id.nil? +# +# begin +# inst_map = rpc.request('hva-collector', 'get_instance', inst_id) +# ng = rpc.request('hva-collector', 'get_netfilter_groups_of_instance', inst_map[:uuid]) +# +# inst_maps = ng.map { |g| +# rpc.request('hva-collector', 'get_instances_of_netfilter_group', g[:id]) +# } +# +# # my instance_id +# # when terminated? +# if state == 'terminated' +# unless inst_map.nil? +# refresh_iptables(inst_map) if @node.manifest.config.enable_iptables +# refresh_ebtables(inst_map) if @node.manifest.config.enable_ebtables +# end +# end +# +# # friend instance(s) +# if inst_maps.size > 0 +# inst_maps.flatten.uniq.each { |inst_map| +# unless inst_map.nil? +# refresh_iptables(inst_map) if @node.manifest.config.enable_iptables +# refresh_ebtables(inst_map) if @node.manifest.config.enable_ebtables +# end +# } +# end +# rescue Exception => e +# p e +# end +# end - begin - inst_map = rpc.request('hva-collector', 'get_instance', inst_id) - ng = rpc.request('hva-collector', 'get_netfilter_groups_of_instance', inst_map[:uuid]) + # from event_subscriber +# def refresh_netfilter_by_joined_netfilter_group_id(netfilter_group_id) +# raise "UnknownNetfilterGroupID" if netfilter_group_id.nil? +# +# begin +# inst_maps = rpc.request('hva-collector', 'get_instances_of_netfilter_group', netfilter_group_id) +# inst_maps.each { |inst_map| +# unless inst_map.nil? +# refresh_iptables(inst_map) if @node.manifest.config.enable_iptables +# refresh_ebtables(inst_map) if @node.manifest.config.enable_ebtables +# end +# } +# rescue Exception => e +# p e +# end +# end - inst_maps = ng.map { |g| - rpc.request('hva-collector', 'get_instances_of_netfilter_group', g[:id]) + def init_ebtables(inst_maps = [], viftable_map = {}) + cmd = "ebtables --init-table" + puts cmd + system(cmd) + + basic_cmds = [] + group_cmds = [] + final_cmds = [] + + inst_maps.each { |inst_map| + vif_map = { + :uuid => inst_map[:instance_nics].first[:uuid], + :mac => inst_map[:instance_nics].first[:mac_addr].unpack('A2'*6).join(':'), + :ipv4 => inst_map[:ips].first, } - if inst_maps.size > 0 - inst_maps.flatten.uniq.each { |inst_map| - unless inst_map.nil? - refresh_ebtables(inst_map) if @node.manifest.config.enable_ebtables - refresh_iptables(inst_map) if @node.manifest.config.enable_iptables - end - } - end - rescue Exception => e - p e - end - end + basic_cmds << build_ebtables_basic_part(vif_map, inst_map) + group_cmds << build_ebtables_group_part(vif_map, inst_map, viftable_map) + final_cmds << build_ebtables_final_part(vif_map) + } - # from event_subscriber - def refresh_netfilter_by_joined_netfilter_group_id(netfilter_group_id) - raise "UnknownNetfilterGroupID" if netfilter_group_id.nil? + viftable_map.each { |k,v| + p "#{v} <-> #{k}" + } - begin - inst_maps = rpc.request('hva-collector', 'get_instances_of_netfilter_group', netfilter_group_id) - inst_maps.each { |inst_map| - unless inst_map.nil? - refresh_ebtables(inst_map) if @node.manifest.config.enable_ebtables - refresh_iptables(inst_map) if @node.manifest.config.enable_iptables - end - } - rescue Exception => e - p e - end - end + logger.debug("basic_cmds ...") + basic_cmds.flatten.uniq.each { |cmd| + system(cmd) + } - def init_ebtables(inst_maps = []) - cmd = "sudo ebtables --init-table" - puts cmd - system(cmd) + logger.debug("group_cmds ...") + group_cmds.flatten.uniq.each { |cmd| + system(cmd) + } - inst_maps.each { |inst_map| - refresh_ebtables(inst_map) + logger.debug("final_cmds ...") + final_cmds.flatten.uniq.each { |cmd| + system(cmd) } end def init_iptables(inst_maps = []) [ 'nat', 'filter' ].each { |table| [ 'F', 'Z', 'X' ].each { |xcmd| - cmd = "sudo iptables -t #{table} -#{xcmd}" + cmd = "iptables -t #{table} -#{xcmd}" puts cmd system(cmd) } } inst_maps.each { |inst_map| - refresh_iptables(inst_map) + refresh_iptables(inst_map, false) } end def valid_vif?(vif) cmd = "ifconfig #{vif} >/dev/null 2>&1" @@ -140,140 +199,249 @@ logger.warn("#{vif}: error fetching interface information: Device not found") false end end - def refresh_ebtables(inst_map = {}) - logger.debug("refresh_ebtables: #{inst_map[:uuid]} ...") +# def refresh_ebtables(inst_map = {}, viftable_map = {}) +# logger.debug("refresh_ebtables: #{inst_map[:uuid]} ...") +# +# vif_map = { +# :uuid => inst_map[:instance_nics].first[:uuid], +# :mac => inst_map[:instance_nics].first[:mac_addr].unpack('A2'*6).join(':'), +# } +# +# # xtables commands +# basic_cmds = build_ebtables_basic_part(vif_map, inst_map) +# group_cmds = build_ebtables_group_part(vif_map, inst_map, viftable_map) +# final_cmds = build_ebtables_final_part(vif_map) +# +# logger.debug("refresh_ebtables: #{inst_map[:uuid]} done.") +# end - # Does the hva have instance? - unless inst_map[:host_pool][:node_id] == node.node_id - logger.warn("no match for the instance: #{inst_map[:uuid]}") - return - end + def build_ebtables_basic_part(vif_map, inst_map) + basic_cmds = [] + hva_ipv4 = Isono::Util.default_gw_ipaddr - network_map = rpc.request('hva-collector', 'get_network', inst_map[:host_pool][:network_id]) - raise "UnknownNetworkId" if network_map.nil? + ################################ + ## 0. chain name + ################################ - vif = inst_map[:instance_nics].first[:vif] - vif_mac = inst_map[:instance_nics].first[:mac_addr].unpack('A2'*6).join(':') - - flush_ebtables(inst_map) - - # Does host have vif? - unless valid_vif?(vif) - return - end - - # group node IPv4 addresses. - ipv4s = rpc.request('hva-collector', 'get_group_instance_ipv4s', inst_map[:uuid]) - - # xtables commands - cmds = [] - # support IP protocol protocol_maps = { 'ip4' => 'ip4', 'arp' => 'arp', - #ip6' => 'ip6', - #rarp' => '0x8035', + #'ip6' => 'ip6', + #'rarp' => '0x8035', } # make chain names. chains = [] - chains << "s_#{vif}" - chains << "d_#{vif}" - chains << "s_#{vif}_d_host" + chains << "s_#{vif_map[:uuid]}" + chains << "d_#{vif_map[:uuid]}" + chains << "s_#{vif_map[:uuid]}_d_hst" + chains << "d_#{vif_map[:uuid]}_s_hst" protocol_maps.each { |k,v| - chains << "s_#{vif}_#{k}" - chains << "d_#{vif}_#{k}" - chains << "s_#{vif}_d_host_#{k}" + chains << "s_#{vif_map[:uuid]}_#{k}" + chains << "d_#{vif_map[:uuid]}_#{k}" + chains << "s_#{vif_map[:uuid]}_d_hst_#{k}" + chains << "d_#{vif_map[:uuid]}_s_hst_#{k}" } + ################################ + ## 1. basic part + ################################ + # create user defined chains. [ 'N' ].each { |xcmd| chains.each { |chain| - cmds << "sudo ebtables -#{xcmd} #{chain}" + basic_cmds << "ebtables -#{xcmd} #{chain}" } } # jumt to user defined chains - cmds << "sudo ebtables -A FORWARD -i #{vif} -j s_#{vif}" - cmds << "sudo ebtables -A FORWARD -o #{vif} -j d_#{vif}" - cmds << "sudo ebtables -A INPUT -i #{vif} -j s_#{vif}_d_host" + basic_cmds << "ebtables -A FORWARD -i #{vif_map[:uuid]} -j s_#{vif_map[:uuid]}" + basic_cmds << "ebtables -A FORWARD -o #{vif_map[:uuid]} -j d_#{vif_map[:uuid]}" + basic_cmds << "ebtables -A INPUT -i #{vif_map[:uuid]} -j s_#{vif_map[:uuid]}_d_hst" + basic_cmds << "ebtables -A OUTPUT -o #{vif_map[:uuid]} -j d_#{vif_map[:uuid]}_s_hst" # IP protocol routing protocol_maps.each { |k,v| - cmds << "sudo ebtables -A s_#{vif} -p #{v} -j s_#{vif}_#{k}" - cmds << "sudo ebtables -A d_#{vif} -p #{v} -j d_#{vif}_#{k}" - cmds << "sudo ebtables -A s_#{vif}_d_host -p #{v} -j s_#{vif}_d_host_#{k}" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]} -p #{v} -j s_#{vif_map[:uuid]}_#{k}" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]} -p #{v} -j d_#{vif_map[:uuid]}_#{k}" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst -p #{v} -j s_#{vif_map[:uuid]}_d_hst_#{k}" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst -p #{v} -j d_#{vif_map[:uuid]}_s_hst_#{k}" } # default drop - cmds << "sudo ebtables -A s_#{vif} --log-level warning --log-ip --log-arp --log-prefix 's_#{vif} DROP:' -j CONTINUE" - cmds << "sudo ebtables -A s_#{vif}_d_host --log-level warning --log-ip --log-arp --log-prefix 's_#{vif}_d_host DROP:' -j CONTINUE" - cmds << "sudo ebtables -A s_#{vif} -j DROP" - cmds << "sudo ebtables -A s_#{vif}_d_host -j DROP" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]} --log-level warning --log-ip --log-arp --log-prefix 'D s_#{vif_map[:uuid]}:' -j CONTINUE" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst --log-level warning --log-ip --log-arp --log-prefix 'D s_#{vif_map[:uuid]}_d_hst:' -j CONTINUE" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]} -j DROP" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst -j DROP" - # anti spoof - #cmds << "sudo ebtables -A s_#{vif}_arp --protocol arp --arp-mac-src ! #{vif_mac} -j DROP" - #cmds << "sudo ebtables -A d_#{vif}_arp --protocol arp --arp-mac-dst ! #{vif_mac} -j DROP" + # anti spoof: mac + # guest -> * + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_arp --protocol arp --arp-mac-src ! #{vif_map[:mac]} --log-ip --log-arp --log-prefix 'Dmc s_#{vif_map[:uuid]}_arp:' -j CONTINUE" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-mac-src ! #{vif_map[:mac]} --log-ip --log-arp --log-prefix 'Dmc s_#{vif_map[:uuid]}_d_hst_arp:' -j CONTINUE" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_arp --protocol arp --arp-mac-src ! #{vif_map[:mac]} -j DROP" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-mac-src ! #{vif_map[:mac]} -j DROP" - # group nodes. - ipv4s << network_map[:ipv4_gw] - ipv4s << network_map[:dns_server] - ipv4s << network_map[:dhcp_server] - ipv4s.uniq.each do |ipv4| - cmds << "sudo ebtables -A d_#{vif}_arp --protocol arp --arp-ip-src #{ipv4} -j ACCEPT" - end + # guest <- * (broadcast) + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-mac-dst 00:00:00:00:00:00 --log-ip --log-arp --log-prefix 'Amc d_#{vif_map[:uuid]}_arp:' -j CONTINUE" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst_arp --protocol arp --arp-ip-src=#{hva_ipv4} --arp-mac-dst 00:00:00:00:00:00 --log-ip --log-arp --log-prefix 'Amc d_#{vif_map[:uuid]}_hst_arp:' -j CONTINUE" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-mac-dst 00:00:00:00:00:00 -j ACCEPT" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst_arp --protocol arp --arp-ip-src=#{hva_ipv4} --arp-mac-dst 00:00:00:00:00:00 -j ACCEPT" - # deny,allow - cmds << "sudo ebtables -A d_#{vif}_arp --log-level warning --log-ip --log-arp --log-prefix 's_#{vif}_arp DROP:' -j CONTINUE" - cmds << "sudo ebtables -A s_#{vif}_d_host_arp --log-level warning --log-ip --log-arp --log-prefix 's_#{vif}_d_host_arp DROP:' -j CONTINUE" - cmds << "sudo ebtables -A d_#{vif}_arp -j DROP" - cmds << "sudo ebtables -A s_#{vif}_d_host_arp -j DROP" + # guest <- * + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-mac-dst ! #{vif_map[:mac]} --log-ip --log-arp --log-prefix 'Dmc d_#{vif_map[:uuid]}_arp:' -j CONTINUE" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst_arp --protocol arp --arp-mac-dst ! #{vif_map[:mac]} --log-ip --log-arp --log-prefix 'Dmc d_#{vif_map[:uuid]}_s_hst_arp:' -j CONTINUE" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-mac-dst ! #{vif_map[:mac]} -j DROP" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst_arp --protocol arp --arp-mac-dst ! #{vif_map[:mac]} -j DROP" - cmds.uniq! if cmds.size > 0 - cmds.compact.each { |cmd| - puts cmd - system(cmd) + # anti spoof: ipv4 + inst_map[:ips].each { |ipv4| + # guest -> * + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-src ! #{ipv4} --log-ip --log-arp --log-prefix 'Dip s_#{vif_map[:uuid]}_arp:' -j CONTINUE" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-ip-src ! #{ipv4} --log-ip --log-arp --log-prefix 'Dip s_#{vif_map[:uuid]}_d_hst_arp:' -j CONTINUE" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-src ! #{ipv4} -j DROP" + basic_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-ip-src ! #{ipv4} -j DROP" + # guest <- * + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-dst ! #{ipv4} --log-ip --log-arp --log-prefix 'Dip d_#{vif_map[:uuid]}_arp:' -j CONTINUE" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst_arp --protocol arp --arp-ip-dst ! #{ipv4} --log-ip --log-arp --log-prefix 'Dip d_#{vif_map[:uuid]}_s_hst_arp:' -j CONTINUE" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-dst ! #{ipv4} -j DROP" + basic_cmds << "ebtables -A d_#{vif_map[:uuid]}_s_hst_arp --protocol arp --arp-ip-dst ! #{ipv4} -j DROP" } - logger.debug("refresh_ebtables: #{inst_map[:uuid]} done.") + basic_cmds end - def refresh_iptables(inst_map = {}) + + def build_ebtables_group_part(vif_map, inst_map, viftable_map) + group_cmds = [] + hva_ipv4 = Isono::Util.default_gw_ipaddr + + ################################ + ## 2. group part + ################################ + same_subnet_ipv4s = rpc.request('hva-collector', 'get_group_instance_ipv4s', inst_map[:uuid]) + + # detect node joined network(s). + network_map = rpc.request('hva-collector', 'get_network', inst_map[:instance_nics].first[:network_id]) + raise "UnknownNetworkId" if network_map.nil? + joined_network = IPAddress("#{network_map[:ipv4_gw]}/#{network_map[:prefix]}") + [ network_map[:dns_server], network_map[:dhcp_server] ].each { |ipv4| + next unless joined_network.include? IPAddress(ipv4) + same_subnet_ipv4s << ipv4 + } + + # network resource node(s) + ng_maps = rpc.request('hva-collector', 'get_netfilter_groups_of_instance', inst_map[:uuid]) + rules = ng_maps.map { |ng_map| + ng_map[:rules].map { |rule| rule[:permission] } + }.flatten + build_rule(rules).each do |rule| + next unless joined_network.include? IPAddress(rule[:ip_source]) + same_subnet_ipv4s << rule[:ip_source] + end + same_subnet_ipv4s << network_map[:ipv4_gw] + + # guest node(s) in HyperVisor. + alive_inst_maps = rpc.request('hva-collector', 'get_alive_instances', node.node_id) + guest_ipv4s = alive_inst_maps.map { |alive_inst_map| + alive_inst_map[:ips] + }.flatten.uniq.compact + + same_subnet_ipv4s.uniq.reverse_each do |ipv4| + next if vif_map[:ipv4] == ipv4 + + # get_macaddr_by_ipv4, ipv4 + if ipv4 == hva_ipv4 + #p "#{vif_map[:uuid]}(#{vif_map[:ipv4]}) -> [host] ***-****** (#{ipv4})" + group_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-ip-src #{vif_map[:ipv4]} --arp-ip-dst #{ipv4} --log-ip --log-arp --log-prefix 'Afw s_#{vif_map[:uuid]}_d_hst_arp:' -j CONTINUE" + group_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-ip-src #{vif_map[:ipv4]} --arp-ip-dst #{ipv4} -j ACCEPT" + elsif guest_ipv4s.include?(ipv4) + #p "#{vif_map[:uuid]}(#{vif_map[:ipv4]}) -> [guest] #{viftable_map[ipv4]}(#{ipv4})" + + # guest->guest + group_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-src #{ipv4} --arp-ip-dst #{vif_map[:ipv4]} --log-ip --log-arp --log-prefix 'Afw d_#{vif_map[:uuid]}_arp:' -j CONTINUE" + group_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-src #{ipv4} --arp-ip-dst #{vif_map[:ipv4]} -j ACCEPT" + # guest->host + group_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-ip-src #{vif_map[:ipv4]} --arp-ip-dst #{ipv4} --log-ip --log-arp --log-prefix 'Afw s_#{vif_map[:uuid]}_d_hst_arp:' -j CONTINUE" + group_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --protocol arp --arp-ip-src #{vif_map[:ipv4]} --arp-ip-dst #{ipv4} -j ACCEPT" + + unless viftable_map[ipv4].nil? + # guest->guest + group_cmds << "ebtables -A d_#{viftable_map[ipv4]}_arp --protocol arp --arp-ip-src #{vif_map[:ipv4]} --arp-ip-dst #{ipv4} --log-ip --log-arp --log-prefix 'Arv d_#{viftable_map[ipv4]}_arp:' -j CONTINUE" + group_cmds << "ebtables -A d_#{viftable_map[ipv4]}_arp --protocol arp --arp-ip-src #{vif_map[:ipv4]} --arp-ip-dst #{ipv4} -j ACCEPT" + + # guest->host + group_cmds << "ebtables -A s_#{viftable_map[ipv4]}_d_hst_arp --protocol arp --arp-ip-src #{ipv4} --arp-ip-dst #{vif_map[:ipv4]} --log-ip --log-arp --log-prefix 'Arv s_#{viftable_map[ipv4]}_d_hst_arp:' -j CONTINUE" + group_cmds << "ebtables -A s_#{viftable_map[ipv4]}_d_hst_arp --protocol arp --arp-ip-src #{ipv4} --arp-ip-dst #{vif_map[:ipv4]} -j ACCEPT" + end + else + #p "#{vif_map[:uuid]}(#{vif_map[:ipv4]}) -> [other] ***-******** (#{ipv4})" + group_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-src #{ipv4} --arp-ip-dst #{vif_map[:ipv4]} --log-ip --log-arp --log-prefix 'Afw :d_#{vif_map[:uuid]}_arp' -j CONTINUE" + group_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --protocol arp --arp-ip-src #{ipv4} --arp-ip-dst #{vif_map[:ipv4]} -j ACCEPT" + end + end + + group_cmds + end + + + def build_ebtables_final_part(vif_map) + final_cmds = [] + + ################################ + ## 3. final part + ################################ + # deny,allow + final_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp --log-level warning --log-ip --log-arp --log-prefix 'D d_#{vif_map[:uuid]}_arp:' -j CONTINUE" + final_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp --log-level warning --log-ip --log-arp --log-prefix 'D s_#{vif_map[:uuid]}_d_hst_arp:' -j CONTINUE" + final_cmds << "ebtables -A d_#{vif_map[:uuid]}_arp -j DROP" + final_cmds << "ebtables -A s_#{vif_map[:uuid]}_d_hst_arp -j DROP" + + final_cmds + end + + def refresh_iptables(inst_map = {}, with_flush = 1) logger.debug("refresh_iptables: #{inst_map[:uuid]} ...") # Does the hva have instance? unless inst_map[:host_pool][:node_id] == node.node_id logger.warn "no match for the instance: #{inst_map[:uuid]}" return end - network_map = rpc.request('hva-collector', 'get_network', inst_map[:host_pool][:network_id]) + network_map = rpc.request('hva-collector', 'get_network', inst_map[:instance_nics].first[:network_id]) raise "UnknownNetworkId" if network_map.nil? - vif = inst_map[:instance_nics].first[:vif] + vif = inst_map[:instance_nics].first[:uuid] vif_mac = inst_map[:instance_nics].first[:mac_addr].unpack('A2'*6).join(':') - flush_iptables(inst_map) + if with_flush + flush_iptables(inst_map) + end # Does host have vif? unless valid_vif?(vif) return end + + + # group node IPv4 addresses. ipv4s = rpc.request('hva-collector', 'get_group_instance_ipv4s', inst_map[:uuid]) - ng = rpc.request('hva-collector', 'get_netfilter_groups_of_instance', inst_map[:uuid]) - rules = ng.map { |g| - g[:rules].map { |rule| rule[:permission] } - } - rules.flatten! if rules.size > 0 + ng_maps = rpc.request('hva-collector', 'get_netfilter_groups_of_instance', inst_map[:uuid]) + rules = ng_maps.map { |ng_map| + ng_map[:rules].map { |rule| rule[:permission] } + }.flatten + + + # xtables commands cmds = [] # support IP protocol protocol_maps = { @@ -291,120 +459,126 @@ chains << "s_#{vif}" chains << "d_#{vif}" # metadata-server [ 'A' ].each { |xcmd| - system("sudo iptables -t nat -#{xcmd} PREROUTING -m physdev --physdev-is-bridged --physdev-in #{vif} -s 0.0.0.0 -d 169.254.169.254 -p tcp --dport 80 -j DNAT --to-destination #{network_map[:metadata_server]}:80") + system("iptables -t nat -#{xcmd} PREROUTING -m physdev --physdev-is-bridged --physdev-in #{vif} -s 0.0.0.0 -d 169.254.169.254 -p tcp --dport 80 -j DNAT --to-destination #{network_map[:metadata_server]}:80") } # create user defined chains. [ 'N' ].each { |xcmd| chains.each { |chain| - cmds << "sudo iptables -#{xcmd} #{chain}" + cmds << "iptables -#{xcmd} #{chain}" # logger & drop - cmds << "sudo iptables -N #{chain}_drop" - cmds << "sudo iptables -A #{chain}_drop -j LOG --log-level 4 --log-prefix '#{chain} DROP:'" - cmds << "sudo iptables -A #{chain}_drop -j DROP" + cmds << "iptables -N #{chain}_drop" + cmds << "iptables -A #{chain}_drop -j LOG --log-level 4 --log-prefix 'D #{chain}:'" + cmds << "iptables -A #{chain}_drop -j DROP" } } # group nodes ipv4s << network_map[:ipv4_gw] - ipv4s.each { |addr| - cmds << "sudo iptables -A d_#{vif} -s #{addr} -j ACCEPT" + ipv4s.uniq.reverse_each { |addr| + cmds << "iptables -A d_#{vif} -s #{addr} -j ACCEPT" } # IP protocol routing [ 's', 'd' ].each do |bound| protocol_maps.each { |k,v| - cmds << "sudo iptables -N #{bound}_#{vif}_#{k}" + cmds << "iptables -N #{bound}_#{vif}_#{k}" case k when 'tcp' case bound when 's' - cmds << "sudo iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" + cmds << "iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" when 'd' - #cmds << "sudo iptables -A #{bound}_#{vif} -m state --state ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" - cmds << "sudo iptables -A #{bound}_#{vif} -m state --state RELATED,ESTABLISHED -p #{k} -j ACCEPT" - cmds << "sudo iptables -A #{bound}_#{vif} -p #{k} -j #{bound}_#{vif}_#{k}" + #cmds << "iptables -A #{bound}_#{vif} -m state --state ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" + cmds << "iptables -A #{bound}_#{vif} -m state --state RELATED,ESTABLISHED -p #{k} -j ACCEPT" + cmds << "iptables -A #{bound}_#{vif} -p #{k} -j #{bound}_#{vif}_#{k}" end when 'udp' case bound when 's' - cmds << "sudo iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" + cmds << "iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" when 'd' - #cmds << "sudo iptables -A #{bound}_#{vif} -m state --state ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" - cmds << "sudo iptables -A #{bound}_#{vif} -m state --state ESTABLISHED -p #{k} -j ACCEPT" - cmds << "sudo iptables -A #{bound}_#{vif} -p #{k} -j #{bound}_#{vif}_#{k}" + #cmds << "iptables -A #{bound}_#{vif} -m state --state ESTABLISHED -p #{k} -j #{bound}_#{vif}_#{k}" + cmds << "iptables -A #{bound}_#{vif} -m state --state ESTABLISHED -p #{k} -j ACCEPT" + cmds << "iptables -A #{bound}_#{vif} -p #{k} -j #{bound}_#{vif}_#{k}" end when 'icmp' case bound when 's' - cmds << "sudo iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED,RELATED -p #{k} -j #{bound}_#{vif}_#{k}" + cmds << "iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED,RELATED -p #{k} -j #{bound}_#{vif}_#{k}" when 'd' - #cmds << "sudo iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED,RELATED -p #{k} -j #{bound}_#{vif}_#{k}" - cmds << "sudo iptables -A #{bound}_#{vif} -m state --state ESTABLISHED,RELATED -p #{k} -j ACCEPT" - cmds << "sudo iptables -A #{bound}_#{vif} -p #{k} -j #{bound}_#{vif}_#{k}" + #cmds << "iptables -A #{bound}_#{vif} -m state --state NEW,ESTABLISHED,RELATED -p #{k} -j #{bound}_#{vif}_#{k}" + cmds << "iptables -A #{bound}_#{vif} -m state --state ESTABLISHED,RELATED -p #{k} -j ACCEPT" + cmds << "iptables -A #{bound}_#{vif} -p #{k} -j #{bound}_#{vif}_#{k}" end end } end - cmds << "sudo iptables -A FORWARD -m physdev --physdev-is-bridged --physdev-in #{vif} -j s_#{vif}" - cmds << "sudo iptables -A FORWARD -m physdev --physdev-is-bridged --physdev-out #{vif} -j d_#{vif}" + cmds << "iptables -A FORWARD -m physdev --physdev-is-bridged --physdev-in #{vif} -j s_#{vif}" + cmds << "iptables -A FORWARD -m physdev --physdev-is-bridged --physdev-out #{vif} -j d_#{vif}" ## ## ACCEPT ## # DHCP Server - cmds << "sudo iptables -A d_#{vif}_udp -p udp -s #{network_map[:dhcp_server]} --sport 67 -j ACCEPT" - #cmds << "sudo iptables -A d_#{vif}_udp -p udp --sport 67 -j d_#{vif}_udp_drop" + cmds << "iptables -A d_#{vif}_udp -p udp -s #{network_map[:dhcp_server]} --sport 67 -j ACCEPT" + cmds << "iptables -A d_#{vif}_udp -p udp -s #{network_map[:dhcp_server]} --sport 68 -j ACCEPT" + + #cmds << "iptables -A d_#{vif}_udp -p udp --sport 67 -j d_#{vif}_udp_drop" # DNS Server - cmds << "sudo iptables -A s_#{vif}_udp -p udp -d #{network_map[:dns_server]} --dport 53 -j ACCEPT" + cmds << "iptables -A s_#{vif}_udp -p udp -d #{network_map[:dns_server]} --dport 53 -j ACCEPT" ## ## DROP ## protocol_maps.each { |k,v| # DHCP - cmds << "sudo iptables -A s_#{vif} -d #{network_map[:dhcp_server]} -p #{k} -j s_#{vif}_#{k}_drop" + cmds << "iptables -A s_#{vif} -d #{network_map[:dhcp_server]} -p #{k} -j s_#{vif}_#{k}_drop" # DNS - cmds << "sudo iptables -A s_#{vif} -d #{network_map[:dns_server]} -p #{k} -j s_#{vif}_#{k}_drop" + cmds << "iptables -A s_#{vif} -d #{network_map[:dns_server]} -p #{k} -j s_#{vif}_#{k}_drop" } # security group - # rules build_rule(rules).each do |rule| case rule[:ip_protocol] when 'tcp', 'udp' - cmds << "sudo iptables -A d_#{vif}_#{rule[:ip_protocol]} -p #{rule[:ip_protocol]} -s #{rule[:ip_source]} --dport #{rule[:ip_dport]} -j ACCEPT" + cmds << "iptables -A d_#{vif}_#{rule[:ip_protocol]} -p #{rule[:ip_protocol]} -s #{rule[:ip_source]} --dport #{rule[:ip_dport]} -j ACCEPT" when 'icmp' - # ToDo: implement - # - icmp_type : -1... - # - icmp_code : -1... - # cmds << "sudo iptables -A d_#{vif}_#{rule[:ip_protocol]} -p #{rule[:ip_protocol]} -s #{rule[:ip_source]} --icmp-type #{rule[:icmp_type]}/#{rule[:icmp_code]} -j ACCEPT" - cmds << "sudo iptables -A d_#{vif}_#{rule[:ip_protocol]} -p #{rule[:ip_protocol]} -s #{rule[:ip_source]} -j ACCEPT" + # icmp + # This extension can be used if `--protocol icmp' is specified. It provides the following option: + # [!] --icmp-type {type[/code]|typename} + # This allows specification of the ICMP type, which can be a numeric ICMP type, type/code pair, or one of the ICMP type names shown by the command + # iptables -p icmp -h + if rule[:icmp_type] == -1 && rule[:icmp_code] == -1 + cmds << "iptables -A d_#{vif}_#{rule[:ip_protocol]} -p #{rule[:ip_protocol]} -s #{rule[:ip_source]} -j ACCEPT" + else + cmds << "iptables -A d_#{vif}_#{rule[:ip_protocol]} -p #{rule[:ip_protocol]} -s #{rule[:ip_source]} --icmp-type #{rule[:icmp_type]}/#{rule[:icmp_code]} -j ACCEPT" + end end end # drop other routings protocol_maps.each { |k,v| - cmds << "sudo iptables -A d_#{vif}_#{k} -p #{k} -j d_#{vif}_#{k}_drop" + cmds << "iptables -A d_#{vif}_#{k} -p #{k} -j d_#{vif}_#{k}_drop" } # IP protocol routing [ 'd' ].each do |bound| protocol_maps.each { |k,v| - cmds << "sudo iptables -A #{bound}_#{vif}_#{k} -j #{bound}_#{vif}_#{k}_drop" + cmds << "iptables -A #{bound}_#{vif}_#{k} -j #{bound}_#{vif}_#{k}_drop" } end cmds.uniq! if cmds.size > 0 cmds.compact.each { |cmd| - puts cmd + #puts cmd system(cmd) } logger.debug("refresh_iptables: #{inst_map[:uuid]} done.") end @@ -425,52 +599,61 @@ # support IP protocol protocol_maps = { 'ip4' => 'ip4', 'arp' => 'arp', - #ip6' => 'ip6', - #rarp' => '0x8035', + #'ip6' => 'ip6', + #'rarp' => '0x8035', } # make chain names. chains = [] chains << "s_#{vif}" chains << "d_#{vif}" - chains << "s_#{vif}_d_host" + chains << "s_#{vif}_d_hst" + chains << "d_#{vif}_s_hst" protocol_maps.each { |k,v| chains << "s_#{vif}_#{k}" chains << "d_#{vif}_#{k}" - chains << "s_#{vif}_d_host_#{k}" + chains << "s_#{vif}_d_hst_#{k}" + chains << "d_#{vif}_s_hst_#{k}" } # clear rules if exists. - system("sudo ebtables -L s_#{vif} >/dev/null 2>&1") + system("ebtables -L s_#{vif} >/dev/null 2>&1") if $?.exitstatus == 0 - cmd = "sudo ebtables -D FORWARD -i #{vif} -j s_#{vif}" + cmd = "ebtables -D FORWARD -i #{vif} -j s_#{vif}" puts cmd system(cmd) end - system("sudo ebtables -L d_#{vif} >/dev/null 2>&1") + system("ebtables -L d_#{vif} >/dev/null 2>&1") if $?.exitstatus == 0 - cmd = "sudo ebtables -D FORWARD -o #{vif} -j d_#{vif}" + cmd = "ebtables -D FORWARD -o #{vif} -j d_#{vif}" puts cmd system(cmd) end - system("sudo ebtables -L s_#{vif}_d_host >/dev/null 2>&1") + system("ebtables -L s_#{vif}_d_hst >/dev/null 2>&1") if $?.exitstatus == 0 - cmd = "sudo ebtables -D INPUT -i #{vif} -j s_#{vif}_d_host" + cmd = "ebtables -D INPUT -i #{vif} -j s_#{vif}_d_hst" puts cmd system(cmd) end + system("ebtables -L d_#{vif}_s_hst >/dev/null 2>&1") + if $?.exitstatus == 0 + cmd = "ebtables -D OUTPUT -o #{vif} -j d_#{vif}_s_hst" + puts cmd + system(cmd) + end + [ 'F', 'Z', 'X' ].each { |xcmd| chains.each { |chain| - system("sudo ebtables -L #{chain} >/dev/null 2>&1") + system("ebtables -L #{chain} >/dev/null 2>&1") if $?.exitstatus == 0 - cmd = "sudo ebtables -#{xcmd} #{chain}" + cmd = "ebtables -#{xcmd} #{chain}" puts cmd system(cmd) end } } @@ -512,108 +695,182 @@ chains << "s_#{vif}_drop" chains << "d_#{vif}_drop" # metadata-server [ 'D' ].each { |xcmd| - system("sudo iptables -t nat -#{xcmd} PREROUTING -m physdev --physdev-is-bridged --physdev-in #{vif} -s 0.0.0.0 -d 169.254.169.254 -p tcp --dport 80 -j DNAT --to-destination #{network_map[:metadata_server]}:80 >/dev/null 2>&1") + system("iptables -t nat -#{xcmd} PREROUTING -m physdev --physdev-is-bridged --physdev-in #{vif} -s 0.0.0.0 -d 169.254.169.254 -p tcp --dport 80 -j DNAT --to-destination #{network_map[:metadata_server]}:80 >/dev/null 2>&1") } # clean rules if exists. - system("sudo iptables -nL s_#{vif} >/dev/null 2>&1") + system("iptables -nL s_#{vif} >/dev/null 2>&1") if $?.exitstatus == 0 - system("sudo iptables -D FORWARD -m physdev --physdev-is-bridged --physdev-in #{vif} -j s_#{vif}") + system("iptables -D FORWARD -m physdev --physdev-is-bridged --physdev-in #{vif} -j s_#{vif}") end - system("sudo iptables -nL d_#{vif} >/dev/null 2>&1") + system("iptables -nL d_#{vif} >/dev/null 2>&1") if $?.exitstatus == 0 - system("sudo iptables -D FORWARD -m physdev --physdev-is-bridged --physdev-out #{vif} -j d_#{vif}") + system("iptables -D FORWARD -m physdev --physdev-is-bridged --physdev-out #{vif} -j d_#{vif}") end [ 'F', 'Z', 'X' ].each { |xcmd| chains.each { |chain| - system("sudo iptables -nL #{chain} >/dev/null 2>&1") + system("iptables -nL #{chain} >/dev/null 2>&1") if $?.exitstatus == 0 - system("sudo iptables -#{xcmd} #{chain}") + system("iptables -#{xcmd} #{chain}") end } } logger.debug("flush_iptables: #{inst_map[:uuid]} #{vif} done.") end def build_rule(rules = []) - require 'ipaddress' - rule_maps = [] rules.each do |rule| + rule = rule.strip.gsub(/[\s\t]+/, '') + from_group = false + ipv4s = [] + # ex. # "tcp:22,22,ip4:0.0.0.0" # "udp:53,53,ip4:0.0.0.0" # "icmp:-1,-1,ip4:0.0.0.0" # 1st phase # ip_dport : tcp,udp? 1 - 16bit, icmp: -1 # id_port has been separeted in first phase. from_pair, ip_dport, source_pair = rule.split(',') + # TODO: more strict validations + next if from_pair.nil? + next if ip_dport.nil? + next if source_pair.nil? + # 2nd phase # ip_protocol : [ tcp | udp | icmp ] # ip_sport : tcp,udp? 1 - 16bit, icmp: -1 ip_protocol, ip_sport = from_pair.split(':') - # protocol : [ ip4 | ip6 ] - # ip_source : ip4? xxx.xxx.xxx.xxx./[0-32], ip6?: not yet supprted. + # protocol : [ ip4 | ip6 | #{account_id} ] + # ip_source : ip4? xxx.xxx.xxx.xxx./[0-32], ip6? (not yet supprted), #{netfilter_group_id} protocol, ip_source = source_pair.split(':') - # validate - next unless protocol == 'ip4' - # next unless IPAddress.valid?(ip_source) - - # IPAddress does't support prefix '0'. - ip_addr, prefix = ip_source.split('/', 2) - if prefix.to_i == 0 - ip_source = ip_addr + begin + s = StringScanner.new(protocol) + until s.eos? + case + when s.scan(/ip6/) + # TODO#FUTURE: support IPv6 address format + next + when s.scan(/ip4/) + # IPAddress does't support prefix '0'. + ip_addr, prefix = ip_source.split('/', 2) + if prefix.to_i == 0 + ip_source = ip_addr + end + when s.scan(/a-\w{8}/) + from_group = true + inst_maps = rpc.request('hva-collector', 'get_instances_of_account_netfilter_group', protocol, ip_source) + inst_maps.each { |inst_map| + ipv4s << inst_map[:ips] + } + else + raise "unexpected protocol '#{s.peep(20)}'" + end + end + rescue Exception => e + p e + next end begin - ip = IPAddress(ip_source) - ip_source = case ip.u32 - when 0 - "#{ip.address}/0" - else - "#{ip.address}/#{ip.prefix}" - end - + if from_group == false + #p "from_group:(#{from_group}) ip_source -> #{ip_source}" + ip = IPAddress(ip_source) + ip_source = case ip.u32 + when 0 + "#{ip.address}/0" + else + "#{ip.address}/#{ip.prefix}" + end + else + ipv4s = ipv4s.flatten.uniq + end rescue Exception => e p e next end case ip_protocol when 'tcp', 'udp' - rule_maps << { - :ip_protocol => ip_protocol, - :ip_sport => ip_sport.to_i, - :ip_dport => ip_dport.to_i, - :protocol => protocol, - :ip_source => ip_source, - } + if from_group == false + rule_maps << { + :ip_protocol => ip_protocol, + :ip_sport => ip_sport.to_i, + :ip_dport => ip_dport.to_i, + :protocol => protocol, + :ip_source => ip_source, + } + else + ipv4s.each { |ip| + rule_maps << { + :ip_protocol => ip_protocol, + :ip_sport => ip_sport.to_i, + :ip_dport => ip_dport.to_i, + :protocol => 'ip4', + :ip_source => ip, + } + } + end when 'icmp' # via http://docs.amazonwebservices.com/AWSEC2/latest/CommandLineReference/ # # For the ICMP protocol, the ICMP type and code must be specified. # This must be specified in the format type:code where both are integers. # Type, code, or both can be specified as -1, which is a wildcard. - rule_maps << { - :ip_protocol => ip_protocol, - :icmp_type => -1, # ip_dport.to_i, # -1 or 0, 3, 5, 8, 11, 12, 13, 14, 15, 16, 17, 18 - :icmp_code => -1, # ip_sport.to_i, # -1 or 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 - :protocol => protocol, - :ip_source => ip_source, - } + icmp_type = ip_dport.to_i + icmp_code = ip_sport.to_i + + # icmp_type + case icmp_type + when -1 + when 0, 3, 5, 8, 11, 12, 13, 14, 15, 16, 17, 18 + else + next + end + + # icmp_code + case icmp_code + when -1 + when 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + # when icmp_type equals -1 icmp_code must equal -1. + next if icmp_type == -1 + else + next + end + + if from_group == false + rule_maps << { + :ip_protocol => ip_protocol, + :icmp_type => ip_dport.to_i, # ip_dport.to_i, # -1 or 0, 3, 5, 8, 11, 12, 13, 14, 15, 16, 17, 18 + :icmp_code => ip_sport.to_i, # ip_sport.to_i, # -1 or 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + :protocol => protocol, + :ip_source => ip_source, + } + else + ipv4s.each { |ip| + rule_maps << { + :ip_protocol => ip_protocol, + :icmp_type => ip_dport.to_i, # ip_dport.to_i, # -1 or 0, 3, 5, 8, 11, 12, 13, 14, 15, 16, 17, 18 + :icmp_code => ip_sport.to_i, # ip_sport.to_i, # -1 or 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + :protocol => 'ip4', + :ip_source => ip, + } + } + end end end rule_maps end @@ -626,316 +883,460 @@ @event ||= Isono::NodeModules::EventChannel.new(@node) end end -require 'shellwords' -raise "Shellword is old version." unless Shellwords.respond_to?(:shellescape) -require 'open4' - -module CliHelper - class TimeoutError < RuntimeError; end - - def tryagain(opts={:timeout=>60, :retry=>3}, &blk) - timedout = false - curthread = Thread.current - - timersig = EventMachine.add_timer(opts[:timeout]) { - timedout = true - if curthread - curthread.raise(TimeoutError.new("timeout")) - curthread.pass - end - } - - begin - count = 0 - begin - break if blk.call - end while !timedout && ((count += 1) < opts[:retry]) - rescue TimeoutError => e - raise e - ensure - curthread = nil - EventMachine.cancel_timer(timersig) rescue nil - end - end - - class CommandError < StandardError - attr_reader :stderr, :stdout - def initialize(msg, stdout, stderr) - super(msg) - @stdout = stdout - @stderr = stderr - end - end - - def sh(cmd, args=[], opts={}) - opts = opts.merge({:expect_exitcode=>0}) - cmd = sprintf(cmd, *args.map {|a| Shellwords.shellescape(a.to_s) }) - - outbuf = errbuf = '' - blk = proc {|pid, stdin, stdout, stderr| - stdin.close - outbuf = stdout - errbuf = stderr - } - stat = Open4::popen4(cmd, &blk) - if self.respond_to? :logger - logger.debug("Exec command (pid=#{stat.pid}): #{cmd}") - logger.debug("STDOUT:\n#{outbuf}\nSTDERR:\n#{errbuf}") - end - if stat.exitstatus != opts[:expect_exitcode] - raise CommandError, "Unexpected exit code=#{stat.extstatus} (expected=#{opts{:expect_exitcode}})", - outbuf, errbuf - end - true - end -end - require 'net/telnet' module KvmHelper # Establish telnet connection to KVM monitor console def connect_monitor(port, &blk) begin telnet = ::Net::Telnet.new("Host" => "localhost", "Port"=>port.to_s, - "Prompt" => /\n\(qemu\) /, + "Prompt" => /\n\(qemu\) \z/, "Timeout" => 60, "Waittime" => 0.2) blk.call(telnet) rescue => e logger.error(e) if self.respond_to?(:logger) + raise e ensure telnet.close end end end +class InstanceMonitor < Isono::NodeModules::Base + include KvmHelper + include Dcmgr::Logger + + initialize_hook do + @thread_pool = Isono::ThreadPool.new(1) + @monitor = EventMachine::PeriodicTimer.new(5) { + @thread_pool.pass { + myinstance.check_instance + } + } + end + + terminate_hook do + @monitor.cancel + @thread_pool.shutdown + end + + def check_instance() + instlst = rpc.request('hva-collector', 'get_alive_instances', manifest.node_id) + instlst.find_all{|i| i[:state] == 'running' }.each { |i| + begin + check_kvm_process(i) + rescue Exception => e + if i[:status] == 'online' + logger.error("#{e.class}, #{e.message}") + + rpc.request('hva-collector', 'update_instance', i[:uuid], {:status=>:offline}) { |req| + req.oneshot = true + } + event.publish('hva/fault_instance', :args=>[i[:uuid]]) + end + next + end + + if i[:status] != 'online' + rpc.request('hva-collector', 'update_instance', i[:uuid], {:status=>:online}) { |req| + req.oneshot = true + } + end + } + end + + private + def check_kvm_process(i) + pid = File.read(File.expand_path("#{i[:uuid]}/kvm.pid", node.manifest.config.vm_data_dir)).to_i + unless File.exists?(File.expand_path(pid.to_s, '/proc')) + raise "Unable to find the pid of kvm process: #{pid}" + end + end + + def rpc + @rpc ||= Isono::NodeModules::RpcChannel.new(@node) + end + + def event + @event ||= Isono::NodeModules::EventChannel.new(@node) + end +end + class KvmHandler < EndpointBuilder include Dcmgr::Logger - include CliHelper + include Dcmgr::Helpers::CliHelper include KvmHelper - job :run_local_store do - #hva = rpc.delegate('hva-collector') - inst_id = request.args[0] - logger.info("Booting #{inst_id}") - #inst = hva.get_instance(inst_id) + def find_nic(ifindex = 2) + ifindex_map = {} + Dir.glob("/sys/class/net/*/ifindex").each do |ifindex_path| + device_name = File.split(File.split(ifindex_path).first)[1] + ifindex_num = File.readlines(ifindex_path).first.strip + ifindex_map[ifindex_num] = device_name + end + #p ifindex_map + ifindex_map[ifindex.to_s] + end - inst = rpc.request('hva-collector', 'get_instance', inst_id) - raise "Invalid instance state: #{inst[:state]}" unless inst[:state].to_s == 'init' + def nic_state(if_name = 'eth0') + operstate_path = "/sys/class/net/#{if_name}/operstate" + if File.exists?(operstate_path) + File.readlines(operstate_path).first.strip + end + end + def run_kvm(os_devpath) + # run vm + cmd = "kvm -m %d -smp %d -name vdc-%s -vnc :%d -drive file=%s -pidfile %s -daemonize -monitor telnet::%d,server,nowait" + args=[@inst[:instance_spec][:memory_size], + @inst[:instance_spec][:cpu_cores], + @inst_id, + @inst[:runtime_config][:vnc_port], + os_devpath, + File.expand_path('kvm.pid', @inst_data_dir), + @inst[:runtime_config][:telnet_port] + ] + if vnic = @inst[:instance_nics].first + cmd += " -net nic,macaddr=%s -net tap,ifname=%s,script=,downscript=" + args << vnic[:mac_addr].unpack('A2'*6).join(':') + args << vnic[:uuid] + end + sh(cmd, args) + + unless vnic.nil? + network_map = rpc.request('hva-collector', 'get_network', @inst[:instance_nics].first[:network_id]) + + # physical interface + physical_if = find_nic(@node.manifest.config.hv_ifindex) + raise "UnknownPhysicalNIC" if physical_if.nil? + + if network_map[:vlan_id] == 0 + # bridge interface + p bridge_if = @node.manifest.config.bridge_novlan + unless FileTest.exist?("/sys/class/net/#{bridge_if}/ifindex") + sh("/usr/sbin/brctl addbr %s", [bridge_if]) + sh("/usr/sbin/brctl addif %s %s", [bridge_if, physical_if]) + end + else + # vlan interface + vlan_if = "#{physical_if}.#{network_map[:vlan_id]}" + unless FileTest.exist?("/sys/class/net/#{vlan_if}/ifindex") + sh("/sbin/vconfig add #{physical_if} #{network_map[:vlan_id]}") + end + + # bridge interface + bridge_if = "#{@node.manifest.config.bridge_prefix}-#{physical_if}.#{network_map[:vlan_id]}" + unless FileTest.exist?("/sys/class/net/#{bridge_if}/ifindex") + sh("/usr/sbin/brctl addbr %s", [bridge_if]) + sh("/usr/sbin/brctl addif %s %s", [bridge_if, vlan_if]) + end + end + + + # interface up? down? + [ vlan_if, bridge_if ].each do |ifname| + if nic_state(ifname) == "down" + sh("/sbin/ifconfig #{ifname} 0.0.0.0 up") + end + end + + sh("/sbin/ifconfig %s 0.0.0.0 up", [vnic[:uuid]]) + sh("/usr/sbin/brctl addif %s %s", [bridge_if, vnic[:uuid]]) + end + end + + def attach_volume_to_host + # check under until the dev file is created. + # /dev/disk/by-path/ip-192.168.1.21:3260-iscsi-iqn.1986-03.com.sun:02:a1024afa-775b-65cf-b5b0-aa17f3476bfc-lun-0 + linux_dev_path = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{@vol[:storage_pool][:ipaddr]}:3260", + @vol[:transport_information][:iqn], + @vol[:transport_information][:lun]] + + tryagain do + next true if File.exist?(linux_dev_path) + + sh("iscsiadm -m discovery -t sendtargets -p %s", [@vol[:storage_pool][:ipaddr]]) + sh("iscsiadm -m node -l -T '%s' --portal '%s'", + [@vol[:transport_information][:iqn], @vol[:storage_pool][:ipaddr]]) + sleep 1 + end + + rpc.request('sta-collector', 'update_volume', { + :volume_id=>@vol_id, + :state=>:attaching, + :host_device_name => linux_dev_path}) + end + + def detach_volume_from_host + # iscsi logout + sh("iscsiadm -m node -T '%s' --logout", [@vol[:transport_information][:iqn]]) + + rpc.request('sta-collector', 'update_volume', { + :volume_id=>@vol_id, + :state=>:available, + :host_device_name=>nil, + :instance_id=>nil, + }) + event.publish('hva/volume_detached', :args=>[@inst_id, @vol_id]) + end + + def terminate_instance + kvm_pid=`pgrep -u root -f vdc-#{@inst_id}` + if $?.exitstatus == 0 && kvm_pid.to_s =~ /^\d+$/ + sh("/bin/kill #{kvm_pid}") + else + logger.error("Can not find the KVM process. Skipping: kvm -name vdc-#{@inst_id}") + end + end + + def update_instance_state(opts, ev) + raise "Can't update instance info without setting @inst_id" if @inst_id.nil? + rpc.request('hva-collector', 'update_instance', @inst_id, opts) + event.publish(ev, :args=>[@inst_id]) + end + + def update_volume_state(opts, ev) + raise "Can't update volume info without setting @vol_id" if @vol_id.nil? + rpc.request('sta-collector', 'update_volume', opts.merge(:volume_id=>@vol_id)) + event.publish(ev, :args=>[@vol_id]) + end + + job :run_local_store, proc { + @inst_id = request.args[0] + logger.info("Booting #{@inst_id}") + + @inst = rpc.request('hva-collector', 'get_instance', @inst_id) + raise "Invalid instance state: #{@inst[:state]}" unless %w(init failingover).member?(@inst[:state].to_s) + + rpc.request('hva-collector', 'update_instance', @inst_id, {:state=>:starting}) # setup vm data folder - inst_data_dir = File.expand_path("#{inst_id}", @node.manifest.config.vm_data_dir) - FileUtils.mkdir(inst_data_dir) + @inst_data_dir = File.expand_path("#{@inst_id}", @node.manifest.config.vm_data_dir) + FileUtils.mkdir(@inst_data_dir) unless File.exists?(@inst_data_dir) # copy image file - img_src = inst[:image][:source] + img_src = @inst[:image][:source] case img_src[:type].to_sym when :http - img_path = File.expand_path("#{inst_id}/#{inst[:uuid]}", @node.manifest.config.vm_data_dir) + img_path = File.expand_path("#{@inst[:uuid]}", @inst_data_dir) sh("curl --silent -o '#{img_path}' #{img_src[:uri]}") else raise "Unknown image source type: #{img_src[:type]}" end - # boot virtual machine - cmd = "kvm -m %d -smp %d -name vdc-%s -vnc :%d -drive file=%s -pidfile %s -daemonize -monitor telnet::%d,server,nowait" - args = [ - inst[:instance_spec][:memory_size], - inst[:instance_spec][:cpu_cores], - inst_id, - inst[:runtime_config][:vnc_port], - img_path, - File.expand_path('kvm.pid', inst_data_dir), - inst[:runtime_config][:telnet_port] - ] - sh(cmd, args) - - rpc.request('hva-collector', 'update_instance', inst_id, {:state=>:running}) - event.publish('hva/instance_started', :args=>[inst_id]) - end + run_kvm(img_path) + update_instance_state({:state=>:running}, 'hva/instance_started') + }, proc { + update_instance_state({:state=>:terminated, :terminated_at=>Time.now}, + 'hva/instance_terminated') + } - job :run_vol_store do - inst_id = request.args[0] - vol_id = request.args[1] + job :run_vol_store, proc { + @inst_id = request.args[0] + @vol_id = request.args[1] - inst = rpc.request('hva-collector', 'get_instance', inst_id) - vol = rpc.request('sta-collector', 'get_volume', vol_id) - logger.info("Booting #{inst_id}") - raise "Invalid instance state: #{inst[:state]}" unless inst[:state].to_s == 'init' - + @inst = rpc.request('hva-collector', 'get_instance', @inst_id) + @vol = rpc.request('sta-collector', 'get_volume', @vol_id) + logger.info("Booting #{@inst_id}") + raise "Invalid instance state: #{@inst[:state]}" unless %w(init failingover).member?(@inst[:state].to_s) + + rpc.request('hva-collector', 'update_instance', @inst_id, {:state=>:starting}) + # setup vm data folder - inst_data_dir = File.expand_path("#{inst_id}", @node.manifest.config.vm_data_dir) - FileUtils.mkdir(inst_data_dir) + @inst_data_dir = File.expand_path("#{@inst_id}", @node.manifest.config.vm_data_dir) + FileUtils.mkdir(@inst_data_dir) unless File.exists?(@inst_data_dir) # create volume from snapshot - jobreq.run("zfs-handle.#{vol[:storage_pool][:node_id]}", "create_volume", vol_id) + jobreq.run("zfs-handle.#{@vol[:storage_pool][:node_id]}", "create_volume", @vol_id) - logger.debug("volume created on #{vol[:storage_pool][:node_id]}: #{vol_id}") + logger.debug("volume created on #{@vol[:storage_pool][:node_id]}: #{@vol_id}") # reload volume info - vol = rpc.request('sta-collector', 'get_volume', vol_id) + @vol = rpc.request('sta-collector', 'get_volume', @vol_id) + rpc.request('sta-collector', 'update_volume', {:volume_id=>@vol_id, :state=>:attaching}) + logger.info("Attaching #{@vol_id} on #{@inst_id}") # check under until the dev file is created. # /dev/disk/by-path/ip-192.168.1.21:3260-iscsi-iqn.1986-03.com.sun:02:a1024afa-775b-65cf-b5b0-aa17f3476bfc-lun-0 - linux_dev_path = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{vol[:storage_pool][:ipaddr]}:3260", - vol[:transport_information][:iqn], - vol[:transport_information][:lun]] + linux_dev_path = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{@vol[:storage_pool][:ipaddr]}:3260", + @vol[:transport_information][:iqn], + @vol[:transport_information][:lun]] # attach disk - tryagain do - sh("iscsiadm -m discovery -t sendtargets -p #{vol[:storage_pool][:ipaddr]}") - sh("iscsiadm -m node -l -T '#{vol[:transport_information][:iqn]}' --portal '#{vol[:storage_pool][:ipaddr]}:3260'") - sleep 1 - File.exist?(linux_dev_path) - end - + attach_volume_to_host + # run vm - cmd = "kvm -m %d -smp %d -name vdc-%s -vnc :%d -drive file=%s -pidfile %s -daemonize -monitor telnet::%d,server,nowait" - args=[inst[:instance_spec][:memory_size], - inst[:instance_spec][:cpu_cores], - inst_id, - inst[:runtime_config][:vnc_port], - linux_dev_path, - File.expand_path('kvm.pid', inst_data_dir), - inst[:runtime_config][:telnet_port] - ] - if vnic = inst[:instance_nics].first - cmd += " -net nic,macaddr=%s -net tap,ifname=%s" - args << vnic[:mac_addr].unpack('A2'*6).join(':') - args << vnic[:vif] - end - sh(cmd, args) - - rpc.request('hva-collector', 'update_instance', inst_id, {:state=>:running}) - event.publish('hva/instance_started', :args=>[inst_id]) - end + run_kvm(linux_dev_path) + update_instance_state({:state=>:running}, 'hva/instance_started') + update_volume_state({:state=>:attached}, 'hva/volume_attached') + }, proc { + update_instance_state({:state=>:terminated, :terminated_at=>Time.now}, + 'hva/instance_terminated') + } job :terminate do - inst_id = request.args[0] - - inst = rpc.request('hva-collector', 'get_instance', inst_id) - raise "Invalid instance state: #{inst[:state]}" unless inst[:state].to_s == 'running' - - rpc.request('hva-collector', 'update_instance', inst_id, {:state=>:shuttingdown}) + @inst_id = request.args[0] - kvm_pid=`pgrep -u root -f vdc-#{inst_id}` - unless $?.exitstatus == 0 && kvm_pid.to_s =~ /^\d+$/ - raise "No such VM process: kvm -name vdc-#{inst_id}" - end + @inst = rpc.request('hva-collector', 'get_instance', @inst_id) + raise "Invalid instance state: #{@inst[:state]}" unless @inst[:state].to_s == 'running' - sh("/bin/kill #{kvm_pid}") + begin + rpc.request('hva-collector', 'update_instance', @inst_id, {:state=>:shuttingdown}) + + terminate_instance - unless inst[:volume].nil? - inst[:volume].each { |volid, v| - sh("iscsiadm -m node -T '#{v[:transport_information][:iqn]}' --logout") - } + unless @inst[:volume].nil? + @inst[:volume].each { |volid, v| + @vol_id = volid + @vol = v + # force to continue detaching volumes during termination. + detach_volume_from_host rescue logger.error($!) + } + end + + # cleanup vm data folder + FileUtils.rm_r(File.expand_path("#{@inst_id}", @node.manifest.config.vm_data_dir)) + ensure + update_instance_state({:state=>:terminated,:terminated_at=>Time.now}, + 'hva/instance_terminated') end + end - # cleanup vm data folder - FileUtils.rm_r(File.expand_path("#{inst_id}", @node.manifest.config.vm_data_dir)) + # just do terminate instance and unmount volumes not to affect + # state management. + # called from HA at which the faluty instance get cleaned properly. + job :cleanup do + @inst_id = request.args[0] - rpc.request('hva-collector', 'update_instance', inst_id, {:state=>:terminated}) - event.publish('hva/instance_terminated', :args=>[inst_id]) + @inst = rpc.request('hva-collector', 'get_instance', @inst_id) + raise "Invalid instance state: #{@inst[:state]}" unless @inst[:state].to_s == 'running' + + begin + terminate_instance + + unless @inst[:volume].nil? + @inst[:volume].each { |volid, v| + @vol_id = volid + @vol = v + # force to continue detaching volumes during termination. + detach_volume_from_host rescue logger.error($!) + } + end + end + end - job :attach do - inst_id = request.args[0] - vol_id = request.args[1] + job :attach, proc { + @inst_id = request.args[0] + @vol_id = request.args[1] - job = Dcmgr::Stm::VolumeContext.new(vol_id) - inst = rpc.request('hva-collector', 'get_instance', inst_id) - vol = rpc.request('sta-collector', 'get_volume', vol_id) - logger.info("Attaching #{vol_id}") - job.stm.state = vol[:state].to_sym - raise "Invalid volume state: #{vol[:state]}" unless vol[:state].to_s == 'available' + @job = Dcmgr::Stm::VolumeContext.new(@vol_id) + @inst = rpc.request('hva-collector', 'get_instance', @inst_id) + @vol = rpc.request('sta-collector', 'get_volume', @vol_id) + logger.info("Attaching #{@vol_id}") + @job.stm.state = @vol[:state].to_sym + raise "Invalid volume state: #{@vol[:state]}" unless @vol[:state].to_s == 'available' - job.stm.on_attach + @job.stm.on_attach + rpc.request('sta-collector', 'update_volume', {:volume_id=>@vol_id, :state=>:attaching}) # check under until the dev file is created. # /dev/disk/by-path/ip-192.168.1.21:3260-iscsi-iqn.1986-03.com.sun:02:a1024afa-775b-65cf-b5b0-aa17f3476bfc-lun-0 - linux_dev_path = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{vol[:storage_pool][:ipaddr]}:3260", - vol[:transport_information][:iqn], - vol[:transport_information][:lun]] + linux_dev_path = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{@vol[:storage_pool][:ipaddr]}:3260", + @vol[:transport_information][:iqn], + @vol[:transport_information][:lun]] # attach disk on host os - tryagain do - sh("iscsiadm -m discovery -t sendtargets -p #{vol[:storage_pool][:ipaddr]}") - sh("iscsiadm -m node -l -T '#{vol[:transport_information][:iqn]}' --portal '#{vol[:storage_pool][:ipaddr]}:3260'") - sleep 1 - File.exist?(linux_dev_path) - end + attach_volume_to_host - rpc.request('sta-collector', 'update_volume', job.to_hash(:host_device_name => linux_dev_path)) - logger.info("Attaching #{vol_id} on #{inst_id}") - job.stm.on_attach - job.on_attach + logger.info("Attaching #{@vol_id} on #{@inst_id}") + @job.stm.on_attach + @job.on_attach # attach disk on guest os - require 'net/telnet' - slot_number = nil - pci = nil - slink = `ls -la #{linux_dev_path}`.scan(/.+\s..\/..\/([a-z]+)/) - raise "volume has not attached host os" if slink.nil? + # pci_devddr consists of three hex numbers with colon separator. + # dom <= 0xffff && bus <= 0xff && val <= 0x1f + # see: qemu-0.12.5/hw/pci.c + # /* + # * Parse [[<domain>:]<bus>:]<slot>, return -1 on error + # */ + # static int pci_parse_devaddr(const char *addr, int *domp, int *busp, unsigned *slotp) + pci_devaddr = nil - begin - telnet = ::Net::Telnet.new("Host" => "localhost", "Port"=>"#{inst[:runtime_config][:telnet_port]}", "Prompt" => /\n\(qemu\) /, "Timeout" => 60, "Waittime" => 0.2) - telnet.cmd({"String" => "pci_add auto storage file=/dev/#{slink},if=scsi", "Match" => /.+slot\s[0-9]+.+/}){|c| - pci_add = c.scan(/.+slot\s([0-9]+).+/) - slot_number = pci_add unless pci_add.empty? - } - telnet.cmd("info pci"){|c| - pci = c.scan(/^(.+[a-zA-z]+.+[0-9],.+device.+#{slot_number},.+:)/) - } - rescue => e - logger.error(e) - ensure - telnet.close - end - raise "volume has not attached" if pci.nil? - rpc.request('sta-collector', 'update_volume', job.to_hash(:guest_device_name=>slot_number)) - logger.info("Attached #{vol_id} on #{inst_id}") - end + sddev = File.expand_path(File.readlink(linux_dev_path), '/dev/disk/by-path') + connect_monitor(@inst[:runtime_config][:telnet_port]) { |t| + # success message: + # OK domain 0, bus 0, slot 4, function 0 + # error message: + # failed to add file=/dev/xxxx,if=virtio + c = t.cmd("pci_add auto storage file=#{sddev},if=scsi") + # Note: pci_parse_devaddr() called in "pci_add" uses strtoul() + # with base 16 so that the input is expected in hex. however + # at the result display, void pci_device_hot_add_print() uses + # %d for showing bus and slot addresses. use hex to preserve + # those values to keep consistent. + if c =~ /\nOK domain ([0-9a-fA-F]+), bus ([0-9a-fA-F]+), slot ([0-9a-fA-F]+), function/m + # numbers in OK result is decimal. convert them to hex. + pci_devaddr = [$1, $2, $3].map{|i| i.to_i.to_s(16) } + else + raise "Error in qemu console: #{c}" + end + + # double check the pci address. + c = t.cmd("info pci") + # static void pci_info_device(PCIBus *bus, PCIDevice *d) + # called in "info pci" gets back PCI bus info with %d. + if c.split(/\n/).grep(/^\s+Bus\s+#{pci_devaddr[1].to_i(16)}, device\s+#{pci_devaddr[2].to_i(16)}, function/).empty? + raise "Could not find new disk device attached to qemu-kvm: #{pci_devaddr.join(':')}" + end + } + + rpc.request('sta-collector', 'update_volume', @job.to_hash(:guest_device_name=>pci_devaddr.join(':'))) + event.publish('hva/volume_attached', :args=>[@inst_id, @vol_id]) + logger.info("Attached #{@vol_id} on #{@inst_id}") + } + job :detach do - inst_id = request.args[0] - vol_id = request.args[1] + @inst_id = request.args[0] + @vol_id = request.args[1] - job = Dcmgr::Stm::VolumeContext.new(vol_id) - inst = rpc.request('hva-collector', 'get_instance', inst_id) - vol = rpc.request('sta-collector', 'get_volume', vol_id) - logger.info("Detaching #{vol_id} on #{inst_id}") - job.stm.state = vol[:state].to_sym - raise "Invalid volume state: #{vol[:state]}" unless vol[:state].to_s == 'attached' + @job = Dcmgr::Stm::VolumeContext.new(@vol_id) + @inst = rpc.request('hva-collector', 'get_instance', @inst_id) + @vol = rpc.request('sta-collector', 'get_volume', @vol_id) + logger.info("Detaching #{@vol_id} on #{@inst_id}") + @job.stm.state = @vol[:state].to_sym + raise "Invalid volume state: #{@vol[:state]}" unless @vol[:state].to_s == 'attached' - job.stm.on_detach + @job.stm.on_detach + rpc.request('sta-collector', 'update_volume', @job.to_hash) # detach disk on guest os - require 'net/telnet' - pci = nil + pci_devaddr = @vol[:guest_device_name] - begin - telnet = ::Net::Telnet.new("Host" => "localhost", "Port"=>"#{inst[:runtime_config][:telnet_port]}", "Prompt" => /\n\(qemu\) /, "Timeout" => 60, "Waittime" => 0.2) - telnet.cmd("pci_del #{vol[:guest_device_name]}") - telnet.cmd("info pci"){|c| - pci = c.scan(/^(.+[a-zA-z]+.+[0-9],.+device.+#{vol[:guest_device_name]},.+:)/) - } - rescue => e - logger.error(e) - ensure - telnet.close - end - raise "volume has not detached" unless pci.empty? - rpc.request('sta-collector', 'update_volume', job.to_hash) - - # iscsi logout - job.stm.on_detach - job.on_detach - logger.info("iscsi logout #{vol_id}: #{vol[:transport_information][:iqn]}") - initiator = `sudo iscsiadm -m node -T '#{vol[:transport_information][:iqn]}' --logout` - rpc.request('sta-collector', 'update_volume', job.to_hash) + connect_monitor(@inst[:runtime_config][:telnet_port]) { |t| + t.cmd("pci_del #{pci_devaddr}") + # + # Bus 0, device 4, function 0: + # SCSI controller: PCI device 1af4:1001 + # IRQ 0. + # BAR0: I/O at 0x1000 [0x103f]. + # BAR1: 32 bit memory at 0x08000000 [0x08000fff]. + # id "" + c = t.cmd("info pci") + pci_devaddr = pci_devaddr.split(':') + unless c.split(/\n/).grep(/\s+Bus\s+#{pci_devaddr[1].to_i(16)}, device\s+#{pci_devaddr[2].to_i(16)}, function/).empty? + raise "Detached disk device still be attached in qemu-kvm: #{pci_devaddr.join(':')}" + end + } + + detach_volume_from_host + + @job.stm.on_detach + @job.on_detach end def rpc @rpc ||= Isono::NodeModules::RpcChannel.new(@node) end @@ -954,14 +1355,18 @@ manifest.instance_eval do node_name 'hva' node_instance_id "#{Isono::Util.default_gw_ipaddr}" load_module Isono::NodeModules::NodeHeartbeat load_module ServiceNetfilter + load_module InstanceMonitor config do |c| c.vm_data_dir = '/var/lib/vm' c.enable_ebtables = true c.enable_iptables = true + c.hv_ifindex = 2 # ex. /sys/class/net/eth0/ifindex => 2 + c.bridge_prefix = 'br' + c.bridge_novlan = 'br0' end config_path File.expand_path('config/hva.conf', app_root) load_config end