From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 9342 invoked by alias); 22 Jun 2009 17:59:01 -0000 Received: (qmail 9327 invoked by alias); 22 Jun 2009 17:59:00 -0000 X-SWARE-Spam-Status: No, hits=-2.1 required=5.0 tests=AWL,BAYES_00,SPF_HELO_PASS X-Spam-Status: No, hits=-2.1 required=5.0 tests=AWL,BAYES_00,SPF_HELO_PASS X-Spam-Check-By: sourceware.org X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on bastion2.fedora.phx.redhat.com Subject: resource-agents: master - rgmanager: Make vm.sh use libvirt To: cluster-cvs-relay@redhat.com X-Project: Cluster Project X-Git-Module: resource-agents.git X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: fc54b6f4f32c4c9b2bc5e5069d8b3c4f4752eb5c X-Git-Newrev: 29095d987e34a8832217f7d834de0f13454856fb From: Lon Hohberger Message-Id: <20090622175821.F254512027E@lists.fedorahosted.org> Date: Mon, 22 Jun 2009 17:59:00 -0000 X-Scanned-By: MIMEDefang 2.58 on 172.16.52.254 Mailing-List: contact cluster-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cluster-cvs-owner@sourceware.org X-SW-Source: 2009-q2/txt/msg00663.txt.bz2 Gitweb: http://git.fedorahosted.org/git/resource-agents.git?p=resource-agents.git;a=commitdiff;h=29095d987e34a8832217f7d834de0f13454856fb Commit: 29095d987e34a8832217f7d834de0f13454856fb Parent: fc54b6f4f32c4c9b2bc5e5069d8b3c4f4752eb5c Author: Lon Hohberger AuthorDate: Wed May 20 17:41:38 2009 -0400 Committer: Lon Hohberger CommitterDate: Mon Jun 22 13:55:00 2009 -0400 rgmanager: Make vm.sh use libvirt This makes rgmanager use virsh/libvirt instead of xm so it can manage either KVM or Xen virtual machines. It will still use xm if a user has a non-standard config file path, however, the recommendation is that the user store all Xen configuration files in one place and use bind mounts to mount the requisite path on /etc/xen. Virsh does not have a notion of a search path like xm, therefore, further support of the path attribute in xm will be limited. During this partial overhaul, extraneous attribute parsing was removed and the 'stop' phase now correctly returns failure if it cannot contact the hypervisor. Users may force the use of 'xm' by adding: use_virsh="0" to vm resources. Furthermore, users may force use of a specific hypervisor by using: hypervisor="qemu" or: hypervisor="xen" The other important note is that libvirtd is now required to be running in order to operate with the virsh command set even if using the Xen hypervisor. Resolves: 412911 468691 Signed-off-by: Lon Hohberger --- rgmanager/src/resources/vm.sh | 606 ++++++++++++++++++++++++++++------------- 1 files changed, 413 insertions(+), 193 deletions(-) diff --git a/rgmanager/src/resources/vm.sh b/rgmanager/src/resources/vm.sh index 07561d0..1d6a0da 100644 --- a/rgmanager/src/resources/vm.sh +++ b/rgmanager/src/resources/vm.sh @@ -6,12 +6,13 @@ export PATH . $(dirname $0)/ocf-shellfuncs || exit 1 -. $(dirname $0)/ocf-shellfuncs - # -# Virtual Machine start/stop script (requires the xm command) +# Virtual Machine start/stop script (requires the virsh command) # +# Indeterminate state: xend/libvirtd is down. +export OCF_APP_ERR_INDETERMINATE=150 + meta_data() { cat < - - - Memory size. This can be reconfigured on the fly. - - - Memory Size - - - - Mapping of the hostname of a target cluster member to a different hostname - memeberhost:targethost,memeberhost:targethost .. + memberhost:targethost,memberhost:targethost .. - - - Boot loader that can start the VM from physical image - - - Boot loader that can start the VM from physical image - - - - - + - Path specification 'xm create' will search for the specified - VM configuration file + Use virsh instead of XM - Path to virtual machine configuration files + If set to 1, vm.sh will use the virsh command to manage + virtual machines instead of xm. This is required when + using non-Xen virtual machines (e.g. qemu / KVM). - - - - - - - Root disk for the virtual machine. (physical, on the host) - - - Root disk (physical) - - - - - - - Root disk for the virtual machine. (as presented to the VM) - - - Root disk (virtual) - - - - - - - - Swap disk for the virtual machine. (physical, on the host) - - - Swap disk (physical) - - - - - - - Swap disk for the virtual machine. (as presented to the VM) - - - Swap disk (virtual) - - - - - - - Virtual interface MAC address - - - Virtual interface MAC address - - + @@ -260,19 +190,41 @@ meta_data() + + + + + + Hypervisor + - Restart expiration time + Specify hypervisor tricks to use. Default = auto. + Other supported options are xen and qemu. + + + + - Restart expiration time. A restart is forgotten - after this time. When combined with the max_restarts - option, this lets administrators specify a threshold - for when to fail over services. If max_restarts - is exceeded in this given expiration time, the service - is relocated instead of restarted again. - - - + Hypervisor URI + + + Hypervisor URI. Generally, this is keyed off of the + hypervisor and does not need to be set. + + + + + + + Migration URI + + + Migration URI. Generally, this is keyed off of the + hypervisor and does not need to be set. + + + @@ -287,9 +239,6 @@ meta_data() NOT OCF COMPATIBLE AT ALL --> - - @@ -309,89 +258,79 @@ EOT } +build_virsh_cmdline() +{ + declare cmdline="" + declare operation=$1 + + if [ -n "$OCF_RESKEY_hypervisor_uri" ]; then + cmdline="$cmdline -c $OCF_RESKEY_hypervisor_uri" + fi + + cmdline="$cmdline $operation $OCF_RESKEY_name" + + echo $cmdline +} + + +# this is only used on startup build_xm_cmdline() { + declare operation=$1 # # Virtual domains should never restart themselves when # controlled externally; the external monitoring app # should. # declare cmdline="on_shutdown=\"destroy\" on_reboot=\"destroy\" on_crash=\"destroy\"" - declare varp val temp - # - # Transliterate the OCF_RESKEY_* to something the xm - # command can recognize. - # - for var in ${!OCF_RESKEY_*}; do - varp=${var/OCF_RESKEY_/} - val=`eval "echo \\$$var"` - - case $varp in - bootloader) - cmdline="$cmdline bootloader=\"$val\"" - ;; - rootdisk_physical) - [ -n "$OCF_RESKEY_rootdisk_virtual" ] || exit 2 - cmdline="$cmdline disk=\"phy:$val,$OCF_RESKEY_rootdisk_virtual,w\"" - ;; - swapdisk_physical) - [ -n "$OCF_RESKEY_swapdisk_virtual" ] || exit 2 - cmdline="$cmdline disk=\"phy:$val,$OCF_RESKEY_swapdisk_virtual,w\"" - ;; - vif) - cmdline="$cmdline vif=\"mac=$val\"" - ;; - recovery|autostart|domain) - ;; - memory) - cmdline="$cmdline $varp=$val" - ;; - swapdisk_virtual) - ;; - rootdisk_virtual) - ;; - name) # Do nothing with name; add it later - ;; - path) - cmdline="$cmdline --path=\"$val\"" - ;; - migrate) - ;; - snapshot) - ;; - *) - cmdline="$cmdline $varp=\"$val\"" - ;; - esac - done + if [ -n "$OCF_RESKEY_path" ]; then + operation="$operation --path=\"$OCF_RESKEY_path\"" + fi if [ -n "$OCF_RESKEY_name" ]; then - cmdline="$OCF_RESKEY_name $cmdline" + cmdline="$operation $OCF_RESKEY_name $cmdline" fi echo $cmdline } +do_xm_start() +{ + # Use /dev/null for the configuration file, if xmdefconfig + # doesn't exist... + # + declare cmdline + + echo -n "Virtual machine $OCF_RESKEY_name is " + do_status && return 0 + + cmdline="`build_xm_cmdline create`" + + ocf_log debug "xm $cmdline" + + eval xm $cmdline + return $? +} + + # # Start a virtual machine given the parameters from # the environment. # -do_start() +do_virsh_start() { - # Use /dev/null for the configuration file, if xmdefconfig - # doesn't exist... - # declare cmdline declare snapshotimage - status && return 0 + echo -n "Virtual machine $OCF_RESKEY_name is " + do_status && return 0 snapshotimage="$OCF_RESKEY_snapshot/$OCF_RESKEY_name" if [ -n "$OCF_RESKEY_snapshot" -a -f "$snapshotimage" ]; then - eval xm restore $snapshotimage + eval virsh restore $snapshotimage if [ $? -eq 0 ]; then rm -f $snapshotimage return 0 @@ -399,38 +338,29 @@ do_start() return 1 fi - cmdline="`build_xm_cmdline`" + cmdline="virsh $(build_virsh_cmdline start)" + ocf_log debug "$cmdline" - echo "# xm command line: $cmdline" - - eval xm create $cmdline + $cmdline return $? } -# -# Stop a VM. Try to shut it down. Wait a bit, and if it -# doesn't shut down, destroy it. -# -do_stop() +do_xm_stop() { declare -i timeout=60 declare -i ret=1 declare st - if [ -n "$OCF_RESKEY_snapshot" ]; then - xm save $OCF_RESKEY_name "$OCF_RESKEY_snapshot/$OCF_RESKEY_name" - fi - for op in $*; do - echo xm $op $OCF_RESKEY_name ... + echo "CMD: xm $op $OCF_RESKEY_name" xm $op $OCF_RESKEY_name timeout=60 while [ $timeout -gt 0 ]; do sleep 5 ((timeout -= 5)) - status || return 0 + do_status&>/dev/null || return 0 while read dom state; do # # State is "stopped". Kill it. @@ -451,105 +381,394 @@ do_stop() # -# Reconfigure a running VM. Currently, all we support is -# memory ballooning. +# Stop a VM. Try to shut it down. Wait a bit, and if it +# doesn't shut down, destroy it. # -reconfigure() +do_virsh_stop() +{ + declare -i timeout=60 + declare -i ret=1 + declare state + + state=$(do_status) + [ $? -eq 0 ] || return 0 + + if [ -n "$OCF_RESKEY_snapshot" ]; then + virsh save $OCF_RESKEY_name "$OCF_RESKEY_snapshot/$OCF_RESKEY_name" + fi + + for op in $*; do + echo virsh $op $OCF_RESKEY_name ... + virsh $op $OCF_RESKEY_name + + timeout=60 + while [ $timeout -gt 0 ]; do + sleep 5 + ((timeout -= 5)) + state=$(do_status) + [ $? -eq 0 ] || return 0 + + if [ "$state" = "paused" ]; then + virsh destroy $OCF_RESKEY_name + fi + done + done + + return 1 +} + + +do_start() { - if [ -n "$OCF_RESKEY_memory" ]; then - echo "xm balloon $OCF_RESKEY_name $OCF_RESKEY_memory" - xm balloon $OCF_RESKEY_name $OCF_RESKEY_memory + if [ "$OCF_RESKEY_use_virsh" = "1" ]; then + do_virsh_start $* return $? fi - return 0 + + do_xm_start $* + return $? +} + + +do_stop() +{ + declare domstate rv + + domstate=$(do_status) + rv=$? + ocf_log debug "Virtual machine $OCF_RESKEY_name is $domstate" + if [ $rv -eq $OCF_APP_ERR_INDETERMINATE ]; then + ocf_log crit "xend/libvirtd is dead; cannot stop $OCF_RESKEY_name" + return 1 + fi + + if [ "$OCF_RESKEY_use_virsh" = "1" ]; then + do_virsh_stop $* + return $? + fi + + do_xm_stop $* + return $? } # -# Simple status check: Find the VM in the list of running -# VMs +# Reconfigure a running VM. # -do_status() +reconfigure() { + return 0 +} + + +xm_status() +{ + service xend status &> /dev/null + if [ $? -ne 0 ]; then + # if xend died + echo indeterminate + return $OCF_APP_ERR_INDETERMINATE + fi + xm list $OCF_RESKEY_name &> /dev/null if [ $? -eq 0 ]; then + echo "running" return 0 fi xm list migrating-$OCF_RESKEY_name &> /dev/null + if [ $? -eq 0 ]; then + echo "running" + return 0 + fi + echo "not running" + return 1 +} + + +virsh_status() +{ + declare state pid + + if [ "$OCF_RESKEY_hypervisor" = "xen" ]; then + service xend status &> /dev/null + if [ $? -ne 0 ]; then + echo indeterminate + return $OCF_APP_ERR_INDETERMINATE + fi + fi + + # + # libvirtd is required when using virsh even though + # not specifically when also using Xen. This is because + # libvirtd is required for migration. + # + pid=$(pidof libvirtd) + if [ -z "$pid" ]; then + echo indeterminate + return $OCF_APP_ERR_INDETERMINATE + fi + + state=$(virsh domstate $OCF_RESKEY_name) + + echo $state + + if [ "$state" = "running" ] || [ "$state" = "paused" ] || + [ "$state" = "idle" ]; then + return 0 + fi + + return 1 +} + + +# +# Simple status check: Find the VM in the list of running +# VMs +# +do_status() +{ + if [ "$OCF_RESKEY_use_virsh" = "1" ]; then + virsh_status + return $? + fi + + xm_status return $? } -verify_all() +validate_all() { - declare errors=0 + [ "$(id -u)" = "0" ] || return 1 - if [ -n "$OCF_RESKEY_bootloader" ] && \ - ! [ -x "$OCF_RESKEY_bootloader" ]; then - echo "$OCF_RESKEY_bootloader is not executable" - ((errors++)) + # + # If someone selects a hypervisor, honor it. + # Otherwise, ask virsh what the hypervisor is. + # + if [ -z "$OCF_RESKEY_hypervisor" ] || + [ "$OCF_RESKEY_hypervisor" = "auto" ]; then + export OCF_RESKEY_hypervisor="`virsh version | grep \"Running hypervisor:\" | awk '{print $3}' | tr A-Z a-z`" + if [ -z "$OCF_RESKEY_hypervisor" ]; then + ocf_log err "Could not determine Hypervisor" + return $OCF_ERR_ARGS + fi + echo Hypervisor: $OCF_RESKEY_hypervisor fi + + # + # Xen hypervisor only for when use_virsh = 0. + # + if [ "$OCF_RESKEY_use_virsh" = "0" ]; then + if [ "$OCF_RESKEY_hypervisor" != "xen" ]; then + ocf_log err "Cannot use $OCF_RESKEY_hypervisor hypervisor without using virsh" + return $OCF_ERR_ARGS + fi + else + + # + # If no path is set, use virsh. Otherwise, use xm. + # xm only works with Xen. + # + if [ -z "$OCF_RESKEY_path" ] || + [ "$OCF_RESKEY_path" = "/etc/xen" ]; then + echo "Management tool: virsh" + export OCF_RESKEY_use_virsh=1 + else + echo "Management tool: xm" + export OCF_RESKEY_use_virsh=0 + fi + fi + + # + # Set the hypervisor URI + # + if [ -z "$OCF_RESKEY_hypervisor_uri" -o "$OCF_RESKEY_hypervisor_uri" = "auto" ] && + [ "$OCF_RESKEY_use_virsh" = "1" ]; then + + # Virsh makes it easier to do this. Really. + if [ "$OCF_RESKEY_hypervisor" = "qemu" ]; then + OCF_RESKEY_hypervisor_uri="qemu:///system" + fi + + # I just need to believe in it more. + if [ "$OCF_RESKEY_hypervisor" = "xen" ]; then + OCF_RESKEY_hypervisor_uri="xen:///" + fi + + echo Hypervisor URI: $OCF_RESKEY_hypervisor_uri + fi + + # + # Set the migration URI + # + if [ -z "$OCF_RESKEY_migration_uri" -o "$OCF_RESKEY_migration_uri" = "auto" ] && + [ "$OCF_RESKEY_use_virsh" = "1" ]; then + + # Virsh makes it easier to do this. Really. + if [ "$OCF_RESKEY_hypervisor" = "qemu" ]; then + export OCF_RESKEY_migration_uri="qemu+ssh://%s/system" + fi + + # I just need to believe in it more. + if [ "$OCF_RESKEY_hypervisor" = "xen" ]; then + export OCF_RESKEY_migration_uri="xenmigr://%s/" + fi + + [ -n "$OCF_RESKEY_migration_uri" ] && echo Migration URI format: $(printf $OCF_RESKEY_migration_uri target_host) + fi + + if [ -z "$OCF_RESKEY_name" ]; then + echo No domain name specified + return $OCF_ERR_ARGS + fi + + #virsh list --all | awk '{print $2}' | grep -q "^$OCF_RESKEY_name\$" + return $? } -migrate() +virsh_migrate() +{ + declare $target=$1 + declare rv=1 + + # + # Xen and qemu have different migration mechanisms + # + if [ "$OCF_RESKEY_hypervisor" = "xen" ]; then + cmd="virsh migrate $migrate_opt $OCF_RESKEY_name $OCF_RESKEY_hypervisor_uri $(printf $OCF_RESKEY_migration_uri $target)" + ocf_log debug "$cmd" + + err=$($cmd 2>&1 | head -1; exit ${PIPESTATUS[0]}) + rv=$? + elif [ "$OCF_RESKEY_hypervisor" = "qemu" ]; then + cmd="virsh migrate $migrate_opt $OCF_RESKEY_name $(printf $OCF_RESKEY_migration_uri $target)" + ocf_log debug "$cmd" + + err=$($cmd 2>&1 | head -1; exit ${PIPESTATUS[0]}) + rv=$? + fi + + if [ $rv -ne 0 ]; then + ocf_log err "Migrate $OCF_RESKEY_name to $target failed:" + ocf_log err "$err" + + if [ "$err" != "${err/does not exist/}" ]; then + return $OCF_NOT_RUNNING + fi + if [ "$err" != "${err/Domain not found/}" ]; then + return $OCF_NOT_RUNNING + fi + if [ "$err" != "${err/Connection refused/}" ]; then + return $OCF_ERR_CONFIGURED + fi + + return $OCF_ERR_GENERIC + fi + + return $rv +} + + +# +# XM migrate +# +xm_migrate() { declare target=$1 - declare errstr rv migrate_opt + declare errstr rv migrate_opt cmd + + rv=1 if [ "$OCF_RESKEY_migrate" = "live" ]; then migrate_opt="-l" fi - # Patch from Marcelo Azevedo to migrate over private - # LANs instead of public LANs - if [ -n "$OCF_RESKEY_migration_mapping" ] ; then - target=${OCF_RESKEY_migration_mapping#*$target:} target=${target%%,*} - fi + # migrate() function sets target using migration_mapping; + # no need to do it here anymore + cmd="xm migrate $migrate_opt $OCF_RESKEY_name $target" + ocf_log debug "$cmd" - err=$(xm migrate $migrate_opt $OCF_RESKEY_name $target 2>&1 | head -1; exit ${PIPESTATUS[0]}) + err=$($cmd 2>&1 | head -1; exit ${PIPESTATUS[0]}) rv=$? if [ $rv -ne 0 ]; then + ocf_log err "Migrate $OCF_RESKEY_name to $target failed:" + ocf_log err "$err" + if [ "$err" != "${err/does not exist/}" ]; then - ocf_log warn "Trying to migrate '$OCF_RESKEY_name' - domain does not exist" return $OCF_NOT_RUNNING fi if [ "$err" != "${err/Connection refused/}" ]; then - ocf_log warn "Trying to migrate '$OCF_RESKEY_name' - connect refused" return $OCF_ERR_CONFIGURED fi + + return $OCF_ERR_GENERIC fi return $? } +# +# Virsh migrate +# +migrate() +{ + declare target=$1 + declare rv migrate_opt + + if [ "$OCF_RESKEY_migrate" = "live" ]; then + migrate_opt="--live" + fi + + # Patch from Marcelo Azevedo to migrate over private + # LANs instead of public LANs + if [ -n "$OCF_RESKEY_migration_mapping" ] ; then + target=${OCF_RESKEY_migration_mapping#*$target:} target=${target%%,*} + fi + + if [ "$OCF_RESKEY_use_virsh" = "1" ]; then + virsh_migrate $target + rv=$? + else + xm_migrate $target + rv=$? + fi + + return $rv +} + +# # -# A Resource group is abstract, but the OCF RA API doesn't allow for abstract -# resources, so here it is. # case $1 in start) + validate_all || exit $OCF_ERR_ARGS do_start exit $? ;; stop) + validate_all || exit $OCF_ERR_ARGS do_stop shutdown destroy exit $? ;; kill) - stop destroy + validate_all || exit $OCF_ERR_ARGS + do_stop destroy exit $? ;; recover|restart) exit 0 ;; status|monitor) + validate_all || exit $OCF_ERR_ARGS + echo -n "Virtual machine $OCF_RESKEY_name is " do_status exit $? ;; migrate) + validate_all || exit $OCF_ERR_ARGS migrate $2 # Send VM to this node exit $? ;; @@ -557,6 +776,7 @@ case $1 in exit 0 ;; reconfig) + validate_all || exit $OCF_ERR_ARGS echo "$0 RECONFIGURING $OCF_RESKEY_memory" reconfigure exit $? @@ -566,7 +786,7 @@ case $1 in exit 0 ;; validate-all) - verify_all + validate_all exit $? ;; *)