From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 7171 invoked by alias); 17 Mar 2009 18:39:21 -0000 Received: (qmail 7164 invoked by alias); 17 Mar 2009 18:39:20 -0000 X-SWARE-Spam-Status: No, hits=-0.7 required=5.0 tests=AWL,BAYES_00,J_CHICKENPOX_33,J_CHICKENPOX_43,J_CHICKENPOX_52,J_CHICKENPOX_63,SPF_HELO_PASS X-Spam-Status: No, hits=-0.7 required=5.0 tests=AWL,BAYES_00,J_CHICKENPOX_33,J_CHICKENPOX_43,J_CHICKENPOX_52,J_CHICKENPOX_63,SPF_HELO_PASS X-Spam-Check-By: sourceware.org X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on bastion.fedora.phx.redhat.com Subject: cluster: STABLE3 - fence_tool, init.d/cman: fix wait/retry options To: cluster-cvs-relay@redhat.com X-Project: Cluster Project X-Git-Module: cluster.git X-Git-Refname: refs/heads/STABLE3 X-Git-Reftype: branch X-Git-Oldrev: 0eec7f7ae4eb29877631ad6ca4d50b8eb8b03bca X-Git-Newrev: f77f286c34c8cd0ac904528341c07f5b4370470a From: David Teigland Message-Id: <20090317183900.1286E120150@lists.fedorahosted.org> Date: Tue, 17 Mar 2009 18:39:00 -0000 X-Scanned-By: MIMEDefang 2.58 on 172.16.52.254 Mailing-List: contact cluster-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cluster-cvs-owner@sourceware.org X-SW-Source: 2009-q1/txt/msg00806.txt.bz2 Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=f77f286c34c8cd0ac904528341c07f5b4370470a Commit: f77f286c34c8cd0ac904528341c07f5b4370470a Parent: 0eec7f7ae4eb29877631ad6ca4d50b8eb8b03bca Author: David Teigland AuthorDate: Mon Mar 16 16:37:42 2009 -0500 Committer: David Teigland CommitterDate: Tue Mar 17 13:34:36 2009 -0500 fence_tool, init.d/cman: fix wait/retry options Bring some sanity to the fence_tool ad hoc wait/retry options, which are used by init.d/cman. At a high level we want: . fence_tool join to fail right away, with an error, if cman or fenced fail or aren't running . fence_tool join to exit with 0 if the join succeeds and with 1 if it fails . do these things properly even when fenced is slow starting up, or in processing the join Signed-off-by: David Teigland --- cman/init.d/cman.in | 20 +-- fence/fence_tool/fence_tool.c | 335 +++++++++++++++++++++++++---------------- 2 files changed, 215 insertions(+), 140 deletions(-) diff --git a/cman/init.d/cman.in b/cman/init.d/cman.in index 288c267..bbaec7a 100644 --- a/cman/init.d/cman.in +++ b/cman/init.d/cman.in @@ -92,12 +92,11 @@ fi # are found in @NOTIFYDDIR@ [ -z "$CMAN_NOTIFYD_START" ] && CMAN_NOTIFYD_START=conditional -# FENCED_START_TIMEOUT -- amount of time to wait for starting fenced -# before giving up. If FENCED_START_TIMEOUT is positive, then we will -# wait FENCED_START_TIMEOUT seconds before giving up and failing when -# fenced does not start. If FENCED_START_TIMEOUT is zero, then -# wait indefinately for fenced to start. -[ -z "$FENCED_START_TIMEOUT" ] && FENCED_START_TIMEOUT=300 +# FENCE_JOIN_TIMEOUT -- seconds to wait for fence domain join to +# complete. If the join hasn't completed in this time, fence_tool join +# exits with an error, and this script exits with an error. To wait +# indefinitely set the value to -1. +[ -z "$FENCE_JOIN_TIMEOUT" ] && FENCE_JOIN_TIMEOUT=20 # NET_RMEM_DEFAULT -- minimum value for rmem_default. If this is set # higher elsewhere it will not be reduced here. @@ -299,11 +298,11 @@ start_fence() @SBINDIR@/cman_tool status | grep Flags | grep 2node > /dev/null 2>&1 if [ $? -ne 0 ] then - errmsg=$( @SBINDIR@/fence_tool -w -t $FENCED_START_TIMEOUT join \ + errmsg=$( @SBINDIR@/fence_tool join -w $FENCE_JOIN_TIMEOUT \ > /dev/null 2>&1 ) || return 1 else - errmsg=$( @SBINDIR@/fence_tool -w -t $FENCED_START_TIMEOUT \ - -m $FENCED_MEMBER_DELAY join \ + errmsg=$( @SBINDIR@/fence_tool join -w $FENCE_JOIN_TIMEOUT \ + -m $FENCED_MEMBER_DELAY join \ > /dev/null 2>&1 ) || return 1 fi return 0 @@ -590,9 +589,8 @@ stop_fence() { if pidof fenced > /dev/null 2>&1 then - @SBINDIR@/fence_tool -w leave > /dev/null 2>&1 + @SBINDIR@/fence_tool leave -w 10 > /dev/null 2>&1 rtrn=$? - sleep 1 # A bit of time for fenced to exit return $rtrn fi return 0 # all ok diff --git a/fence/fence_tool/fence_tool.c b/fence/fence_tool/fence_tool.c index 6e37444..45d3ff2 100644 --- a/fence/fence_tool/fence_tool.c +++ b/fence/fence_tool/fence_tool.c @@ -26,23 +26,28 @@ #define OP_LIST 3 #define OP_DUMP 4 -#define DEFAULT_WAIT_TIMEOUT 300 /* five minutes */ - #define MAX_NODES 128 int all_nodeids[MAX_NODES]; int all_nodeids_count; +cman_handle_t ch; cman_node_t cman_nodes[MAX_NODES]; int cman_nodes_count; struct fenced_node nodes[MAX_NODES]; char *prog_name; int operation; -int ls_all_nodes = 0; -int inquorate_fail = 0; -int wait_join = 0; /* default: don't wait for join */ -int wait_leave = 0; /* default: don't wait for leave */ -int wait_members = 0; /* default: don't wait for members */ -int wait_timeout = DEFAULT_WAIT_TIMEOUT; + +#define DEFAULT_RETRY_CMAN 0 /* fail immediately if we can't connect to cman */ +#define DEFAULT_DELAY_QUORUM 0 +#define DEFAULT_DELAY_MEMBERS 0 +#define DEFAULT_WAIT_JOINLEAVE 0 + +int opt_all_nodes = 0; +int opt_retry_cman = DEFAULT_RETRY_CMAN; +int opt_delay_quorum = DEFAULT_DELAY_QUORUM; +int opt_delay_members = DEFAULT_DELAY_MEMBERS; +int opt_wait_joinleave = DEFAULT_WAIT_JOINLEAVE; + #define die(fmt, args...) \ do { \ @@ -70,19 +75,30 @@ static int do_write(int fd, void *buf, size_t count) return 0; } -static int get_int_arg(char argopt, char *arg) +#define LOCKFILE_NAME "/var/run/fenced.pid" + +static void check_fenced_running(void) { - char *tmp; - int val; - - val = strtol(arg, &tmp, 10); - if (tmp == arg || tmp != arg + strlen(arg)) - die("argument to %c (%s) is not an integer", argopt, arg); - - if (val < 0) - die("argument to %c cannot be negative", argopt); - - return val; + struct flock lock; + int fd, rv; + + fd = open(LOCKFILE_NAME, O_RDONLY); + if (fd < 0) + die("fenced not running, no lockfile"); + + lock.l_type = F_RDLCK; + lock.l_start = 0; + lock.l_whence= SEEK_SET; + lock.l_len = 0; + + rv = fcntl(fd, F_GETLK, &lock); + if (rv < 0) + die("fenced not running, get lockfile"); + + if (lock.l_type == F_UNLCK) + die("fenced not running, unlocked lockfile"); + + close(fd); } static int check_gfs(void) @@ -102,7 +118,7 @@ static int check_gfs(void) if (sscanf(line, "%s %s %s", device, path, type) != 3) continue; if (!strcmp(type, "gfs") || !strcmp(type, "gfs2")) { - printf("found %s file system mounted from %s on %s\n", + fprintf(stderr, "found %s file system mounted from %s on %s\n", type, device, path); count++; } @@ -131,7 +147,7 @@ static int check_controlled_dir(char *path) continue; #endif - printf("found dlm lockspace %s/%s\n", path, de->d_name); + fprintf(stderr, "found dlm lockspace %s/%s\n", path, de->d_name); count++; } @@ -177,6 +193,9 @@ static void wait_domain(int joining) int in, tries = 0; while (1) { + if (joining) + check_fenced_running(); + in = we_are_in_fence_domain(); if (joining && in) @@ -185,19 +204,25 @@ static void wait_domain(int joining) if (!joining && !in) break; - if (tries++ >= wait_timeout) - goto fail; + tries++; - if (!(tries % 5)) - printf("Waiting for fenced to %s the fence group.\n", - joining ? "join" : "leave"); + if (opt_wait_joinleave < 0) + goto retry_domain; + + if (!opt_wait_joinleave || tries >= opt_wait_joinleave) { + fprintf(stderr, "%s: %s not complete\n", + prog_name, joining ? "join" : "leave"); + break; + } + retry_domain: + if (!(tries % 10)) + fprintf(stderr, "%s: waiting for fenced to %s the fence group.\n", + prog_name, joining ? "join" : "leave"); sleep(1); } return; - fail: - printf("Error %s the fence group.\n", joining ? "joining" : "leaving"); } static void read_ccs_nodeids(int cd) @@ -223,7 +248,7 @@ static void read_ccs_nodeids(int cd) } } -static int all_nodeids_are_members(cman_handle_t ch) +static int all_nodeids_are_members(void) { int i, j, rv, found; @@ -231,10 +256,8 @@ static int all_nodeids_are_members(cman_handle_t ch) cman_nodes_count = 0; rv = cman_get_nodes(ch, MAX_NODES, &cman_nodes_count, cman_nodes); - if (rv < 0) { - printf("cman_get_nodes error %d %d\n", rv, errno); - return 0; - } + if (rv < 0) + return -1; for (i = 0; i < all_nodeids_count; i++) { found = 0; @@ -253,30 +276,25 @@ static int all_nodeids_are_members(cman_handle_t ch) return 1; } -static void wait_cman(void) +static int connect_cman(void) { - cman_handle_t ch; - int try_init = 0, try_active = 0, try_quorate = 0; - int try_ccs = 0, try_members = 0; - int rv, cd; + int rv, tries = 0; while (1) { ch = cman_init(NULL); if (ch) break; - if (inquorate_fail) - goto fail; - - if (try_init++ >= wait_timeout) { - printf("%s: timed out waiting for cman init\n", - prog_name); - goto fail; - } + tries++; - if (!(try_init % 10)) - printf("%s: waiting for cman to start\n", prog_name); + if (opt_retry_cman < 0) + goto retry_init; + if (!opt_retry_cman || tries >= opt_retry_cman) + return -1; + retry_init: + if (!(tries % 10)) + fprintf(stderr, "%s: retrying cman connection\n", prog_name); sleep(1); } @@ -285,97 +303,140 @@ static void wait_cman(void) if (rv) break; - if (inquorate_fail) - goto fail; + tries++; - if (try_active++ >= wait_timeout) { - printf("%s: timed out waiting for cman active\n", - prog_name); - goto fail; - } + if (opt_retry_cman < 0) + goto retry_active; - if (!(try_active % 10)) - printf("%s: waiting for cman active\n", prog_name); + if (!opt_retry_cman || tries >= opt_retry_cman) { + cman_finish(ch); + return -1; + } + retry_active: + if (!(tries % 10)) + fprintf(stderr, "%s: retrying cman active check\n", prog_name); sleep(1); } + return 0; +} + +static void delay_quorum(void) +{ + int rv, tries = 0; + while (1) { rv = cman_is_quorate(ch); if (rv) break; - if (inquorate_fail) - goto fail; - - if (try_quorate++ >= wait_timeout) { - printf("%s: timed out waiting for cman quorum\n", - prog_name); - goto fail; + rv = cman_is_active(ch); + if (!rv) { + cman_finish(ch); + die("lost cman connection"); } + + tries++; + + if (opt_delay_quorum < 0) + goto retry_quorum; - if (!(try_quorate % 10)) - printf("%s: waiting for cman quorum\n", prog_name); + if (!opt_delay_quorum || tries >= opt_delay_quorum) { + fprintf(stderr, "%s: continuing without quorum\n", prog_name); + break; + } + retry_quorum: + if (!(tries % 10)) + fprintf(stderr, "%s: delaying for quorum\n", prog_name); sleep(1); } - while (1) { - cd = ccs_connect(); - if (cd > 0) - break; - - if (try_ccs++ >= wait_timeout) { - printf("%s: timed out waiting for ccs connect\n", - prog_name); - goto fail; - } + return; +} - if (!(try_ccs % 10)) - printf("%s: waiting for ccs connect\n", prog_name); +static void delay_members(void) +{ + int rv, tries = 0; + int cd; - sleep(1); + cd = ccs_connect(); + if (cd < 0) { + cman_finish(ch); + die("lost cman/ccs connection"); } - if (!wait_members) - goto out; read_ccs_nodeids(cd); while (1) { - rv = all_nodeids_are_members(ch); + rv = all_nodeids_are_members(); + if (rv < 0) { + ccs_disconnect(cd); + cman_finish(ch); + die("lost cman connection"); + } if (rv) break; - if (try_members++ >= wait_members) + tries++; + + if (opt_delay_members < 0) + goto retry_members; + + if (!opt_delay_members || tries > opt_delay_members) { + fprintf(stderr, "%s: continuing without all members\n", prog_name); break; + } + retry_members: + if (!(tries % 10)) + fprintf(stderr, "%s: delaying for members\n", prog_name); - if (!(try_members % 10)) - printf("%s: waiting for all %d nodes to be members\n", - prog_name, all_nodeids_count); sleep(1); } - out: ccs_disconnect(cd); - cman_finish(ch); return; - - fail: - if (ch) - cman_finish(ch); - exit(EXIT_FAILURE); } static void do_join(int argc, char *argv[]) { - int rv; - - wait_cman(); + int rv, tries = 0; - rv = fenced_join(); + rv = connect_cman(); if (rv < 0) - die("can't communicate with fenced"); + die("can't connect to cman"); + + /* if delay_quorum() or delay_members() fail on any cman/ccs + connection or operation, they call cman_finish() and exit + with failure */ + + if (opt_delay_quorum) + delay_quorum(); + + if (opt_delay_members) + delay_members(); + + cman_finish(ch); + + /* This loop deals with the case where fenced is slow enough starting + up that fenced_join fails. Do we also want to add a delay here to + deal with the case where fenced is so slow starting up that it hasn't + locked its lockfile yet, causing check_fenced_running to fail? */ - if (wait_join) + while (1) { + rv = fenced_join(); + if (!rv) + break; + + check_fenced_running(); + + tries++; + if (!(tries % 10)) + fprintf(stderr, "%s: retrying join\n", prog_name); + sleep(1); + } + + if (opt_wait_joinleave) wait_domain(1); exit(EXIT_SUCCESS); @@ -389,9 +450,9 @@ static void do_leave(void) rv = fenced_leave(); if (rv < 0) - die("can't communicate with fenced"); + die("leave: can't communicate with fenced"); - if (wait_leave) + if (opt_wait_joinleave) wait_domain(0); exit(EXIT_SUCCESS); @@ -404,7 +465,7 @@ static void do_dump(void) rv = fenced_dump_debug(buf); if (rv < 0) - die("can't communicate with fenced"); + die("dump: can't communicate with fenced"); do_write(STDOUT_FILENO, buf, strlen(buf)); @@ -500,7 +561,7 @@ static int do_list(void) } printf("\n"); - if (!ls_all_nodes) { + if (!opt_all_nodes) { printf("\n"); exit(EXIT_SUCCESS); } @@ -539,27 +600,40 @@ static void print_usage(void) { printf("Usage:\n"); printf("\n"); - printf("%s [options]\n", prog_name); + printf("%s [options]\n", prog_name); printf("\n"); printf("Actions:\n"); + printf(" ls List nodes status\n"); printf(" join Join the default fence domain\n"); printf(" leave Leave default fence domain\n"); - printf(" ls List nodes status\n"); printf(" dump Dump debug buffer from fenced\n"); printf("\n"); printf("Options:\n"); printf(" -n Show all node information in ls\n"); + + printf(" -t Retry cman connection for .\n"); + printf(" Default %d. 0 no retry, -1 indefinite retry.\n", + DEFAULT_RETRY_CMAN); + + printf(" -q Delay join up to for the cluster to have quorum.\n"); + printf(" Default %d. 0 no delay, -1 indefinite delay.\n", + DEFAULT_DELAY_QUORUM); + printf(" -m Delay join up to for all nodes in cluster.conf\n"); - printf(" to be cluster members\n"); - printf(" -w Wait for join or leave to complete\n"); - printf(" -t Maximum time in seconds to wait (default %d)\n", DEFAULT_WAIT_TIMEOUT); - printf(" -Q Fail if cluster is not quorate, don't wait\n"); + printf(" to be cluster members.\n"); + printf(" Default %d. 0 no delay, -1 indefinite delay\n", + DEFAULT_DELAY_MEMBERS); + + printf(" -w Wait up to for join or leave result.\n"); + printf(" Default %d. 0 no wait, -1 indefinite wait.\n", + DEFAULT_WAIT_JOINLEAVE); + printf(" -V Print program version information, then exit\n"); printf(" -h Print this help, then exit\n"); printf("\n"); } -#define OPTION_STRING "Vht:wQm:n" +#define OPTION_STRING "nt:q:m:w:Vh" static void decode_arguments(int argc, char *argv[]) { @@ -571,37 +645,36 @@ static void decode_arguments(int argc, char *argv[]) switch (optchar) { - case 'V': - printf("fence_tool %s (built %s %s)\n", - RELEASE_VERSION, __DATE__, __TIME__); - printf("%s\n", REDHAT_COPYRIGHT); - exit(EXIT_SUCCESS); + case 'n': + opt_all_nodes = 1; break; - case 'n': - ls_all_nodes = 1; + case 't': + opt_retry_cman = atoi(optarg); break; - case 'h': - print_usage(); - exit(EXIT_SUCCESS); + case 'q': + opt_delay_quorum = atoi(optarg); break; - case 'Q': - inquorate_fail = 1; + case 'm': + opt_delay_members = atoi(optarg); break; case 'w': - wait_join = 1; - wait_leave = 1; + opt_wait_joinleave = atoi(optarg); break; - case 'm': - wait_members = atoi(optarg); + case 'V': + printf("fence_tool %s (built %s %s)\n", + RELEASE_VERSION, __DATE__, __TIME__); + printf("%s\n", REDHAT_COPYRIGHT); + exit(EXIT_SUCCESS); break; - case 't': - wait_timeout = get_int_arg(optchar, optarg); + case 'h': + print_usage(); + exit(EXIT_SUCCESS); break; case ':': @@ -647,12 +720,16 @@ int main(int argc, char *argv[]) switch (operation) { case OP_JOIN: do_join(argc, argv); + break; case OP_LEAVE: do_leave(); + break; case OP_DUMP: do_dump(); + break; case OP_LIST: do_list(); + break; } return EXIT_FAILURE;