From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 3717 invoked by alias); 17 Mar 2009 20:02:36 -0000 Received: (qmail 3390 invoked by alias); 17 Mar 2009 20:02:36 -0000 X-SWARE-Spam-Status: No, hits=-0.7 required=5.0 tests=AWL,BAYES_00,J_CHICKENPOX_33,J_CHICKENPOX_43,J_CHICKENPOX_52,J_CHICKENPOX_63,SPF_HELO_PASS X-Spam-Status: No, hits=-0.7 required=5.0 tests=AWL,BAYES_00,J_CHICKENPOX_33,J_CHICKENPOX_43,J_CHICKENPOX_52,J_CHICKENPOX_63,SPF_HELO_PASS X-Spam-Check-By: sourceware.org X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on bastion.fedora.phx.redhat.com Subject: fence: master - fence_tool, init.d/cman: fix wait/retry options To: cluster-cvs-relay@redhat.com X-Project: Cluster Project X-Git-Module: fence.git X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: d3b33ac0f2afa8c44dc3c7fd5dee6a4f769eb409 X-Git-Newrev: fe1dacafe59f2512ca5eac9069951c285013dbfa From: David Teigland Message-Id: <20090317200142.663B7120150@lists.fedorahosted.org> Date: Tue, 17 Mar 2009 20:02:00 -0000 X-Scanned-By: MIMEDefang 2.58 on 172.16.52.254 Mailing-List: contact cluster-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cluster-cvs-owner@sourceware.org X-SW-Source: 2009-q1/txt/msg00808.txt.bz2 Gitweb: http://git.fedorahosted.org/git/fence.git?p=fence.git;a=commitdiff;h=fe1dacafe59f2512ca5eac9069951c285013dbfa Commit: fe1dacafe59f2512ca5eac9069951c285013dbfa Parent: d3b33ac0f2afa8c44dc3c7fd5dee6a4f769eb409 Author: David Teigland AuthorDate: Tue Mar 17 14:57:58 2009 -0500 Committer: David Teigland CommitterDate: Tue Mar 17 14:57:58 2009 -0500 fence_tool, init.d/cman: fix wait/retry options Bring some sanity to the fence_tool ad hoc wait/retry options, which are used by init.d/cman. At a high level we want: . fence_tool join to fail right away, with an error, if cman or fenced fail or aren't running . fence_tool join to exit with 0 if the join succeeds and with 1 if it fails . do these things properly even when fenced is slow starting up, or in processing the join Signed-off-by: David Teigland --- fence/fence_tool/fence_tool.c | 317 +++++++++++++++++++++++++---------------- 1 files changed, 195 insertions(+), 122 deletions(-) diff --git a/fence/fence_tool/fence_tool.c b/fence/fence_tool/fence_tool.c index 8ee5544..7a900c1 100644 --- a/fence/fence_tool/fence_tool.c +++ b/fence/fence_tool/fence_tool.c @@ -27,23 +27,27 @@ #define OP_LIST 3 #define OP_DUMP 4 -#define DEFAULT_WAIT_TIMEOUT 300 /* five minutes */ - #define MAX_NODES 128 int all_nodeids[MAX_NODES]; int all_nodeids_count; +static quorum_handle_t qh; static uint32_t quorum_nodes[MAX_NODES]; static int quorum_node_count; struct fenced_node nodes[MAX_NODES]; char *prog_name; int operation; -int ls_all_nodes = 0; -int inquorate_fail = 0; -int wait_join = 0; /* default: don't wait for join */ -int wait_leave = 0; /* default: don't wait for leave */ -int wait_members = 0; /* default: don't wait for members */ -int wait_timeout = DEFAULT_WAIT_TIMEOUT; + +#define DEFAULT_RETRY_QUORUM 0 /* fail immediately if we can't connect to quorum */ +#define DEFAULT_DELAY_QUORUM 0 +#define DEFAULT_DELAY_MEMBERS 0 +#define DEFAULT_WAIT_JOINLEAVE 0 + +int opt_all_nodes = 0; +int opt_retry_quorum = DEFAULT_RETRY_QUORUM; +int opt_delay_quorum = DEFAULT_DELAY_QUORUM; +int opt_delay_members = DEFAULT_DELAY_MEMBERS; +int opt_wait_joinleave = DEFAULT_WAIT_JOINLEAVE; #define die(fmt, args...) \ do { \ @@ -71,19 +75,30 @@ static int do_write(int fd, void *buf, size_t count) return 0; } -static int get_int_arg(char argopt, char *arg) +#define LOCKFILE_NAME "/var/run/fenced.pid" + +static void check_fenced_running(void) { - char *tmp; - int val; - - val = strtol(arg, &tmp, 10); - if (tmp == arg || tmp != arg + strlen(arg)) - die("argument to %c (%s) is not an integer", argopt, arg); - - if (val < 0) - die("argument to %c cannot be negative", argopt); - - return val; + struct flock lock; + int fd, rv; + + fd = open(LOCKFILE_NAME, O_RDONLY); + if (fd < 0) + die("fenced not running, no lockfile"); + + lock.l_type = F_RDLCK; + lock.l_start = 0; + lock.l_whence= SEEK_SET; + lock.l_len = 0; + + rv = fcntl(fd, F_GETLK, &lock); + if (rv < 0) + die("fenced not running, get lockfile"); + + if (lock.l_type == F_UNLCK) + die("fenced not running, unlocked lockfile"); + + close(fd); } static int check_gfs(void) @@ -103,7 +118,7 @@ static int check_gfs(void) if (sscanf(line, "%s %s %s", device, path, type) != 3) continue; if (!strcmp(type, "gfs") || !strcmp(type, "gfs2")) { - printf("found %s file system mounted from %s on %s\n", + fprintf(stderr, "found %s file system mounted from %s on %s\n", type, device, path); count++; } @@ -132,7 +147,7 @@ static int check_controlled_dir(char *path) continue; #endif - printf("found dlm lockspace %s/%s\n", path, de->d_name); + fprintf(stderr, "found dlm lockspace %s/%s\n", path, de->d_name); count++; } @@ -178,6 +193,9 @@ static void wait_domain(int joining) int in, tries = 0; while (1) { + if (joining) + check_fenced_running(); + in = we_are_in_fence_domain(); if (joining && in) @@ -186,19 +204,25 @@ static void wait_domain(int joining) if (!joining && !in) break; - if (tries++ >= wait_timeout) - goto fail; + tries++; - if (!(tries % 5)) - printf("Waiting for fenced to %s the fence group.\n", - joining ? "join" : "leave"); + if (opt_wait_joinleave < 0) + goto retry_domain; + + if (!opt_wait_joinleave || tries >= opt_wait_joinleave) { + fprintf(stderr, "%s: %s not complete\n", + prog_name, joining ? "join" : "leave"); + break; + } + retry_domain: + if (!(tries % 10)) + fprintf(stderr, "%s: waiting for fenced to %s the fence group.\n", + prog_name, joining ? "join" : "leave"); sleep(1); } return; - fail: - printf("Error %s the fence group.\n", joining ? "joining" : "leaving"); } static void read_ccs_nodeids(int cd) @@ -237,7 +261,7 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate, quorum_nodes[quorum_node_count++] = node_list[i]; } -static int all_nodeids_are_members(quorum_handle_t qh) +static int all_nodeids_are_members(void) { cs_error_t err; int i, j, found; @@ -281,116 +305,149 @@ static quorum_callbacks_t quorum_callbacks = .quorum_notify_fn = quorum_callback, }; -static void wait_quorum(void) +static int connect_quorum(void) { - quorum_handle_t qh; cs_error_t err; - int try_init = 0, try_quorate = 0; - int try_ccs = 0, try_members = 0; - int rv, cd, quorate; + int tries = 0; while (1) { err = quorum_initialize(&qh, &quorum_callbacks); if (err == CS_OK) break; - if (inquorate_fail) - goto fail; - - if (try_init++ >= wait_timeout) { - printf("%s: timed out waiting for quorum init\n", - prog_name); - goto fail; - } + tries++; - if (!(try_init % 10)) - printf("%s: waiting for quorum init\n", prog_name); + if (opt_retry_quorum < 0) + goto retry_init; + if (!opt_retry_quorum || tries >= opt_retry_quorum) + return -1; + retry_init: + if (!(tries % 10)) + fprintf(stderr, "%s: retrying quorum connection\n", prog_name); sleep(1); } + return 0; +} + +static void delay_quorum(void) +{ + cs_error_t err; + int tries = 0; + int quorate = 0; + while (1) { err = quorum_getquorate(qh, &quorate); - if (err != CS_OK) - goto fail; + if (err != CS_OK) { + quorum_finalize(qh); + die("lost quorum connection"); + } if (quorate) break; - if (inquorate_fail) - goto fail; + tries++; - if (try_quorate++ >= wait_timeout) { - printf("%s: timed out waiting for quorum\n", - prog_name); - goto fail; - } + if (opt_delay_quorum < 0) + goto retry_quorum; - if (!(try_quorate % 10)) - printf("%s: waiting for quorum\n", prog_name); + if (!opt_delay_quorum || tries >= opt_delay_quorum) { + fprintf(stderr, "%s: continuing without quorum\n", prog_name); + break; + } + retry_quorum: + if (!(tries % 10)) + fprintf(stderr, "%s: delaying for quorum\n", prog_name); sleep(1); } - while (1) { - cd = ccs_connect(); - if (cd > 0) - break; - - if (try_ccs++ >= wait_timeout) { - printf("%s: timed out waiting for ccs connect\n", - prog_name); - goto fail; - } + return; +} - if (!(try_ccs % 10)) - printf("%s: waiting for ccs connect\n", prog_name); +static void delay_members(void) +{ + int rv, tries = 0; + int cd; - sleep(1); + cd = ccs_connect(); + if (cd < 0) { + quorum_finalize(qh); + die("failed ccs connection"); } - if (!wait_members) - goto out; read_ccs_nodeids(cd); while (1) { - rv = all_nodeids_are_members(qh); - if (rv < 0) - goto fail; - if (rv > 0) + rv = all_nodeids_are_members(); + if (rv < 0) { + ccs_disconnect(cd); + quorum_finalize(qh); + die("lost quorum connection"); + } + if (rv) break; - if (try_members++ >= wait_members) + tries++; + + if (opt_delay_members < 0) + goto retry_members; + + if (!opt_delay_members || tries > opt_delay_members) { + fprintf(stderr, "%s: continuing without all members\n", prog_name); break; + } + retry_members: + if (!(tries % 10)) + fprintf(stderr, "%s: delaying for members\n", prog_name); - if (!(try_members % 10)) - printf("%s: waiting for all %d nodes to be members\n", - prog_name, all_nodeids_count); sleep(1); } - out: ccs_disconnect(cd); - quorum_finalize(qh); return; - - fail: - if (qh) - quorum_finalize(qh); - exit(EXIT_FAILURE); } static void do_join(int argc, char *argv[]) { - int rv; - - wait_quorum(); + int rv, tries = 0; - rv = fenced_join(); + rv = connect_quorum(); if (rv < 0) - die("can't communicate with fenced"); + die("can't connect to quorum"); + + /* if delay_quorum() or delay_members() fail on any quorum/ccs + connection or operation, they call quorum_finalize() and exit + with failure */ + + if (opt_delay_quorum) + delay_quorum(); + + if (opt_delay_members) + delay_members(); + + quorum_finalize(qh); + + /* This loop deals with the case where fenced is slow enough starting + up that fenced_join fails. Do we also want to add a delay here to + deal with the case where fenced is so slow starting up that it hasn't + locked its lockfile yet, causing check_fenced_running to fail? */ + + while (1) { + rv = fenced_join(); + if (!rv) + break; + + check_fenced_running(); + + tries++; + if (!(tries % 10)) + fprintf(stderr, "%s: retrying join\n", prog_name); + sleep(1); + } - if (wait_join) + if (opt_wait_joinleave) wait_domain(1); exit(EXIT_SUCCESS); @@ -404,9 +461,9 @@ static void do_leave(void) rv = fenced_leave(); if (rv < 0) - die("can't communicate with fenced"); + die("leave: can't communicate with fenced"); - if (wait_leave) + if (opt_wait_joinleave) wait_domain(0); exit(EXIT_SUCCESS); @@ -419,7 +476,7 @@ static void do_dump(void) rv = fenced_dump_debug(buf); if (rv < 0) - die("can't communicate with fenced"); + die("dump: can't communicate with fenced"); do_write(STDOUT_FILENO, buf, strlen(buf)); @@ -515,7 +572,7 @@ static int do_list(void) } printf("\n"); - if (!ls_all_nodes) { + if (!opt_all_nodes) { printf("\n"); exit(EXIT_SUCCESS); } @@ -554,27 +611,40 @@ static void print_usage(void) { printf("Usage:\n"); printf("\n"); - printf("%s [options]\n", prog_name); + printf("%s [options]\n", prog_name); printf("\n"); printf("Actions:\n"); + printf(" ls List nodes status\n"); printf(" join Join the default fence domain\n"); printf(" leave Leave default fence domain\n"); - printf(" ls List nodes status\n"); - printf(" dump Dump debug buffer from fenced\n"); + printf(" dump Dump debug buffer from fenced\n"); printf("\n"); printf("Options:\n"); printf(" -n Show all node information in ls\n"); + + printf(" -t Retry quorum connection for .\n"); + printf(" Default %d. 0 no retry, -1 indefinite retry.\n", + DEFAULT_RETRY_QUORUM); + + printf(" -q Delay join up to for the cluster to have quorum.\n"); + printf(" Default %d. 0 no delay, -1 indefinite delay.\n", + DEFAULT_DELAY_QUORUM); + printf(" -m Delay join up to for all nodes in cluster.conf\n"); - printf(" to be cluster members\n"); - printf(" -w Wait for join or leave to complete\n"); - printf(" -t Maximum time in seconds to wait (default %d)\n", DEFAULT_WAIT_TIMEOUT); - printf(" -Q Fail if cluster is not quorate, don't wait\n"); + printf(" to be cluster members.\n"); + printf(" Default %d. 0 no delay, -1 indefinite delay\n", + DEFAULT_DELAY_MEMBERS); + + printf(" -w Wait up to for join or leave result.\n"); + printf(" Default %d. 0 no wait, -1 indefinite wait.\n", + DEFAULT_WAIT_JOINLEAVE); + printf(" -V Print program version information, then exit\n"); printf(" -h Print this help, then exit\n"); printf("\n"); } -#define OPTION_STRING "Vht:wQm:n" +#define OPTION_STRING "nt:q:m:w:Vh" static void decode_arguments(int argc, char *argv[]) { @@ -586,37 +656,36 @@ static void decode_arguments(int argc, char *argv[]) switch (optchar) { - case 'V': - printf("fence_tool %s (built %s %s)\n", - RELEASE_VERSION, __DATE__, __TIME__); - printf("%s\n", REDHAT_COPYRIGHT); - exit(EXIT_SUCCESS); + case 'n': + opt_all_nodes = 1; break; - case 'n': - ls_all_nodes = 1; + case 't': + opt_retry_quorum = atoi(optarg); break; - case 'h': - print_usage(); - exit(EXIT_SUCCESS); + case 'q': + opt_delay_quorum = atoi(optarg); break; - case 'Q': - inquorate_fail = 1; + case 'm': + opt_delay_members = atoi(optarg); break; case 'w': - wait_join = 1; - wait_leave = 1; + opt_wait_joinleave = atoi(optarg); break; - case 'm': - wait_members = atoi(optarg); + case 'V': + printf("fence_tool %s (built %s %s)\n", + RELEASE_VERSION, __DATE__, __TIME__); + printf("%s\n", REDHAT_COPYRIGHT); + exit(EXIT_SUCCESS); break; - case 't': - wait_timeout = get_int_arg(optchar, optarg); + case 'h': + print_usage(); + exit(EXIT_SUCCESS); break; case ':': @@ -662,12 +731,16 @@ int main(int argc, char *argv[]) switch (operation) { case OP_JOIN: do_join(argc, argv); + break; case OP_LEAVE: do_leave(); + break; case OP_DUMP: do_dump(); + break; case OP_LIST: do_list(); + break; } return EXIT_FAILURE;