public inbox for cluster-cvs@sourceware.org help / color / mirror / Atom feed
From: David Teigland <teigland@fedoraproject.org> To: cluster-cvs-relay@redhat.com Subject: dlm: master - dlm_controld: replace cman with votequorum and cfg Date: Tue, 10 Feb 2009 18:21:00 -0000 [thread overview] Message-ID: <20090210182027.3F48512011A@lists.fedorahosted.org> (raw) Gitweb: http://git.fedorahosted.org/git/dlm.git?p=dlm.git;a=commitdiff;h=5045786d61b87eae5d1c3a9fe2e23e00e2da36ac Commit: 5045786d61b87eae5d1c3a9fe2e23e00e2da36ac Parent: af52b7bb26a05a8f73d1076254e178b02ed192cd Author: David Teigland <teigland@redhat.com> AuthorDate: Tue Feb 10 12:17:05 2009 -0600 Committer: David Teigland <teigland@redhat.com> CommitterDate: Tue Feb 10 12:17:05 2009 -0600 dlm_controld: replace cman with votequorum and cfg libcman calls have been replaced with libvotequorum and libcfg calls, which are corosync libs. Signed-off-by: David Teigland <teigland@redhat.com> --- group/dlm_controld/Makefile | 6 +- group/dlm_controld/config.c | 54 ++++++- group/dlm_controld/dlm_daemon.h | 4 +- group/dlm_controld/main.c | 6 + group/dlm_controld/member_cman.c | 317 +++++++++++++++++++------------------- 5 files changed, 220 insertions(+), 167 deletions(-) diff --git a/group/dlm_controld/Makefile b/group/dlm_controld/Makefile index 7e1e1c7..9e1f9e4 100644 --- a/group/dlm_controld/Makefile +++ b/group/dlm_controld/Makefile @@ -31,7 +31,7 @@ OBJS += config.o \ logging.o endif -CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${logtincdir} +CFLAGS += -I${ccsincdir} -I${logtincdir} CFLAGS += -I${dlmincdir} -I${dlmcontrolincdir} CFLAGS += -I${corosyncincdir} -I${openaisincdir} CFLAGS += -I${fencedincdir} @@ -47,14 +47,14 @@ CFLAGS += -I${incdir} LDFLAGS += -L${dlmlibdir} -ldlm LDFLAGS += -L${logtlibdir} -llogthread LDFLAGS += -L${openaislibdir} -lSaCkpt -LDFLAGS += -L${corosynclibdir} -lcpg +LDFLAGS += -L${corosynclibdir} -lcpg -lcfg -lvotequorum LDFLAGS += -L${libdir} PCMK_LDFLAGS += -lcib -lcrmcommon -lcrmcluster -ltotem_pg PCMK_LDFLAGS += `pkg-config glib-2.0 --libs` PCMK_LDFLAGS += `xml2-config --libs` -CMAN_LDFLAGS += -L${ccslibdir} -L${cmanlibdir} -lccs -lcman +CMAN_LDFLAGS += -L${ccslibdir} -lccs CMAN_LDFLAGS += -L${fencedlibdir} -lfenced CMAN_LDDEPS += ${fencedlibdir}/libfenced.a diff --git a/group/dlm_controld/config.c b/group/dlm_controld/config.c index f56eadb..384499d 100644 --- a/group/dlm_controld/config.c +++ b/group/dlm_controld/config.c @@ -101,23 +101,65 @@ static int get_weight_clusternode(char *node, char *lockspace) return weight; } +static int nodeid2name(int nodeid, char *name_out) +{ + char path[PATH_MAX]; + char *str, *name; + int i, error; + + for (i = 1; ; i++) { + str = NULL; + memset(path, 0, sizeof(path)); + sprintf(path, "/cluster/clusternodes/clusternode[%d]/@nodeid", i); + + error = ccs_get(ccs_handle, path, &str); + if (error || !str) + break; + + if (atoi(str) != nodeid) { + free(str); + continue; + } + free(str); + + name = NULL; + memset(path, 0, sizeof(path)); + sprintf(path, "/cluster/clusternodes/clusternode[%d]/@name", i); + + error = ccs_get(ccs_handle, path, &name); + if (error || !name) { + log_error("node name query failed for num %d nodeid %d", + i, nodeid); + break; + } + + strncpy(name_out, name, PATH_MAX); + free(name); + return 0; + } + + return -1; +} + int get_weight(int nodeid, char *lockspace) { - char *node; - int w; + char nodename[PATH_MAX]; + int w, rv; + + memset(nodename, 0, sizeof(nodename)); - node = nodeid2name(nodeid); - if (!node) { + rv = nodeid2name(nodeid, nodename); + if (rv < 0) { log_error("no name for nodeid %d", nodeid); w = 1; goto out; } - w = get_weight_lockspace(node, lockspace); + w = get_weight_lockspace(nodename, lockspace); if (w >= 0) goto out; - w = get_weight_clusternode(node, lockspace); + w = get_weight_clusternode(nodename, lockspace); if (w >= 0) goto out; diff --git a/group/dlm_controld/dlm_daemon.h b/group/dlm_controld/dlm_daemon.h index 8c751a7..5d4ce37 100644 --- a/group/dlm_controld/dlm_daemon.h +++ b/group/dlm_controld/dlm_daemon.h @@ -270,7 +270,9 @@ void close_cluster(void); void process_cluster(int ci); void update_cluster(void); int is_cluster_member(int nodeid); -char *nodeid2name(int nodeid); +int setup_cluster_cfg(void); +void close_cluster_cfg(void); +void process_cluster_cfg(int ci); void kick_node_from_cluster(int nodeid); int fence_node_time(int nodeid, uint64_t *last_fenced_time); int fence_in_progress(int *count); diff --git a/group/dlm_controld/main.c b/group/dlm_controld/main.c index 12fb04e..a7470c7 100644 --- a/group/dlm_controld/main.c +++ b/group/dlm_controld/main.c @@ -853,6 +853,11 @@ static void loop(void) goto out; client_add(rv, process_listener, NULL); + rv = setup_cluster_cfg(); + if (rv < 0) + goto out; + client_add(rv, process_cluster_cfg, cluster_dead); + rv = setup_cluster(); if (rv < 0) goto out; @@ -963,6 +968,7 @@ static void loop(void) close_logging(); close_ccs(); close_cluster(); + close_cluster_cfg(); if (!list_empty(&lockspaces)) log_error("lockspaces abandoned"); diff --git a/group/dlm_controld/member_cman.c b/group/dlm_controld/member_cman.c index 19403f5..96cacf2 100644 --- a/group/dlm_controld/member_cman.c +++ b/group/dlm_controld/member_cman.c @@ -1,34 +1,37 @@ #include "dlm_daemon.h" #include "config.h" -#include <libcman.h> +#include <corosync/corotypes.h> +#include <corosync/cfg.h> +#include <corosync/votequorum.h> #include "libfenced.h" -static cman_handle_t ch; -static cman_handle_t ch_admin; -static cman_node_t old_nodes[MAX_NODES]; -static int old_node_count; -static cman_node_t cman_nodes[MAX_NODES]; -static int cman_node_count; +static corosync_cfg_handle_t ch; +static votequorum_handle_t qh; +static votequorum_node_t old_nodes[MAX_NODES]; +static int old_node_count; +static votequorum_node_t quorum_nodes[MAX_NODES]; +static int quorum_node_count; void kick_node_from_cluster(int nodeid) { if (!nodeid) { - log_error("telling cman to shut down cluster locally"); - cman_shutdown(ch_admin, CMAN_SHUTDOWN_ANYWAY); + log_error("telling corosync to shut down cluster locally"); + corosync_cfg_try_shutdown(ch, + COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE); } else { - log_error("telling cman to remove nodeid %d from cluster", + log_error("telling corosync to remove nodeid %d from cluster", nodeid); - cman_kill_node(ch_admin, nodeid); + corosync_cfg_kill_node(ch, nodeid, "dlm_controld"); } } -static int is_member(cman_node_t *node_list, int count, int nodeid) +static int is_member(votequorum_node_t *node_list, int count, int nodeid) { int i; for (i = 0; i < count; i++) { - if (node_list[i].cn_nodeid == nodeid) - return node_list[i].cn_member; + if (node_list[i].nodeid == nodeid) + return (node_list[i].state == NODESTATE_MEMBER); } return 0; } @@ -40,202 +43,202 @@ static int is_old_member(int nodeid) int is_cluster_member(int nodeid) { - return is_member(cman_nodes, cman_node_count, nodeid); + return is_member(quorum_nodes, quorum_node_count, nodeid); } -static cman_node_t *find_cman_node(int nodeid) -{ - int i; - - for (i = 0; i < cman_node_count; i++) { - if (cman_nodes[i].cn_nodeid == nodeid) - return &cman_nodes[i]; - } - return NULL; -} - -char *nodeid2name(int nodeid) +/* what's the replacement for this? */ +#if 0 +static void cman_callback(cman_handle_t h, void *private, int reason, int arg) { - cman_node_t *cn; - - cn = find_cman_node(nodeid); - if (!cn) - return NULL; - return cn->cn_name; + case CMAN_REASON_CONFIG_UPDATE: + setup_logging(); + setup_ccs(); + break; } +#endif /* add a configfs dir for cluster members that don't have one, del the configfs dir for cluster members that are now gone */ -static void statechange(void) +static void quorum_callback(votequorum_handle_t h, uint64_t context, + uint32_t quorate, uint32_t node_list_entries, + votequorum_node_t node_list[]) { - int i, j, rv; - struct cman_node_address addrs[MAX_NODE_ADDRESSES]; - int num_addrs; - struct cman_node_address *addrptr = addrs; - - cluster_quorate = cman_is_quorate(ch); - - old_node_count = cman_node_count; - memcpy(&old_nodes, &cman_nodes, sizeof(old_nodes)); - - cman_node_count = 0; - memset(&cman_nodes, 0, sizeof(cman_nodes)); - rv = cman_get_nodes(ch, MAX_NODES, &cman_node_count, cman_nodes); - if (rv < 0) { - log_debug("cman_get_nodes error %d %d", rv, errno); - return; - } + corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES]; + corosync_cfg_node_address_t *addrptr = addrs; + cs_error_t err; + int i, j, num_addrs; - /* Never allow node ID 0 to be considered a member #315711 */ - for (i = 0; i < cman_node_count; i++) { - if (cman_nodes[i].cn_nodeid == 0) { - cman_nodes[i].cn_member = 0; - break; + cluster_quorate = quorate; + + old_node_count = quorum_node_count; + memcpy(&old_nodes, &quorum_nodes, sizeof(old_nodes)); + + quorum_node_count = 0; + memset(&quorum_nodes, 0, sizeof(quorum_nodes)); + + for (i = 0; i < node_list_entries; i++) { + if (node_list[i].state == NODESTATE_MEMBER) { + memcpy(&quorum_nodes[quorum_node_count], + &node_list[i], sizeof(votequorum_node_t)); + quorum_node_count++; } } for (i = 0; i < old_node_count; i++) { - if (old_nodes[i].cn_member && - !is_cluster_member(old_nodes[i].cn_nodeid)) { + if ((old_nodes[i].state == NODESTATE_MEMBER) && + !is_cluster_member(old_nodes[i].nodeid)) { - log_debug("cman: node %d removed", - old_nodes[i].cn_nodeid); + log_debug("quorum: node %d removed", + old_nodes[i].nodeid); - del_configfs_node(old_nodes[i].cn_nodeid); + del_configfs_node(old_nodes[i].nodeid); } } - for (i = 0; i < cman_node_count; i++) { - if (cman_nodes[i].cn_member && - !is_old_member(cman_nodes[i].cn_nodeid)) { - - rv = cman_get_node_addrs(ch, cman_nodes[i].cn_nodeid, - MAX_NODE_ADDRESSES, - &num_addrs, addrs); - if (rv < 0) { - log_debug("cman_get_node_addrs failed, falling back to single-homed. "); - num_addrs = 1; - addrptr = &cman_nodes[i].cn_address; + for (i = 0; i < quorum_node_count; i++) { + if ((quorum_nodes[i].state == NODESTATE_MEMBER) && + !is_old_member(quorum_nodes[i].nodeid)) { + + log_debug("quorum: node %d added", + quorum_nodes[i].nodeid); + + err = corosync_cfg_get_node_addrs(ch, + quorum_nodes[i].nodeid, + MAX_NODE_ADDRESSES, + &num_addrs, addrs); + if (err != CS_OK) { + log_error("corosync_cfg_get_node_addrs failed " + "nodeid %d", quorum_nodes[i].nodeid); + continue; } - log_debug("cman: node %d added", - cman_nodes[i].cn_nodeid); - for (j = 0; j < num_addrs; j++) { - add_configfs_node(cman_nodes[i].cn_nodeid, - addrptr[j].cna_address, - addrptr[j].cna_addrlen, - (cman_nodes[i].cn_nodeid == + add_configfs_node(quorum_nodes[i].nodeid, + addrptr[j].address, + addrptr[j].address_length, + (quorum_nodes[i].nodeid == our_nodeid)); } } } } -static void cman_callback(cman_handle_t h, void *private, int reason, int arg) +static votequorum_callbacks_t quorum_callbacks = { - switch (reason) { - case CMAN_REASON_TRY_SHUTDOWN: - if (list_empty(&lockspaces)) - cman_replyto_shutdown(ch, 1); - else { - log_debug("no to cman shutdown"); - cman_replyto_shutdown(ch, 0); - } - break; - case CMAN_REASON_STATECHANGE: - statechange(); - break; - case CMAN_REASON_CONFIG_UPDATE: - setup_logging(); - setup_ccs(); - break; - } -} + .votequorum_notify_fn = quorum_callback, +}; void process_cluster(int ci) { - int rv; + cs_error_t err; + + err = votequorum_dispatch(qh, CS_DISPATCH_ALL); + if (err != CS_OK) + cluster_dead(0); +} + +/* Force re-read of quorum nodes */ +void update_cluster(void) +{ + cs_error_t err; - rv = cman_dispatch(ch, CMAN_DISPATCH_ALL); - if (rv == -1 && errno == EHOSTDOWN) + err = votequorum_dispatch(qh, CS_DISPATCH_ONE); + if (err != CS_OK) cluster_dead(0); } int setup_cluster(void) { - cman_node_t node; - int rv, fd; - int init = 0, active = 0; - - retry_init: - ch_admin = cman_admin_init(NULL); - if (!ch_admin) { - if (init++ < 2) { - sleep(1); - goto retry_init; - } - log_error("cman_admin_init error %d", errno); - return -ENOTCONN; - } + struct votequorum_info qinfo; + cs_error_t err; + int fd; - ch = cman_init(NULL); - if (!ch) { - log_error("cman_init error %d", errno); - return -ENOTCONN; - } + err = votequorum_initialize(&qh, &quorum_callbacks); + if (err != CS_OK) + return -1; - retry_active: - rv = cman_is_active(ch); - if (!rv) { - if (active++ < 2) { - sleep(1); - goto retry_active; - } - log_error("cman_is_active error %d", errno); - cman_finish(ch); - return -ENOTCONN; - } + err = votequorum_fd_get(qh, &fd); + if (err != CS_OK) + goto fail; - rv = cman_start_notification(ch, cman_callback); - if (rv < 0) { - log_error("cman_start_notification error %d %d", rv, errno); - cman_finish(ch); - return rv; - } + err = votequorum_getinfo(qh, 0, &qinfo); + if (err != CS_OK) + goto fail; + our_nodeid = qinfo.node_id; - fd = cman_get_fd(ch); - - /* FIXME: wait here for us to be a member of the cluster */ - memset(&node, 0, sizeof(node)); - rv = cman_get_node(ch, CMAN_NODEID_US, &node); - if (rv < 0) { - log_error("cman_get_node us error %d %d", rv, errno); - cman_stop_notification(ch); - cman_finish(ch); - fd = rv; - goto out; - } - our_nodeid = node.cn_nodeid; + err = votequorum_trackstart(qh, 0, CS_TRACK_CURRENT); + if (err != CS_OK) + goto fail; old_node_count = 0; memset(&old_nodes, 0, sizeof(old_nodes)); - cman_node_count = 0; - memset(&cman_nodes, 0, sizeof(cman_nodes)); - out: + quorum_node_count = 0; + memset(&quorum_nodes, 0, sizeof(quorum_nodes)); + return fd; + fail: + votequorum_finalize(qh); + return -1; } void close_cluster(void) { - cman_finish(ch); + votequorum_trackstop(qh); + votequorum_finalize(qh); } -/* Force re-read of cman nodes */ -void update_cluster(void) +static void shutdown_callback(corosync_cfg_handle_t h, + corosync_cfg_shutdown_flags_t flags) +{ + if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) { + if (list_empty(&lockspaces)) + corosync_cfg_replyto_shutdown(ch, + COROSYNC_CFG_SHUTDOWN_FLAG_YES); + else { + log_debug("no to corosync shutdown"); + corosync_cfg_replyto_shutdown(ch, + COROSYNC_CFG_SHUTDOWN_FLAG_NO); + } + } +} + +static corosync_cfg_callbacks_t cfg_callbacks = +{ + .corosync_cfg_shutdown_callback = shutdown_callback, + .corosync_cfg_state_track_callback = NULL, +}; + +void process_cluster_cfg(int ci) +{ + cs_error_t err; + + err = corosync_cfg_dispatch(ch, CS_DISPATCH_ALL); + if (err != CS_OK) + cluster_dead(0); +} + +int setup_cluster_cfg(void) +{ + cs_error_t err; + int fd; + + err = corosync_cfg_initialize(&ch, &cfg_callbacks); + if (err != CS_OK) + return -1; + + err = corosync_cfg_fd_get(ch, &fd); + if (err != CS_OK) { + corosync_cfg_finalize(ch); + return -1; + } + + return fd; +} + +void close_cluster_cfg(void) { - statechange(); + corosync_cfg_finalize(ch); } int fence_node_time(int nodeid, uint64_t *last_fenced_time)
reply other threads:[~2009-02-10 18:21 UTC|newest] Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20090210182027.3F48512011A@lists.fedorahosted.org \ --to=teigland@fedoraproject.org \ --cc=cluster-cvs-relay@redhat.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).