dlm: master - dlm_controld: replace cman with votequorum and cfg

dlm: master - dlm_controld: replace cman with votequorum and cfg

[David Teigland]

Gitweb:        http://git.fedorahosted.org/git/dlm.git?p=dlm.git;a=commitdiff;h=5045786d61b87eae5d1c3a9fe2e23e00e2da36ac
Commit:        5045786d61b87eae5d1c3a9fe2e23e00e2da36ac
Parent:        af52b7bb26a05a8f73d1076254e178b02ed192cd
Author:        David Teigland <teigland@redhat.com>
AuthorDate:    Tue Feb 10 12:17:05 2009 -0600
Committer:     David Teigland <teigland@redhat.com>
CommitterDate: Tue Feb 10 12:17:05 2009 -0600

dlm_controld: replace cman with votequorum and cfg

libcman calls have been replaced with libvotequorum
and libcfg calls, which are corosync libs.

Signed-off-by: David Teigland <teigland@redhat.com>
---
 group/dlm_controld/Makefile      |    6 +-
 group/dlm_controld/config.c      |   54 ++++++-
 group/dlm_controld/dlm_daemon.h  |    4 +-
 group/dlm_controld/main.c        |    6 +
 group/dlm_controld/member_cman.c |  317 +++++++++++++++++++-------------------
 5 files changed, 220 insertions(+), 167 deletions(-)

diff --git a/group/dlm_controld/Makefile b/group/dlm_controld/Makefile
index 7e1e1c7..9e1f9e4 100644
--- a/group/dlm_controld/Makefile
+++ b/group/dlm_controld/Makefile
@@ -31,7 +31,7 @@ OBJS +=	config.o \
 	logging.o
 endif
 
-CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${logtincdir} 
+CFLAGS += -I${ccsincdir} -I${logtincdir} 
 CFLAGS += -I${dlmincdir} -I${dlmcontrolincdir}
 CFLAGS += -I${corosyncincdir} -I${openaisincdir}
 CFLAGS += -I${fencedincdir}
@@ -47,14 +47,14 @@ CFLAGS += -I${incdir}
 LDFLAGS += -L${dlmlibdir} -ldlm 
 LDFLAGS += -L${logtlibdir} -llogthread
 LDFLAGS += -L${openaislibdir} -lSaCkpt
-LDFLAGS += -L${corosynclibdir} -lcpg 
+LDFLAGS += -L${corosynclibdir} -lcpg -lcfg -lvotequorum
 LDFLAGS += -L${libdir}
 
 PCMK_LDFLAGS += -lcib -lcrmcommon -lcrmcluster -ltotem_pg
 PCMK_LDFLAGS += `pkg-config glib-2.0 --libs`
 PCMK_LDFLAGS += `xml2-config --libs`
 
-CMAN_LDFLAGS += -L${ccslibdir} -L${cmanlibdir} -lccs -lcman
+CMAN_LDFLAGS += -L${ccslibdir} -lccs
 CMAN_LDFLAGS += -L${fencedlibdir} -lfenced
 CMAN_LDDEPS += ${fencedlibdir}/libfenced.a
 
diff --git a/group/dlm_controld/config.c b/group/dlm_controld/config.c
index f56eadb..384499d 100644
--- a/group/dlm_controld/config.c
+++ b/group/dlm_controld/config.c
@@ -101,23 +101,65 @@ static int get_weight_clusternode(char *node, char *lockspace)
 	return weight;
 }
 
+static int nodeid2name(int nodeid, char *name_out)
+{
+	char path[PATH_MAX];
+	char *str, *name;
+	int i, error;
+
+	for (i = 1; ; i++) {
+		str = NULL;
+		memset(path, 0, sizeof(path));
+		sprintf(path, "/cluster/clusternodes/clusternode[%d]/@nodeid", i);
+
+		error = ccs_get(ccs_handle, path, &str);
+		if (error || !str)
+			break;
+
+		if (atoi(str) != nodeid) {
+			free(str);
+			continue;
+		}
+		free(str);
+
+		name = NULL;
+		memset(path, 0, sizeof(path));
+		sprintf(path, "/cluster/clusternodes/clusternode[%d]/@name", i);
+
+		error = ccs_get(ccs_handle, path, &name);
+		if (error || !name) {
+			log_error("node name query failed for num %d nodeid %d",
+				  i, nodeid);
+			break;
+		}
+
+		strncpy(name_out, name, PATH_MAX);
+		free(name);
+		return 0;
+	}
+
+	return -1;
+}
+
 int get_weight(int nodeid, char *lockspace)
 {
-	char *node;
-	int w;
+	char nodename[PATH_MAX];
+	int w, rv;
+
+	memset(nodename, 0, sizeof(nodename));
 
-	node = nodeid2name(nodeid);
-	if (!node) {
+	rv = nodeid2name(nodeid, nodename);
+	if (rv < 0) {
 		log_error("no name for nodeid %d", nodeid);
 		w = 1;
 		goto out;
 	}
 
-	w = get_weight_lockspace(node, lockspace);
+	w = get_weight_lockspace(nodename, lockspace);
 	if (w >= 0)
 		goto out;
 
-	w = get_weight_clusternode(node, lockspace);
+	w = get_weight_clusternode(nodename, lockspace);
 	if (w >= 0)
 		goto out;
 
diff --git a/group/dlm_controld/dlm_daemon.h b/group/dlm_controld/dlm_daemon.h
index 8c751a7..5d4ce37 100644
--- a/group/dlm_controld/dlm_daemon.h
+++ b/group/dlm_controld/dlm_daemon.h
@@ -270,7 +270,9 @@ void close_cluster(void);
 void process_cluster(int ci);
 void update_cluster(void);
 int is_cluster_member(int nodeid);
-char *nodeid2name(int nodeid);
+int setup_cluster_cfg(void);
+void close_cluster_cfg(void);
+void process_cluster_cfg(int ci);
 void kick_node_from_cluster(int nodeid);
 int fence_node_time(int nodeid, uint64_t *last_fenced_time);
 int fence_in_progress(int *count);
diff --git a/group/dlm_controld/main.c b/group/dlm_controld/main.c
index 12fb04e..a7470c7 100644
--- a/group/dlm_controld/main.c
+++ b/group/dlm_controld/main.c
@@ -853,6 +853,11 @@ static void loop(void)
 		goto out;
 	client_add(rv, process_listener, NULL);
 
+	rv = setup_cluster_cfg();
+	if (rv < 0)
+		goto out;
+	client_add(rv, process_cluster_cfg, cluster_dead);
+
 	rv = setup_cluster();
 	if (rv < 0)
 		goto out;
@@ -963,6 +968,7 @@ static void loop(void)
 	close_logging();
 	close_ccs();
 	close_cluster();
+	close_cluster_cfg();
 
 	if (!list_empty(&lockspaces))
 		log_error("lockspaces abandoned");
diff --git a/group/dlm_controld/member_cman.c b/group/dlm_controld/member_cman.c
index 19403f5..96cacf2 100644
--- a/group/dlm_controld/member_cman.c
+++ b/group/dlm_controld/member_cman.c
@@ -1,34 +1,37 @@
 #include "dlm_daemon.h"
 #include "config.h"
-#include <libcman.h>
+#include <corosync/corotypes.h>
+#include <corosync/cfg.h>
+#include <corosync/votequorum.h>
 #include "libfenced.h"
 
-static cman_handle_t	ch;
-static cman_handle_t	ch_admin;
-static cman_node_t      old_nodes[MAX_NODES];
-static int              old_node_count;
-static cman_node_t      cman_nodes[MAX_NODES];
-static int              cman_node_count;
+static corosync_cfg_handle_t	ch;
+static votequorum_handle_t	qh;
+static votequorum_node_t	old_nodes[MAX_NODES];
+static int			old_node_count;
+static votequorum_node_t	quorum_nodes[MAX_NODES];
+static int			quorum_node_count;
 
 void kick_node_from_cluster(int nodeid)
 {
 	if (!nodeid) {
-		log_error("telling cman to shut down cluster locally");
-		cman_shutdown(ch_admin, CMAN_SHUTDOWN_ANYWAY);
+		log_error("telling corosync to shut down cluster locally");
+		corosync_cfg_try_shutdown(ch,
+				COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
 	} else {
-		log_error("telling cman to remove nodeid %d from cluster",
+		log_error("telling corosync to remove nodeid %d from cluster",
 			  nodeid);
-		cman_kill_node(ch_admin, nodeid);
+		corosync_cfg_kill_node(ch, nodeid, "dlm_controld");
 	}
 }
 
-static int is_member(cman_node_t *node_list, int count, int nodeid)
+static int is_member(votequorum_node_t *node_list, int count, int nodeid)
 {
 	int i;
 
 	for (i = 0; i < count; i++) {
-		if (node_list[i].cn_nodeid == nodeid)
-			return node_list[i].cn_member;
+		if (node_list[i].nodeid == nodeid)
+			return (node_list[i].state == NODESTATE_MEMBER);
 	}
 	return 0;
 }
@@ -40,202 +43,202 @@ static int is_old_member(int nodeid)
 
 int is_cluster_member(int nodeid)
 {
-	return is_member(cman_nodes, cman_node_count, nodeid);
+	return is_member(quorum_nodes, quorum_node_count, nodeid);
 }
 
-static cman_node_t *find_cman_node(int nodeid)
-{
-	int i;
-
-	for (i = 0; i < cman_node_count; i++) {
-		if (cman_nodes[i].cn_nodeid == nodeid)
-			return &cman_nodes[i];
-	}
-	return NULL;
-}
-
-char *nodeid2name(int nodeid)
+/* what's the replacement for this? */
+#if 0
+static void cman_callback(cman_handle_t h, void *private, int reason, int arg)
 {
-	cman_node_t *cn;
-
-	cn = find_cman_node(nodeid);
-	if (!cn)
-		return NULL;
-	return cn->cn_name;
+	case CMAN_REASON_CONFIG_UPDATE:
+		setup_logging();
+		setup_ccs();
+		break;
 }
+#endif
 
 /* add a configfs dir for cluster members that don't have one,
    del the configfs dir for cluster members that are now gone */
 
-static void statechange(void)
+static void quorum_callback(votequorum_handle_t h, uint64_t context,
+			    uint32_t quorate, uint32_t node_list_entries,
+			    votequorum_node_t node_list[])
 {
-	int i, j, rv;
-	struct cman_node_address addrs[MAX_NODE_ADDRESSES];
-	int num_addrs;
-	struct cman_node_address *addrptr = addrs;
-
-	cluster_quorate = cman_is_quorate(ch);
-
-	old_node_count = cman_node_count;
-	memcpy(&old_nodes, &cman_nodes, sizeof(old_nodes));
-
-	cman_node_count = 0;
-	memset(&cman_nodes, 0, sizeof(cman_nodes));
-	rv = cman_get_nodes(ch, MAX_NODES, &cman_node_count, cman_nodes);
-	if (rv < 0) {
-		log_debug("cman_get_nodes error %d %d", rv, errno);
-		return;
-	}
+	corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES];
+	corosync_cfg_node_address_t *addrptr = addrs;
+	cs_error_t err;
+	int i, j, num_addrs;
 
-	/* Never allow node ID 0 to be considered a member #315711 */
-	for (i = 0; i < cman_node_count; i++) {
-		if (cman_nodes[i].cn_nodeid == 0) {
-			cman_nodes[i].cn_member = 0;
-			break;
+	cluster_quorate = quorate;
+
+	old_node_count = quorum_node_count;
+	memcpy(&old_nodes, &quorum_nodes, sizeof(old_nodes));
+
+	quorum_node_count = 0;
+	memset(&quorum_nodes, 0, sizeof(quorum_nodes));
+
+	for (i = 0; i < node_list_entries; i++) {
+		if (node_list[i].state == NODESTATE_MEMBER) {
+			memcpy(&quorum_nodes[quorum_node_count],
+			       &node_list[i], sizeof(votequorum_node_t));
+			quorum_node_count++;
 		}
 	}
 
 	for (i = 0; i < old_node_count; i++) {
-		if (old_nodes[i].cn_member &&
-		    !is_cluster_member(old_nodes[i].cn_nodeid)) {
+		if ((old_nodes[i].state == NODESTATE_MEMBER) &&
+		    !is_cluster_member(old_nodes[i].nodeid)) {
 
-			log_debug("cman: node %d removed",
-				   old_nodes[i].cn_nodeid);
+			log_debug("quorum: node %d removed",
+				   old_nodes[i].nodeid);
 
-			del_configfs_node(old_nodes[i].cn_nodeid);
+			del_configfs_node(old_nodes[i].nodeid);
 		}
 	}
 
-	for (i = 0; i < cman_node_count; i++) {
-		if (cman_nodes[i].cn_member &&
-		    !is_old_member(cman_nodes[i].cn_nodeid)) {
-
-			rv = cman_get_node_addrs(ch, cman_nodes[i].cn_nodeid,
-						 MAX_NODE_ADDRESSES,
-						 &num_addrs, addrs);
-			if (rv < 0) {
-				log_debug("cman_get_node_addrs failed, falling back to single-homed. ");
-				num_addrs = 1;
-				addrptr = &cman_nodes[i].cn_address;
+	for (i = 0; i < quorum_node_count; i++) {
+		if ((quorum_nodes[i].state == NODESTATE_MEMBER) &&
+		    !is_old_member(quorum_nodes[i].nodeid)) {
+
+			log_debug("quorum: node %d added",
+				  quorum_nodes[i].nodeid);
+
+			err = corosync_cfg_get_node_addrs(ch,
+					quorum_nodes[i].nodeid,
+					MAX_NODE_ADDRESSES,
+					&num_addrs, addrs);
+			if (err != CS_OK) {
+				log_error("corosync_cfg_get_node_addrs failed "
+					  "nodeid %d", quorum_nodes[i].nodeid);
+				continue;
 			}
 
-			log_debug("cman: node %d added",
-				  cman_nodes[i].cn_nodeid);
-
 			for (j = 0; j < num_addrs; j++) {
-				add_configfs_node(cman_nodes[i].cn_nodeid,
-						  addrptr[j].cna_address,
-						  addrptr[j].cna_addrlen,
-						  (cman_nodes[i].cn_nodeid ==
+				add_configfs_node(quorum_nodes[i].nodeid,
+						  addrptr[j].address,
+						  addrptr[j].address_length,
+						  (quorum_nodes[i].nodeid ==
 						   our_nodeid));
 			}
 		}
 	}
 }
 
-static void cman_callback(cman_handle_t h, void *private, int reason, int arg)
+static votequorum_callbacks_t quorum_callbacks =
 {
-	switch (reason) {
-	case CMAN_REASON_TRY_SHUTDOWN:
-		if (list_empty(&lockspaces))
-			cman_replyto_shutdown(ch, 1);
-		else {
-			log_debug("no to cman shutdown");
-			cman_replyto_shutdown(ch, 0);
-		}
-		break;
-	case CMAN_REASON_STATECHANGE:
-		statechange();
-		break;
-	case CMAN_REASON_CONFIG_UPDATE:
-		setup_logging();
-		setup_ccs();
-		break;
-	}
-}
+	.votequorum_notify_fn = quorum_callback,
+};
 
 void process_cluster(int ci)
 {
-	int rv;
+	cs_error_t err;
+
+	err = votequorum_dispatch(qh, CS_DISPATCH_ALL);
+	if (err != CS_OK)
+		cluster_dead(0);
+}
+
+/* Force re-read of quorum nodes */
+void update_cluster(void)
+{
+	cs_error_t err;
 
-	rv = cman_dispatch(ch, CMAN_DISPATCH_ALL);
-	if (rv == -1 && errno == EHOSTDOWN)
+	err = votequorum_dispatch(qh, CS_DISPATCH_ONE);
+	if (err != CS_OK)
 		cluster_dead(0);
 }
 
 int setup_cluster(void)
 {
-	cman_node_t node;
-	int rv, fd;
-	int init = 0, active = 0;
-
- retry_init:
-	ch_admin = cman_admin_init(NULL);
-	if (!ch_admin) {
-		if (init++ < 2) {
-			sleep(1);
-			goto retry_init;
-		}
-		log_error("cman_admin_init error %d", errno);
-		return -ENOTCONN;
-	}
+	struct votequorum_info qinfo;
+	cs_error_t err;
+	int fd;
 
-	ch = cman_init(NULL);
-	if (!ch) {
-		log_error("cman_init error %d", errno);
-		return -ENOTCONN;
-	}
+	err = votequorum_initialize(&qh, &quorum_callbacks);
+	if (err != CS_OK)
+		return -1;
 
- retry_active:
-	rv = cman_is_active(ch);
-	if (!rv) {
-		if (active++ < 2) {
-			sleep(1);
-			goto retry_active;
-		}
-		log_error("cman_is_active error %d", errno);
-		cman_finish(ch);
-		return -ENOTCONN;
-	}
+	err = votequorum_fd_get(qh, &fd);
+	if (err != CS_OK)
+		goto fail;
 
-	rv = cman_start_notification(ch, cman_callback);
-	if (rv < 0) {
-		log_error("cman_start_notification error %d %d", rv, errno);
-		cman_finish(ch);
-		return rv;
-	}
+	err = votequorum_getinfo(qh, 0, &qinfo);
+	if (err != CS_OK)
+		goto fail;
+	our_nodeid = qinfo.node_id;
 
-	fd = cman_get_fd(ch);
-
-	/* FIXME: wait here for us to be a member of the cluster */
-	memset(&node, 0, sizeof(node));
-	rv = cman_get_node(ch, CMAN_NODEID_US, &node);
-	if (rv < 0) {
-		log_error("cman_get_node us error %d %d", rv, errno);
-		cman_stop_notification(ch);
-		cman_finish(ch);
-		fd = rv;
-		goto out;
-	}
-	our_nodeid = node.cn_nodeid;
+	err = votequorum_trackstart(qh, 0, CS_TRACK_CURRENT);
+	if (err != CS_OK)
+		goto fail;
 
 	old_node_count = 0;
 	memset(&old_nodes, 0, sizeof(old_nodes));
-	cman_node_count = 0;
-	memset(&cman_nodes, 0, sizeof(cman_nodes));
- out:
+	quorum_node_count = 0;
+	memset(&quorum_nodes, 0, sizeof(quorum_nodes));
+
 	return fd;
+ fail:
+	votequorum_finalize(qh);
+	return -1;
 }
 
 void close_cluster(void)
 {
-	cman_finish(ch);
+	votequorum_trackstop(qh);
+	votequorum_finalize(qh);
 }
 
-/* Force re-read of cman nodes */
-void update_cluster(void)
+static void shutdown_callback(corosync_cfg_handle_t h,
+			      corosync_cfg_shutdown_flags_t flags)
+{
+	if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) {
+		if (list_empty(&lockspaces))
+			corosync_cfg_replyto_shutdown(ch,
+					COROSYNC_CFG_SHUTDOWN_FLAG_YES);
+		else {
+			log_debug("no to corosync shutdown");
+			corosync_cfg_replyto_shutdown(ch,
+					COROSYNC_CFG_SHUTDOWN_FLAG_NO);
+		}
+	}
+}
+
+static corosync_cfg_callbacks_t cfg_callbacks =
+{
+	.corosync_cfg_shutdown_callback = shutdown_callback,
+	.corosync_cfg_state_track_callback = NULL,
+};
+
+void process_cluster_cfg(int ci)
+{
+	cs_error_t err;
+
+	err = corosync_cfg_dispatch(ch, CS_DISPATCH_ALL);
+	if (err != CS_OK)
+		cluster_dead(0);
+}
+
+int setup_cluster_cfg(void)
+{
+	cs_error_t err;
+	int fd;
+
+	err = corosync_cfg_initialize(&ch, &cfg_callbacks);
+	if (err != CS_OK)
+		return -1;
+
+	err = corosync_cfg_fd_get(ch, &fd);
+	if (err != CS_OK) {
+		corosync_cfg_finalize(ch);
+		return -1;
+	}
+
+	return fd;
+}
+
+void close_cluster_cfg(void)
 {
-	statechange();
+	corosync_cfg_finalize(ch);
 }
 
 int fence_node_time(int nodeid, uint64_t *last_fenced_time)