cluster: master - cman: This is the new cman

cluster: master - cman: This is the new cman

[Christine Caulfield]

Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=cbd3091cf83f417f8bcd38bda062d7ecb4cdf78c
Commit:        cbd3091cf83f417f8bcd38bda062d7ecb4cdf78c
Parent:        4bfd7234d19e9c5d2d8e929713a8d65fad919c42
Author:        Christine Caulfield <ccaulfie@redhat.com>
AuthorDate:    Wed Jan 7 15:28:52 2009 +0000
Committer:     Christine Caulfield <ccaulfie@redhat.com>
CommitterDate: Wed Jan 7 15:28:52 2009 +0000

cman: This is the new cman

It's really two corosync plugins, cmanquorum is a more generic quorum
layer that ONLY does quorum.

cmanbits contains the missing pieces for cman.

There is a new libcman library that provides API (not ABI) compatibility
with cman in cluster2/3 though I'd really rather you didn't use it.

Suggestions for a more sensible replacement are welcome :-)

cman_tool still works, other things compile but have not been tested.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>
---
 cman/Makefile                              |    2 +-
 cman/cman_tool/Makefile                    |    3 +-
 cman/cman_tool/main.c                      |   87 ++------
 cman/services/cman/lib/Makefile            |    1 +
 cman/services/cman/lib/libcman.c           |  363 +++++++++++++++++++++++++---
 cman/services/quorum/services/cmanquorum.c |    1 +
 configure                                  |    4 +-
 7 files changed, 349 insertions(+), 112 deletions(-)

diff --git a/cman/Makefile b/cman/Makefile
index ead0baa..8a55b29 100644
--- a/cman/Makefile
+++ b/cman/Makefile
@@ -1,4 +1,4 @@
 include ../make/defines.mk
 include $(OBJDIR)/make/passthrough.mk
 
-SUBDIRS=lib cman_tool daemon qdisk notifyd init.d man
+SUBDIRS=cman_tool config services qdisk notifyd init.d man
diff --git a/cman/cman_tool/Makefile b/cman/cman_tool/Makefile
index 0dee578..71012ee 100644
--- a/cman/cman_tool/Makefile
+++ b/cman/cman_tool/Makefile
@@ -24,7 +24,8 @@ ${TARGET}: ${OBJS}
 	$(CC) -o $@ $^ $(LDFLAGS)
 
 depends:
-	$(MAKE) -C ../lib all
+	$(MAKE) -C ../services/quorum/lib all
+	$(MAKE) -C ../services/cman/lib all
 
 clean: generalclean
 
diff --git a/cman/cman_tool/main.c b/cman/cman_tool/main.c
index a6d35ca..2dcf014 100644
--- a/cman/cman_tool/main.c
+++ b/cman/cman_tool/main.c
@@ -21,7 +21,6 @@
 #define OP_NODES		9
 #define OP_SERVICES		10
 #define OP_DEBUG		11
-#define OP_DUMP_OBJDB		12
 
 
 static void print_usage(int subcmd)
@@ -219,6 +218,7 @@ static void show_status(void)
 	printf("Cluster Id: %d\n", info.ci_number);
 	printf("Cluster Member: Yes\n");
 	printf("Cluster Generation: %d\n", info.ci_generation);
+
 	printf("Membership state: %s\n", membership_state(tmpbuf, sizeof(tmpbuf),
 							  einfo->ei_node_state));
 	printf("Nodes: %d\n", einfo->ei_members);
@@ -229,54 +229,20 @@ static void show_status(void)
 	printf("Node votes: %d\n", einfo->ei_node_votes);
 
 	printf("Quorum: %d %s\n", einfo->ei_quorum, quorate?" ":"Activity blocked");
-	printf("Active subsystems: %d\n", cman_get_subsys_count(h));
 	printf("Flags:");
 	if (einfo->ei_flags & CMAN_EXTRA_FLAG_2NODE)
 		printf(" 2node");
-	if (einfo->ei_flags & CMAN_EXTRA_FLAG_SHUTDOWN)
-		printf(" Shutdown");
-	if (einfo->ei_flags & CMAN_EXTRA_FLAG_ERROR)
-		printf(" Error");
 	if (einfo->ei_flags & CMAN_EXTRA_FLAG_DISALLOWED)
 		printf(" DisallowedNodes");
 	if (einfo->ei_flags & CMAN_EXTRA_FLAG_DIRTY)
 		printf(" Dirty");
 	printf(" \n");
 
-	printf("Ports Bound: ");
-	portnum = 0;
-	for (i=0; i<32; i++) {
-		for (j=0; j<8; j++) {
-			if ((einfo->ei_ports[i] >> j) & 1)
-				printf("%d ", portnum);
-			portnum++;
-		}
-	}
-	printf(" \n");
-
-	node.cn_name[0] = 0;
 	if (cman_get_node(h, CMAN_NODEID_US, &node) == 0) {
 		printf("Node name: %s\n", node.cn_name);
 		printf("Node ID: %d\n", node.cn_nodeid);
 	}
 
-	printf("Multicast addresses: ");
-	addrptr = einfo->ei_addresses;
-	for (i=0; i < einfo->ei_num_addresses; i++) {
-		print_address(addrptr);
-		printf(" ");
-		addrptr += sizeof(struct sockaddr_storage);
-	}
-	printf("\n");
-
-	printf("Node addresses: ");
-	for (i=0; i < einfo->ei_num_addresses; i++) {
-		print_address(addrptr);
-		printf(" ");
-		addrptr += sizeof(struct sockaddr_storage);
-	}
-	printf("\n");
-
 	if (einfo->ei_flags & CMAN_EXTRA_FLAG_DISALLOWED) {
 		int count;
 		int numnodes;
@@ -293,6 +259,7 @@ static void show_status(void)
 			printf("\n");
 		}
 	}
+
 	cman_finish(h);
 }
 
@@ -374,7 +341,7 @@ static void print_node(commandline_t *comline, cman_handle_t h, int *format, str
 		       node->cn_nodeid, member_type,
 		       node->cn_incarnation, jstring, node->cn_name);
 	}
-
+#if 0
 	if (comline->fence_opt && !comline->format_opts) {
 		char agent[255];
 		uint64_t fence_time;
@@ -392,7 +359,8 @@ static void print_node(commandline_t *comline, cman_handle_t h, int *format, str
 			}
 		}
 	}
-
+#endif
+#if 0
 	int numaddrs;
 	struct cman_node_address addrs[MAX_INTERFACES];
 
@@ -411,7 +379,7 @@ static void print_node(commandline_t *comline, cman_handle_t h, int *format, str
 			}
 		}
 	}
-
+#endif
 	if (comline->format_opts) {
 		for (j = 0; j < MAX_FORMAT_OPTS; j++) {
 			switch (format[j]) {
@@ -426,6 +394,7 @@ static void print_node(commandline_t *comline, cman_handle_t h, int *format, str
 			case FMT_TYPE:
 				printf("%c ", member_type);
 				break;
+#if 0
 			case FMT_ADDR:
 				for (k = 0; k < numaddrs; k++) {
 					print_address(addrs[k].cna_address);
@@ -434,6 +403,7 @@ static void print_node(commandline_t *comline, cman_handle_t h, int *format, str
 					}
 				}
 				printf(" ");
+#endif
 				break;
 			default:
 				break;
@@ -616,12 +586,14 @@ static void set_votes(commandline_t *comline)
 	cman_finish(h);
 }
 
+
+// TODO this is obsolete really
 static void version(commandline_t *comline)
 {
 	struct cman_version ver;
 	cman_handle_t h;
 	int result;
-
+#if 0
 	h = open_cman_handle(1);
 
 	if ((result = cman_get_version(h, &ver)))
@@ -639,6 +611,7 @@ static void version(commandline_t *comline)
 		die("can't set version: %s", cman_error(errno));
  out:
 	cman_finish(h);
+#endif
 }
 
 static int cluster_wait(commandline_t *comline)
@@ -700,25 +673,16 @@ static void set_debuglog(commandline_t *comline)
 
 	h = open_cman_handle(1);
 
+#if 0
+// TODO this should be done via cluster.conf ?
+// or can we do something directly in objdb ??
+//
 	if (cman_set_debuglog(h, comline->verbose))
 		perror("setting debuglog failed");
-
+#endif
 	cman_finish(h);
 }
 
-#ifdef DEBUG
-static void dump_objdb(commandline_t *comline)
-{
-	cman_handle_t h;
-
-	h = open_cman_handle(1);
-
-	if (cman_dump_objdb(h, comline->filename))
-		perror("dump objdb failed");
-
-	cman_finish(h);
-}
-#endif
 
 static int get_int_arg(char argopt, char *arg)
 {
@@ -928,18 +892,6 @@ static void decode_arguments(int argc, char *argv[], commandline_t *comline)
 			if (comline->operation)
 				die("can't specify two operations");
 			comline->operation = OP_DEBUG;
-#ifdef DEBUG
-		} else if (strcmp(argv[optind], "dump-db") == 0) {
-			if (comline->operation)
-				die("can't specify two operations");
-			comline->operation = OP_DUMP_OBJDB;
-			if (!argv[optind+1])
-				die("no filename given");
-			comline->filename = strdup(argv[optind+1]);
-			if (comline->filename[0] != '/')
-				die("dump filename must be an absolute path");
-			optind++;
-#endif
 		} else if (strcmp(argv[optind], "remove") == 0) {
 			comline->remove = TRUE;
 		} else if (strcmp(argv[optind], "force") == 0) {
@@ -1061,11 +1013,6 @@ int main(int argc, char *argv[], char *envp[])
 	case OP_DEBUG:
 		set_debuglog(&comline);
 		break;
-#ifdef DEBUG
-	case OP_DUMP_OBJDB:
-		dump_objdb(&comline);
-		break;
-#endif
 	}
 
 	exit(EXIT_SUCCESS);
diff --git a/cman/services/cman/lib/Makefile b/cman/services/cman/lib/Makefile
index f8d9441..dcdf29f 100644
--- a/cman/services/cman/lib/Makefile
+++ b/cman/services/cman/lib/Makefile
@@ -11,3 +11,4 @@ CFLAGS += -fPIC
 CFLAGS += -I${incdir}
 CFLAGS += -I../include -I../../quorum/include
 
+LDFLAGS += -L${corosynclibdir} -lcoroutil -lcfg -lccs -L../../quorum/lib/ -Wl,-Bstatic -lcmanquorum -Wl,-Bdynamic
diff --git a/cman/services/cman/lib/libcman.c b/cman/services/cman/lib/libcman.c
index ca79729..eab0b7b 100644
--- a/cman/services/cman/lib/libcman.c
+++ b/cman/services/cman/lib/libcman.c
@@ -3,12 +3,14 @@
  */
 
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #include <pthread.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <errno.h>
+#include <limits.h>
 
 #include <corosync/corotypes.h>
 #include <corosync/mar_gen.h>
@@ -19,6 +21,7 @@
 #include <corosync/cmanquorum.h>
 #include <corosync/ipc_cman.h>
 
+#include "ccs.h"
 #include "libcman.h"
 
 #define CMAN_MAGIC 0x434d414e
@@ -37,6 +40,10 @@ struct cman_inst {
 	pthread_mutex_t response_mutex;
 	pthread_mutex_t dispatch_mutex;
 
+	int node_count;
+	cmanquorum_node_t * node_list;
+	int node_list_size;
+
 	corosync_cfg_handle_t cfg_handle;
 	cmanquorum_handle_t cmq_handle;
 	confdb_handle_t confdb_handle;
@@ -63,13 +70,23 @@ static CorosyncCfgCallbacksT cfg_callbacks =
 	.corosyncCfgShutdownCallback = cfg_shutdown_callback,
 };
 
-static void cman_instance_destructor (void *instance);
-
 
 #define VALIDATE_HANDLE(h) do {if (!(h) || (h)->magic != CMAN_MAGIC) {errno = EINVAL; return -1;}} while (0)
 
 static struct cman_inst *admin_inst;
 
+static uint16_t generate_cluster_id(char *name)
+{
+	int i;
+	int value = 0;
+
+	for (i=0; i<strlen(name); i++) {
+		value <<= 1;
+		value += name[i];
+	}
+	return value & 0xFFFF;
+}
+
 static void cfg_shutdown_callback(
 	corosync_cfg_handle_t handle,
 	CorosyncCfgShutdownFlagsT flags)
@@ -97,19 +114,21 @@ static void cmanquorum_notification_callback(
 
 	cmanquorum_context_get(handle, (void **)&cman_inst);
 
-	if (cman_inst->notify_callback)
-		cman_inst->notify_callback((void*)cman_inst, cman_inst->privdata, CMAN_REASON_STATECHANGE, quorate);
-}
-
+	/* Save information for synchronous queries */
+	cman_inst->node_count = node_list_entries;
+	if (cman_inst->node_list_size < node_list_entries) {
+		if (cman_inst->node_list)
+			free(cman_inst->node_list);
 
-/*
- * Clean up function for a cman instance (cman_init) handle
- */
-static void cman_instance_destructor (void *instance)
-{
-	struct cman_inst *cman_inst = instance;
+		cman_inst->node_list = malloc(sizeof(cmanquorum_node_t) * node_list_entries * 2);
+		if (cman_inst->node_list) {
+			memcpy(cman_inst->node_list, node_list, sizeof(cmanquorum_node_t) * node_list_entries);
+			cman_inst->node_list_size = node_list_entries;
+		}
+	}
 
-	pthread_mutex_destroy (&cman_inst->response_mutex);
+	if (cman_inst->notify_callback)
+		cman_inst->notify_callback((void*)cman_inst, cman_inst->privdata, CMAN_REASON_STATECHANGE, quorate);
 }
 
 static int cmanquorum_check_and_start(struct cman_inst *cman_inst)
@@ -124,6 +143,17 @@ static int cmanquorum_check_and_start(struct cman_inst *cman_inst)
 	return 0;
 }
 
+static int refresh_node_list(struct cman_inst *cman_inst)
+{
+	int error;
+	if (cmanquorum_check_and_start(cman_inst))
+		return -1;
+
+	cmanquorum_trackstart(cman_inst->cmq_handle, CS_TRACK_CURRENT);
+	error = cmanquorum_dispatch(cman_inst->cmq_handle, CS_DISPATCH_ONE);
+	return error;
+}
+
 cman_handle_t cman_init (
 	void *privdata)
 {
@@ -134,6 +164,7 @@ cman_handle_t cman_init (
 	if (!cman_inst)
 		return NULL;
 
+	memset(cman_inst, 0, sizeof(struct cman_inst));
 	error = saServiceConnect (&cman_inst->dispatch_fd,
 				  &cman_inst->response_fd,
 				  CMAN_SERVICE);
@@ -142,6 +173,7 @@ cman_handle_t cman_init (
 	}
 
 	cman_inst->privdata = privdata;
+	cman_inst->magic = CMAN_MAGIC;
 	pthread_mutex_init (&cman_inst->response_mutex, NULL);
 	pthread_mutex_init (&cman_inst->dispatch_mutex, NULL);
 
@@ -158,10 +190,11 @@ cman_handle_t cman_admin_init (
 {
 	if (admin_inst) {
 		errno = EBUSY;
-		return -1;
+		return NULL;
 	}
 
-	admin_inst = cman_init(NULL);
+	admin_inst = cman_init(privdata);
+	return admin_inst;
 }
 
 
@@ -169,13 +202,12 @@ int cman_finish (
 	cman_handle_t handle)
 {
 	struct cman_inst *cman_inst;
-	cs_error_t error;
 
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
 	if (cman_inst->cmq_handle) {
-		quorum_finalize(cman_inst->cmq_handle);
+		cmanquorum_finalize(cman_inst->cmq_handle);
 		cman_inst->cmq_handle = 0;
 	}
 	if (cman_inst->cfg_handle) {
@@ -372,13 +404,12 @@ int cman_is_quorate(cman_handle_t handle)
 {
 	struct cman_inst *cman_inst;
 	int quorate = -1;
-	int error;
 	struct cmanquorum_info info;
 
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	if (cmanquorum_getinfo(cman_inst->cmq_handle, 0, &info) != CS_OK)
@@ -411,6 +442,10 @@ int cman_shutdown(cman_handle_t handle, int flags)
 
 	error = corosync_cfg_try_shutdown(cman_inst->cfg_handle, cfg_flags);
 
+	/* ERR_LIBRARY happens because corosync shuts down while we are connected */
+	if (error == CS_ERR_LIBRARY || error == CS_OK)
+		error = 0;
+
 	return error;
 }
 
@@ -433,6 +468,9 @@ int cman_leave_cluster(cman_handle_t handle, int flags)
 	cfg_flags = COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE;
 
 	error = corosync_cfg_try_shutdown(cman_inst->cfg_handle, cfg_flags);
+	/* ERR_LIBRARY happens because corosync shuts down while we are connected */
+	if (error == CS_ERR_LIBRARY || error == CS_OK)
+		error = 0;
 
 	return error;
 }
@@ -457,7 +495,7 @@ int cman_replyto_shutdown(cman_handle_t handle, int flags)
 	return error;
 }
 
-int cman_killnode(cman_handle_t handle, unsigned int nodeid)
+int cman_kill_node(cman_handle_t handle, int nodeid)
 {
 	struct cman_inst *cman_inst;
 	int error;
@@ -485,7 +523,7 @@ int cman_set_votes(cman_handle_t handle, int votes, int nodeid)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_setvotes(cman_inst->cmq_handle, nodeid, votes);
@@ -501,7 +539,7 @@ int cman_set_expected_votes(cman_handle_t handle, int expected)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_setexpected(cman_inst->cmq_handle, expected);
@@ -514,7 +552,6 @@ int cman_get_fd (
 {
 	struct cman_inst *cman_inst;
 	int fd;
-	int error;
 
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
@@ -528,7 +565,6 @@ int cman_getprivdata(
 	cman_handle_t handle,
 	void **context)
 {
-	cs_error_t error;
 	struct cman_inst *cman_inst;
 
 	cman_inst = (struct cman_inst *)handle;
@@ -543,7 +579,6 @@ int cman_setprivdata(
 	cman_handle_t handle,
 	void *context)
 {
-	cs_error_t error;
 	struct cman_inst *cman_inst;
 
 	cman_inst = (struct cman_inst *)handle;
@@ -563,7 +598,7 @@ int cman_register_quorum_device(cman_handle_t handle, char *name, int votes)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_qdisk_register(cman_inst->cmq_handle, name, votes);
@@ -579,7 +614,7 @@ int cman_unregister_quorum_device(cman_handle_t handle)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_qdisk_unregister(cman_inst->cmq_handle);
@@ -594,7 +629,7 @@ int cman_poll_quorum_device(cman_handle_t handle, int isavailable)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_qdisk_poll(cman_inst->cmq_handle, 1);
@@ -611,7 +646,7 @@ int cman_get_quorum_device(cman_handle_t handle, struct cman_qdev_info *info)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_qdisk_getinfo(cman_inst->cmq_handle, &qinfo);
@@ -633,7 +668,7 @@ int cman_set_dirty(cman_handle_t handle)
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
+	if (cmanquorum_check_and_start(cman_inst))
 		return -1;
 
 	error = cmanquorum_setdirty(cman_inst->cmq_handle);
@@ -791,22 +826,274 @@ error_put:
 int cman_get_node_count(cman_handle_t handle)
 {
 	struct cman_inst *cman_inst;
-	int error;
+	int ccs_handle;
+	int num_nodes = 0;
+	char path[PATH_MAX];
+	char *value;
 
 	cman_inst = (struct cman_inst *)handle;
 	VALIDATE_HANDLE(cman_inst);
 
-	if (!cmanquorum_check_and_start(cman_inst))
-		return -1;
+	/* Returns the number of nodes known to ccs, not cman! */
+	ccs_handle = ccs_connect();
 
-	cmanquorum_trackstart(cman_inst->cmq_handle, CS_TRACK_CURRENT);
+	sprintf(path, "/cluster/clusternodes/clusternode[%d]/@name", num_nodes+1);
 
-	return error;
+	while (!ccs_get(ccs_handle, path, &value))
+	{
+		num_nodes++;
+		sprintf(path, "/cluster/clusternodes/clusternode[%d]/@name", num_nodes+1);
+	};
+
+	ccs_disconnect(ccs_handle);
+	return num_nodes;
+}
+
+int cman_is_active(cman_handle_t handle)
+{
+	struct cman_inst *cman_inst;
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	/* If we have connected, then 'cman' is active */
+	return 1;
+}
+
+int cman_get_cluster(cman_handle_t handle, cman_cluster_t *clinfo)
+{
+	struct cman_inst *cman_inst;
+	int ccs_handle;
+	char *value;
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	ccs_handle = ccs_connect();
+	if (!ccs_get(ccs_handle, "/cluster/@name", &value)) {
+		strcpy(clinfo->ci_name, value);
+		free(value);
+	}
+	if (!ccs_get(ccs_handle, "/cluster/cman/@cluster_id", &value)) {
+		clinfo->ci_number = atoi(value);
+		free(value);
+	}
+	else {
+		clinfo->ci_number = generate_cluster_id(clinfo->ci_name);
+	}
+	clinfo->ci_generation = 0; // CC: TODO ???
+
+	ccs_disconnect(ccs_handle);
+
+	return 0;
+}
+
+int cman_get_version(cman_handle_t handle, cman_version_t *version)
+{
+	struct cman_inst *cman_inst;
+	int ccs_handle;
+	char *value;
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	ccs_handle = ccs_connect();
+	if (!ccs_get(ccs_handle, "/cluster/@config_version", &value)) {
+		version->cv_config = atoi(value);
+		free(value);
+	}
+	//  CC: TODO ??? */
+	version->cv_major = 7;
+	version->cv_minor = 0;
+	version->cv_patch = 1;
+	ccs_disconnect(ccs_handle);
+
+	return 0;
 }
 
 int cman_get_nodes(cman_handle_t handle, int maxnodes, int *retnodes, cman_node_t *nodes)
-{}
+{
+	struct cman_inst *cman_inst;
+	int ccs_handle;
+	char *value;
+	int ret;
+	int i;
+	int num_nodes = 0;
+	char path[PATH_MAX];
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	refresh_node_list(cman_inst);
+	ccs_handle = ccs_connect();
+
+	do {
+		sprintf(path, "/cluster/clusternodes/clusternode[%d]/@name", num_nodes+1);
+		ccs_get(ccs_handle, path, &value);
+		if (value) {
+			strcpy(nodes[num_nodes].cn_name, value);
+			free(value);
+		}
+
+		sprintf(path, "/cluster/clusternodes/clusternode[%d]/@nodeid", num_nodes+1);
+		ret = ccs_get(ccs_handle, path, &value);
+		if (value) {
+			nodes[num_nodes].cn_nodeid = atoi(value);
+			free(value);
+		}
+
+		/* Reconcile with active nodes list. */
+		for (i=0; i < cman_inst->node_count; i++) {
+			if (cman_inst->node_list[i].nodeid == nodes[num_nodes].cn_nodeid) {
+				nodes[num_nodes].cn_member = (cman_inst->node_list[i].state == NODESTATE_MEMBER);
+			}
+		}
+
+		num_nodes++;
+	} while (ret == 0 && num_nodes < maxnodes);
+
+	*retnodes = num_nodes-1;
+	ccs_disconnect(ccs_handle);
+	return 0;
+}
+
 int cman_get_disallowed_nodes(cman_handle_t handle, int maxnodes, int *retnodes, cman_node_t *nodes)
-{}
+{
+	struct cman_inst *cman_inst;
+	int i;
+	int num_nodes = 0;
+	int ccs_handle;
+	char *value;
+	int ret;
+	char path[PATH_MAX];
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	refresh_node_list(cman_inst);
+	ccs_handle = ccs_connect();
+
+	for (i=0; i < cman_inst->node_count; i++) {
+		if (cman_inst->node_list[i].state == NODESTATE_DISALLOWED) {
+			nodes[num_nodes].cn_nodeid = cman_inst->node_list[i].nodeid;
+
+			/* Find the name: */
+			sprintf(path, "/cluster/clusternodes/clusternode[@nodeid=\"%d\"]/@name", cman_inst->node_list[i].nodeid);
+			ret = ccs_get(ccs_handle, path, &value);
+			if (!ret) {
+				strcpy(nodes[num_nodes].cn_name, value);
+				free(value);
+			}
+		}
+	}
+	*retnodes = num_nodes;
+	ccs_disconnect(ccs_handle);
+	return 0;
+
+}
+
 int cman_get_node(cman_handle_t handle, int nodeid, cman_node_t *node)
-{}
+{
+	struct cman_inst *cman_inst;
+	struct cmanquorum_info qinfo;
+	int i;
+	int ccs_handle;
+	int ret = 0;
+	char *value;
+	char path[PATH_MAX];
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	refresh_node_list(cman_inst);
+	ccs_handle = ccs_connect();
+
+	if (node->cn_name[0] == '\0') {
+		/* Query by node ID */
+		if (nodeid == CMAN_NODEID_US) {
+			if (cmanquorum_getinfo(cman_inst->cmq_handle, 0, &qinfo) != CS_OK) {
+				return -1;
+			}
+			nodeid = node->cn_nodeid = qinfo.node_id;
+		}
+
+		sprintf(path, "/cluster/clusternodes/clusternode[@nodeid=\"%d\"]/@name", nodeid);
+		ret = ccs_get(ccs_handle, path, &value);
+		if (!ret) {
+			strcpy(node->cn_name, value);
+			free(value);
+		}
+	}
+	else {
+		/* Query by node name */
+		sprintf(path, "/cluster/clusternodes/clusternode[@name=\"%s\"]/@nodeid", node->cn_name);
+		ret = ccs_get(ccs_handle, path, &value);
+		if (!ret) {
+			node->cn_nodeid = atoi(value);
+			free(value);
+		}
+	}
+
+	/* Fill in state */
+	node->cn_member = 3; /* Not in cluster */
+	for (i=0; i < cman_inst->node_count; i++) {
+		if (cman_inst->node_list[i].nodeid == node->cn_nodeid) {
+			if (cman_inst->node_list[i].state == NODESTATE_MEMBER)
+				node->cn_member = 2;
+		}
+	}
+
+	return 0;
+}
+
+int cman_start_notification(cman_handle_t handle, cman_callback_t callback)
+{
+	struct cman_inst *cman_inst;
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	cman_inst->notify_callback = callback;
+
+	return 0;
+}
+
+int cman_stop_notification(cman_handle_t handle)
+{
+	struct cman_inst *cman_inst;
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	cman_inst->notify_callback = NULL;
+
+	return 0;
+}
+
+
+/* This is not complete ... but that's my problem, not yours */
+int cman_get_extra_info(cman_handle_t handle, cman_extra_info_t *info, int maxlen)
+{
+	struct cman_inst *cman_inst;
+	struct cmanquorum_info qinfo;
+
+	cman_inst = (struct cman_inst *)handle;
+	VALIDATE_HANDLE(cman_inst);
+
+	refresh_node_list(cman_inst);
+
+	if (cmanquorum_getinfo(cman_inst->cmq_handle, 0, &qinfo) != CS_OK) {
+		return -1;
+	}
+
+	memset(info, 0, sizeof(cman_extra_info_t));
+	info->ei_flags = qinfo.flags;
+	info->ei_node_votes = qinfo.node_votes;
+	info->ei_total_votes = qinfo.total_votes;
+	info->ei_expected_votes = qinfo.node_expected_votes;
+	info->ei_quorum = qinfo.quorum;
+	info->ei_members = cman_inst->node_count;
+	info->ei_node_state = 2;
+	return 0;
+}
+
diff --git a/cman/services/quorum/services/cmanquorum.c b/cman/services/quorum/services/cmanquorum.c
index 4a58f6b..ee0218c 100644
--- a/cman/services/quorum/services/cmanquorum.c
+++ b/cman/services/quorum/services/cmanquorum.c
@@ -1137,6 +1137,7 @@ static void message_handler_req_lib_cmanquorum_getinfo (void *conn, void *messag
 		res_lib_cmanquorum_getinfo.quorum = quorum;
 		res_lib_cmanquorum_getinfo.total_votes = total_votes;
 		res_lib_cmanquorum_getinfo.flags = 0;
+		res_lib_cmanquorum_getinfo.nodeid = node->node_id;
 
 		if (us->flags & NODE_FLAGS_DIRTY)
 			res_lib_cmanquorum_getinfo.flags |= CMANQUORUM_INFO_FLAG_DIRTY;
diff --git a/configure b/configure
index 433c962..fcb49db 100755
--- a/configure
+++ b/configure
@@ -261,10 +261,10 @@ if (!$ccslibdir) {
   $ccslibdir="${objdir}/config/libs/libccsconfdb";
 }
 if (!$cmanincdir) {
-  $cmanincdir="${cdir}/cman/lib";
+  $cmanincdir="${cdir}/cman/services/cman/lib";
 }
 if (!$cmanlibdir) {
-  $cmanlibdir="${objdir}/cman/lib";
+  $cmanlibdir="${objdir}/cman/services/cman/lib";
 }
 if (!$logtincdir) {
   $logtincdir="${cdir}/common/liblogthread";