master - dlm_controld: open dlm-monitor misc device

master - dlm_controld: open dlm-monitor misc device

[David Teigland]

Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=3c6d20abb8feac1d3756a13bf98e062a385b5f10
Commit:        3c6d20abb8feac1d3756a13bf98e062a385b5f10
Parent:        485525632d2763d69eb87bd137e798468cbfc9a3
Author:        David Teigland <teigland@redhat.com>
AuthorDate:    Tue Aug 19 14:39:59 2008 -0500
Committer:     David Teigland <teigland@redhat.com>
CommitterDate: Tue Aug 19 14:43:58 2008 -0500

dlm_controld: open dlm-monitor misc device

if it exists, as it will in future kernels.  This allows dlm-kernel
to detect if dlm_controld fails, and stop lockspaces in response.

Also rework/simplify handling of all misc devices; depend on udev
to create the device nodes.

Adds /dev/misc/dlm-monitor to the dlm's udev config file.

Signed-off-by: David Teigland <teigland@redhat.com>
---
 dlm/libdlm/51-dlm.rules         |    1 +
 group/dlm_controld/action.c     |  100 +++++++++++++++++++++
 group/dlm_controld/dlm_daemon.h |    5 +
 group/dlm_controld/main.c       |   25 +++++
 group/dlm_controld/plock.c      |  189 ++++-----------------------------------
 5 files changed, 148 insertions(+), 172 deletions(-)

diff --git a/dlm/libdlm/51-dlm.rules b/dlm/libdlm/51-dlm.rules
index 70d26dc..f71e79d 100644
--- a/dlm/libdlm/51-dlm.rules
+++ b/dlm/libdlm/51-dlm.rules
@@ -1,4 +1,5 @@
 KERNEL=="dlm-control", NAME="misc/dlm-control", MODE="0666"
+KERNEL=="dlm-monitor", NAME="misc/dlm-monitor", MODE="0666"
 KERNEL=="dlm_default", NAME="misc/dlm_default", MODE="0666"
 KERNEL=="dlm_*", NAME="misc/%k", MODE="0660"
 
diff --git a/group/dlm_controld/action.c b/group/dlm_controld/action.c
index aaf183a..18d0b09 100644
--- a/group/dlm_controld/action.c
+++ b/group/dlm_controld/action.c
@@ -829,3 +829,103 @@ int setup_configfs(void)
 	return 0;
 }
 
+static void find_minors(void)
+{
+	FILE *fl;
+	char name[256];
+	uint32_t number;
+	int found = 0;
+	int c;
+
+	if (!(fl = fopen("/proc/misc", "r"))) {
+		log_error("/proc/misc fopen failed: %s", strerror(errno));
+		return;
+	}
+
+	while (!feof(fl)) {
+		if (fscanf(fl, "%d %255s\n", &number, &name[0]) == 2) {
+
+			if (!strcmp(name, "dlm-control")) {
+				control_minor = number;
+				found++;
+			} else if (!strcmp(name, "dlm-monitor")) {
+				monitor_minor = number;
+				found++;
+			} else if (!strcmp(name, "dlm_plock")) {
+				plock_minor = number;
+				found++;
+			} else if (!strcmp(name, "lock_dlm_plock")) {
+				old_plock_minor = number;
+				found++;
+			}
+
+		} else do {
+			c = fgetc(fl);
+		} while (c != EOF && c != '\n');
+
+		if (found == 3)
+			break;
+	}
+	fclose(fl);
+
+	if (!found)
+		log_error("Is dlm missing from kernel? No misc devices found.");
+}
+
+static int find_udev_device(char *path, uint32_t minor)
+{
+	struct stat st;
+	int i;
+
+	for (i = 0; i < 10; i++) {
+		if (stat(path, &st) == 0 && minor(st.st_rdev) == minor)
+			return 0;
+		sleep(1);
+	}
+
+	log_error("cannot find device %s with minor %d", path, minor);
+	return -1;
+}
+
+int setup_misc_devices(void)
+{
+	int rv;
+
+	find_minors();
+
+	if (control_minor) {
+		rv = find_udev_device("/dev/misc/dlm-control", control_minor);
+		if (rv < 0)
+			return rv;
+		log_debug("found /dev/misc/dlm-control minor %u",
+			  control_minor);
+	}
+
+	if (monitor_minor) {
+		rv = find_udev_device("/dev/misc/dlm-monitor", monitor_minor);
+		if (rv < 0)
+			return rv;
+		log_debug("found /dev/misc/dlm-monitor minor %u",
+			  monitor_minor);
+	}
+
+	if (plock_minor) {
+		rv = find_udev_device("/dev/misc/dlm_plock", plock_minor);
+		if (rv < 0)
+			return rv;
+		log_debug("found /dev/misc/dlm_plock minor %u",
+			  plock_minor);
+	}
+
+	if (old_plock_minor) {
+		rv = find_udev_device("/dev/misc/lock_dlm_plock",
+				      old_plock_minor);
+		if (rv < 0)
+			return rv;
+		log_debug("found /dev/misc/lock_dlm_plock minor %u",
+			  old_plock_minor);
+	}
+
+	return 0;
+}
+
diff --git a/group/dlm_controld/dlm_daemon.h b/group/dlm_controld/dlm_daemon.h
index 0b3feb9..3a4b31c 100644
--- a/group/dlm_controld/dlm_daemon.h
+++ b/group/dlm_controld/dlm_daemon.h
@@ -83,6 +83,10 @@ extern int dump_wrap;
 extern char plock_dump_buf[DLMC_DUMP_SIZE];
 extern int plock_dump_len;
 extern int group_mode;
+extern uint32_t control_minor;
+extern uint32_t monitor_minor;
+extern uint32_t plock_minor;
+extern uint32_t old_plock_minor;
 
 void daemon_dump_save(void);
 
@@ -218,6 +222,7 @@ void del_configfs_node(int nodeid);
 void clear_configfs(void);
 int setup_configfs(void);
 int check_uncontrolled_lockspaces(void);
+int setup_misc_devices(void);
 
 /* config.c */
 int setup_ccs(void);
diff --git a/group/dlm_controld/main.c b/group/dlm_controld/main.c
index afa590d..d712b88 100644
--- a/group/dlm_controld/main.c
+++ b/group/dlm_controld/main.c
@@ -18,6 +18,7 @@ static struct pollfd *pollfd = NULL;
 static pthread_t query_thread;
 static pthread_mutex_t query_mutex;
 static struct list_head fs_register_list;
+static int kernel_monitor_fd;
 
 struct client {
 	int fd;
@@ -840,6 +841,20 @@ static int setup_queries(void)
 	return 0;
 }
 
+/* The dlm in kernels before 2.6.28 do not have the monitor device.  We
+   keep this fd open as long as we're running.  If we exit/terminate while
+   lockspaces exist in the kernel, the kernel will detect a close on this
+   fd and stop the lockspaces. */
+
+static void setup_monitor(void)
+{
+	if (!monitor_minor)
+		return;
+
+	kernel_monitor_fd = open("/dev/misc/dlm-monitor", O_RDONLY);
+	log_debug("/dev/misc/dlm-monitor fd %d", kernel_monitor_fd);
+}
+
 void cluster_dead(int ci)
 {
 	log_error("cluster is down, exiting");
@@ -877,6 +892,12 @@ static void loop(void)
 	if (rv < 0)
 		goto out;
 
+	rv = setup_misc_devices();
+	if (rv < 0)
+		goto out;
+
+	setup_monitor();
+
 	rv = setup_configfs();
 	if (rv < 0)
 		goto out;
@@ -1275,4 +1296,8 @@ int dump_wrap;
 char plock_dump_buf[DLMC_DUMP_SIZE];
 int plock_dump_len;
 int group_mode;
+uint32_t control_minor;
+uint32_t monitor_minor;
+uint32_t plock_minor;
+uint32_t old_plock_minor;
 
diff --git a/group/dlm_controld/plock.c b/group/dlm_controld/plock.c
index e9a47ce..3a95caf 100644
--- a/group/dlm_controld/plock.c
+++ b/group/dlm_controld/plock.c
@@ -3,12 +3,6 @@
 
 #include <linux/dlm_plock.h>
 
-#define PROC_MISC               "/proc/misc"
-#define PROC_DEVICES            "/proc/devices"
-#define MISC_NAME               "misc"
-#define CONTROL_DIR             "/dev/misc"
-#define CONTROL_NAME            "dlm_plock"
-
 static uint32_t plock_read_count;
 static uint32_t plock_recv_count;
 static uint32_t plock_rate_delays;
@@ -16,7 +10,7 @@ static struct timeval plock_read_time;
 static struct timeval plock_recv_time;
 static struct timeval plock_rate_last;
 
-static int control_fd = -1;
+static int plock_device_fd = -1;
 static SaCkptHandleT system_ckpt_handle;
 static SaCkptCallbacksT callbacks = { 0, 0 };
 static SaVersionT version = { 'B', 1, 1 };
@@ -144,133 +138,6 @@ static char *ex_str(int optype, int ex)
 		return "RD";
 }
 
-static int get_proc_number(const char *file, const char *name, uint32_t *number)
-{
-	FILE *fl;
-	char nm[256];
-	int c;
-
-	if (!(fl = fopen(file, "r"))) {
-		log_error("%s: fopen failed: %s", file, strerror(errno));
-		return 0;
-	}
-
-	while (!feof(fl)) {
-		if (fscanf(fl, "%d %255s\n", number, &nm[0]) == 2) {
-			if (!strcmp(name, nm)) {
-				fclose(fl);
-				return 1;
-			}
-		} else do {
-			c = fgetc(fl);
-		} while (c != EOF && c != '\n');
-	}
-	fclose(fl);
-
-	log_error("%s: No entry for %s found", file, name);
-	return 0;
-}
-
-static int control_device_number(const char *plock_misc_name,
-				 uint32_t *major, uint32_t *minor)
-{
-	if (!get_proc_number(PROC_DEVICES, MISC_NAME, major) ||
-	    !get_proc_number(PROC_MISC, plock_misc_name, minor)) {
-		*major = 0;
-		return 0;
-	}
-
-	return 1;
-}
-
-/*
- * Returns 1 if exists; 0 if it doesn't; -1 if it's wrong
- */
-static int control_exists(const char *control, uint32_t major, uint32_t minor)
-{
-	struct stat buf;
-
-	if (stat(control, &buf) < 0) {
-		if (errno != ENOENT)
-			log_error("%s: stat failed: %s", control,
-				  strerror(errno));
-		return 0;
-	}
-
-	if (!S_ISCHR(buf.st_mode)) {
-		log_error("%s: Wrong inode type", control);
-		if (!unlink(control))
-			return 0;
-		log_error("%s: unlink failed: %s", control, strerror(errno));
-		return -1;
-	}
-
-	if (major && buf.st_rdev != makedev(major, minor)) {
-		log_error("%s: Wrong device number: (%u, %u) instead of "
-			  "(%u, %u)", control, major(buf.st_mode),
-			  minor(buf.st_mode), major, minor);
-		if (!unlink(control))
-			return 0;
-		log_error("%s: unlink failed: %s", control, strerror(errno));
-		return -1;
-	}
-
-	return 1;
-}
-
-static int create_control(const char *control, uint32_t major, uint32_t minor)
-{
-	int ret;
-	mode_t old_umask;
-
-	if (!major)
-		return 0;
-
-	old_umask = umask(0022);
-	ret = mkdir(CONTROL_DIR, 0777);
-	umask(old_umask);
-	if (ret < 0 && errno != EEXIST) {
-		log_error("%s: mkdir failed: %s", CONTROL_DIR, strerror(errno));
-		return 0;
-	}
-
-	if (mknod(control, S_IFCHR | S_IRUSR | S_IWUSR, makedev(major, minor)) < 0) {
-		log_error("%s: mknod failed: %s", control, strerror(errno));
-		return 0;
-	}
-
-	return 1;
-}
-
-static int open_control(const char *control_name, const char *plock_misc_name)
-{
-	char control[PATH_MAX];
-	uint32_t major = 0, minor = 0;
-
-	if (control_fd != -1)
-		return 0;
-
-	snprintf(control, sizeof(control), "%s/%s", CONTROL_DIR, control_name);
-
-	if (!control_device_number(plock_misc_name, &major, &minor))
-		return -1;
-
-	if (!control_exists(control, major, minor) &&
-	    !create_control(control, major, minor)) {
-		log_error("Failure to create device file %s", control);
-		return -1;
-	}
-
-	control_fd = open(control, O_RDWR);
-	if (control_fd < 0) {
-		log_error("Failure to open device %s: %s", control,
-			  strerror(errno));
-		return -1;
-	}
-
-	return 0;
-}
-
 /*
  * In kernels before 2.6.26, plocks came from gfs2's lock_dlm module.
  * Reading plocks from there as well should allow us to use cluster3
@@ -278,13 +145,9 @@ static int open_control(const char *control_name, const char *plock_misc_name)
  * structs is the mountgroup id, which we need to translate to the ls id.
  */
 
-#define OLD_CONTROL_NAME "lock_dlm_plock"
-#define OLD_PLOCK_MISC_NAME "lock_dlm_plock"
-
 int setup_plocks(void)
 {
 	SaAisErrorT err;
-	int rv;
 
 	plock_read_count = 0;
 	plock_recv_count = 0;
@@ -302,24 +165,25 @@ int setup_plocks(void)
 		   send ENOSYS back to the kernel if it tries to do a plock */
 	}
 
-
-	rv = open_control(CONTROL_NAME, DLM_PLOCK_MISC_NAME);
-	if (rv) {
-		log_debug("setup_plocks trying old lock_dlm interface");
-		rv = open_control(OLD_CONTROL_NAME, OLD_PLOCK_MISC_NAME);
-		if (rv) {
-			log_error("Is dlm missing from kernel?  No control device.");
-			return rv;
-		}
+	if (plock_minor) {
+		plock_device_fd = open("/dev/misc/dlm_plock", O_RDWR);
+	} else if (old_plock_minor) {
+		log_debug("setup_plocks using old lock_dlm interface");
 		need_fsid_translation = 1;
+		plock_device_fd = open("/dev/misc/lock_dlm_plock", O_RDWR);
 	}
 
-	log_debug("plocks %d", control_fd);
+	if (plock_device_fd < 0) {
+		log_error("Failure to open plock device: %s", strerror(errno));
+		return -1;
+	}
+
+	log_debug("plocks %d", plock_device_fd);
 	log_debug("plock cpg message size: %u bytes",
 		  (unsigned int) (sizeof(struct dlm_header) +
 		                  sizeof(struct dlm_plock_info)));
 
-	return control_fd;
+	return plock_device_fd;
 }
 
 static uint32_t mg_to_ls_id(uint32_t fsid)
@@ -775,7 +639,7 @@ static void write_result(struct lockspace *ls, struct dlm_plock_info *in,
 		in->fsid = ls->associated_mg_id;
 
 	in->rv = rv;
-	write(control_fd, in, sizeof(struct dlm_plock_info));
+	write(plock_device_fd, in, sizeof(struct dlm_plock_info));
 }
 
 static void do_waiters(struct lockspace *ls, struct resource *r)
@@ -1557,10 +1421,10 @@ void process_plocks(int ci)
 
 	memset(&info, 0, sizeof(info));
 
-	rv = do_read(control_fd, &info, sizeof(info));
+	rv = do_read(plock_device_fd, &info, sizeof(info));
 	if (rv < 0) {
 		log_debug("process_plocks: read error %d fd %d\n",
-			  errno, control_fd);
+			  errno, plock_device_fd);
 		return;
 	}
 
@@ -1631,7 +1495,7 @@ void process_plocks(int ci)
 
  fail:
 	info.rv = rv;
-	rv = write(control_fd, &info, sizeof(info));
+	rv = write(plock_device_fd, &info, sizeof(info));
 }
 
 void process_saved_plocks(struct lockspace *ls)
@@ -2283,22 +2147,3 @@ int fill_plock_dump_buf(struct lockspace *ls)
 	return rv;
 }
 
-#if 0
-/* replace all of the above with this to build on 2.6.25 kernels */
-
-int setup_plocks(void) { return 0; };
-void process_plocks(int ci) { };
-int limit_plocks(void) { return 0; };
-void receive_plock(struct lockspace *ls, struct dlm_header *hd, int len) { };
-void receive_own(struct lockspace *ls, struct dlm_header *hd, int len) { };
-void receive_sync(struct lockspace *ls, struct dlm_header *hd, int len) { };
-void receive_drop(struct lockspace *ls, struct dlm_header *hd, int len) { };
-void process_saved_plocks(struct lockspace *ls) { };
-void close_plock_checkpoint(struct lockspace *ls) { };
-void store_plocks(struct lockspace *ls) { };
-void retrieve_plocks(struct lockspace *ls) { };
-void purge_plocks(struct lockspace *ls, int nodeid, int unmount) { };
-int dump_plocks(char *name, int fd) { return 0; };
-
-#endif
-