cluster: STABLE3 - rgmanager: Status check tuning/optimization

cluster: STABLE3 - rgmanager: Status check tuning/optimization

[Lon Hohberger]

Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=75af740e15eb95644d1c51b31b4fee2676e5899e
Commit:        75af740e15eb95644d1c51b31b4fee2676e5899e
Parent:        c34cda66db34b02835a949a6526653d1c1ab7b7c
Author:        Lon Hohberger <lhh@redhat.com>
AuthorDate:    Tue Mar 24 12:49:59 2009 -0400
Committer:     Lon Hohberger <lhh@redhat.com>
CommitterDate: Fri Mar 27 14:53:49 2009 -0400

rgmanager: Status check tuning/optimization

* Don't bother with status checks on 'service'
abstract resource.
* Spread status checks out based on completion
time instead of initiation time.
* Allow administrators to cap simultaneous
status checks to prevent load spikes.

Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
 rgmanager/include/rg_locks.h       |    4 +++
 rgmanager/src/daemons/main.c       |   17 ++++++++++++-
 rgmanager/src/daemons/restree.c    |   10 +++++--
 rgmanager/src/daemons/rg_locks.c   |   48 ++++++++++++++++++++++++++++++++++++
 rgmanager/src/resources/service.sh |    3 +-
 5 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/rgmanager/include/rg_locks.h b/rgmanager/include/rg_locks.h
index 50b181e..a427749 100644
--- a/rgmanager/include/rg_locks.h
+++ b/rgmanager/include/rg_locks.h
@@ -27,6 +27,10 @@ int rg_inc_status(void);
 int rg_dec_status(void);
 int rg_set_statusmax(int max);
 
+int rg_inc_children(void);
+int rg_dec_children(void);
+int rg_set_childmax(int max);
+
 int ccs_lock(void);
 int ccs_unlock(int fd);
 
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index af55987..3106b0e 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -833,6 +833,7 @@ configure_rgmanager(int ccsfd, int dbg)
 {
 	char *v;
 	char internal = 0;
+	int status_child_max = 0;
 
 	if (ccsfd < 0) {
 		internal = 1;
@@ -865,12 +866,26 @@ configure_rgmanager(int ccsfd, int dbg)
 		} else {
 			logt_print(LOG_WARNING, "Ignoring illegal "
 			       "status_poll_interval of %s\n", v);
-			status_poll_interval = 10;
+			status_poll_interval = DEFAULT_CHECK_INTERVAL;
 		}
 		
 		free(v);
 	}
 
+	if (ccs_get(ccsfd, "/cluster/rm/@status_child_max", &v) == 0) {
+		status_child_max = atoi(v);
+		if (status_child_max >= 1) {
+			logt_print(LOG_NOTICE,
+			       "Status Child Max set to %d\n",
+			       status_poll_interval);
+			rg_set_childmax(status_child_max);
+		} else {
+			logt_print(LOG_WARNING, "Ignoring illegal "
+			       "status_child_max of %s\n", v);
+		}
+		
+		free(v);
+	}
 
 	if (internal)
 		ccs_disconnect(ccsfd);
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index 1cec468..c80128a 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1151,15 +1151,19 @@ do_status(resource_node_t *node)
 	}
 
 	/* No check levels ready at the moment. */
-	if (idx == -1) {
+	/* Cap status check children if configured to do so */
+	if (idx == -1 || rg_inc_children() < 0) {
 		if (node->rn_checked)
 			return node->rn_last_status;
 		return 0;
 	}
 
-
-	node->rn_actions[idx].ra_last = now;
 	x = res_exec(node, RS_STATUS, NULL, node->rn_actions[idx].ra_depth);
+	rg_dec_children();
+
+	/* Record status check result *after* the status check has
+	 * completed. */
+	node->rn_actions[idx].ra_last = time(NULL);
 
 	node->rn_last_status = x;
 	node->rn_last_depth = node->rn_actions[idx].ra_depth;
diff --git a/rgmanager/src/daemons/rg_locks.c b/rgmanager/src/daemons/rg_locks.c
index e66cf1c..def034a 100644
--- a/rgmanager/src/daemons/rg_locks.c
+++ b/rgmanager/src/daemons/rg_locks.c
@@ -1,5 +1,6 @@
 #include <pthread.h>
 #include <stdio.h>
+#include <assert.h>
 #ifdef NO_CCS
 #include <libxml/xmlmemory.h>
 #include <libxml/parser.h>
@@ -18,6 +19,9 @@ static int __rg_initialized = 0;
 static int _rg_statuscnt = 0;
 static int _rg_statusmax = 5; /* XXX */
 
+static int _rg_childcnt = 0;
+static int _rg_childmax = 0; /* XXX */
+
 static pthread_cond_t unlock_cond = PTHREAD_COND_INITIALIZER;
 static pthread_cond_t zero_cond = PTHREAD_COND_INITIALIZER;
 static pthread_cond_t init_cond = PTHREAD_COND_INITIALIZER;
@@ -308,6 +312,50 @@ rg_dec_status(void)
 
 
 int
+rg_set_childmax(int max)
+{
+	int old;
+	
+	if (max <= 1)
+		max = 1;
+	
+	pthread_mutex_lock(&locks_mutex);
+	old = _rg_childmax;
+	_rg_childmax = max;
+	pthread_mutex_unlock(&locks_mutex);
+	return old;
+}
+
+
+int
+rg_inc_children(void)
+{
+	pthread_mutex_lock(&locks_mutex);
+	if (_rg_childmax && (_rg_childcnt >= _rg_childmax)) {
+		pthread_mutex_unlock(&locks_mutex);
+		return -1;
+	}
+	++_rg_childcnt;
+	pthread_mutex_unlock(&locks_mutex);
+	return 0;
+}
+
+
+int
+rg_dec_children(void)
+{
+	pthread_mutex_lock(&locks_mutex);
+	--_rg_childcnt;
+	if (_rg_childcnt < 0) {
+		assert(0);
+		_rg_childcnt = 0;
+	}
+	pthread_mutex_unlock(&locks_mutex);
+	return 0;
+}
+
+
+int
 rg_wait_threads(void)
 {
 	pthread_mutex_lock(&locks_mutex);
diff --git a/rgmanager/src/resources/service.sh b/rgmanager/src/resources/service.sh
index 04f968d..6e35ee5 100644
--- a/rgmanager/src/resources/service.sh
+++ b/rgmanager/src/resources/service.sh
@@ -203,9 +203,10 @@ meta_data()
         <action name="start" timeout="5"/>
         <action name="stop" timeout="5"/>
 	
-	<!-- No-ops.  Groups are abstract resource types.  -->
+	<!-- No-ops.  Groups are abstract resource types.
         <action name="status" timeout="5" interval="1h"/>
         <action name="monitor" timeout="5" interval="1h"/>
+       	 -->
 
         <action name="reconfig" timeout="5"/>
         <action name="recover" timeout="5"/>