From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 8837 invoked by alias); 31 Mar 2009 19:00:39 -0000 Received: (qmail 8671 invoked by alias); 31 Mar 2009 19:00:29 -0000 X-SWARE-Spam-Status: No, hits=-2.0 required=5.0 tests=AWL,BAYES_00,SPF_HELO_PASS X-Spam-Status: No, hits=-2.0 required=5.0 tests=AWL,BAYES_00,SPF_HELO_PASS X-Spam-Check-By: sourceware.org X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on bastion.fedora.phx.redhat.com Subject: cluster: RHEL4 - rgmanager: Status check tuning/optimization To: cluster-cvs-relay@redhat.com X-Project: Cluster Project X-Git-Module: cluster.git X-Git-Refname: refs/heads/RHEL4 X-Git-Reftype: branch X-Git-Oldrev: b3f4f87d1a3caea844376719ebaccd1c6ed17666 X-Git-Newrev: 95ae1b2b584e840d23c92df474ca82bf600c8378 From: Lon Hohberger Message-Id: <20090331185944.2AC66120197@lists.fedorahosted.org> Date: Tue, 31 Mar 2009 19:00:00 -0000 X-Scanned-By: MIMEDefang 2.58 on 172.16.52.254 Mailing-List: contact cluster-cvs-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Post: List-Help: , Sender: cluster-cvs-owner@sourceware.org X-SW-Source: 2009-q1/txt/msg01011.txt.bz2 Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=95ae1b2b584e840d23c92df474ca82bf600c8378 Commit: 95ae1b2b584e840d23c92df474ca82bf600c8378 Parent: b3f4f87d1a3caea844376719ebaccd1c6ed17666 Author: Lon Hohberger AuthorDate: Tue Mar 24 12:49:59 2009 -0400 Committer: Lon Hohberger CommitterDate: Tue Mar 31 14:47:04 2009 -0400 rgmanager: Status check tuning/optimization * Don't bother with status checks on 'service' abstract resource. * Spread status checks out based on completion time instead of initiation time. * Allow administrators to cap simultaneous status checks to prevent load spikes. * rhbz487596 Signed-off-by: Lon Hohberger --- rgmanager/include/rg_locks.h | 4 +++ rgmanager/src/daemons/main.c | 33 ++++++++++++++++++++++++- rgmanager/src/daemons/restree.c | 10 +++++-- rgmanager/src/daemons/rg_locks.c | 48 ++++++++++++++++++++++++++++++++++++ rgmanager/src/resources/service.sh | 3 +- 5 files changed, 93 insertions(+), 5 deletions(-) diff --git a/rgmanager/include/rg_locks.h b/rgmanager/include/rg_locks.h index 78d6096..4cb808b 100644 --- a/rgmanager/include/rg_locks.h +++ b/rgmanager/include/rg_locks.h @@ -41,6 +41,10 @@ int rg_inc_status(void); int rg_dec_status(void); int rg_set_statusmax(int max); +int rg_inc_children(void); +int rg_dec_children(void); +int rg_set_childmax(int max); + int ccs_lock(void); int ccs_unlock(int fd); diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 05997aa..ea3f65a 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -59,6 +59,7 @@ int running = 1, need_reconfigure = 0; int shutdown_pending = 0; char debug = 0; /* XXX* */ static int signalled = 0; +static int status_poll_interval = 10; uint64_t next_node_id(cluster_member_list_t *membership, uint64_t me); @@ -525,7 +526,7 @@ event_loop(int clusterfd) struct timeval tv; uint64_t nodeid; - tv.tv_sec = 10; + tv.tv_sec = status_poll_interval; tv.tv_usec = 0; if (signalled) { @@ -649,6 +650,7 @@ configure_logging(int ccsfd) { char *v; char internal = 0; + int status_child_max = 0; if (ccsfd < 0) { internal = 1; @@ -675,6 +677,35 @@ configure_logging(int ccsfd) free(v); } + if (ccs_get(ccsfd, "/cluster/rm/@status_poll_interval", &v) == 0) { + status_poll_interval = atoi(v); + if (status_poll_interval >= 1) { + clulog(LOG_NOTICE, + "Status Polling Interval set to %d\n", v); + } else { + clulog(LOG_WARNING, "Ignoring illegal " + "status_poll_interval of %s\n", v); + status_poll_interval = 10; + } + + free(v); + } + + if (ccs_get(ccsfd, "/cluster/rm/@status_child_max", &v) == 0) { + status_child_max = atoi(v); + if (status_child_max >= 1) { + clulog(LOG_NOTICE, + "Status Child Max set to %d\n", + status_poll_interval); + rg_set_childmax(status_child_max); + } else { + clulog(LOG_WARNING, "Ignoring illegal " + "status_child_max of %s\n", v); + } + + free(v); + } + if (internal) ccs_disconnect(ccsfd); diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c index 828b985..fab77f3 100644 --- a/rgmanager/src/daemons/restree.c +++ b/rgmanager/src/daemons/restree.c @@ -1177,15 +1177,19 @@ do_status(resource_node_t *node) } /* No check levels ready at the moment. */ - if (idx == -1) { + /* Cap status check children if configured to do so */ + if (idx == -1 || rg_inc_children() < 0) { if (node->rn_checked) return node->rn_last_status; return 0; } - - node->rn_actions[idx].ra_last = now; x = res_exec(node, RS_STATUS, NULL, node->rn_actions[idx].ra_depth); + rg_dec_children(); + + /* Record status check result *after* the status check has + * completed. */ + node->rn_actions[idx].ra_last = time(NULL); node->rn_last_status = x; node->rn_last_depth = node->rn_actions[idx].ra_depth; diff --git a/rgmanager/src/daemons/rg_locks.c b/rgmanager/src/daemons/rg_locks.c index 4a89762..759a2de 100644 --- a/rgmanager/src/daemons/rg_locks.c +++ b/rgmanager/src/daemons/rg_locks.c @@ -18,6 +18,7 @@ */ #include #include +#include #ifdef NO_CCS #include #include @@ -36,6 +37,9 @@ static int __rg_initialized = 0; static int _rg_statuscnt = 0; static int _rg_statusmax = 5; /* XXX */ +static int _rg_childcnt = 0; +static int _rg_childmax = 0; /* XXX */ + static pthread_cond_t unlock_cond = PTHREAD_COND_INITIALIZER; static pthread_cond_t zero_cond = PTHREAD_COND_INITIALIZER; static pthread_cond_t init_cond = PTHREAD_COND_INITIALIZER; @@ -315,6 +319,50 @@ rg_dec_status(void) int +rg_set_childmax(int max) +{ + int old; + + if (max <= 1) + max = 1; + + pthread_mutex_lock(&locks_mutex); + old = _rg_childmax; + _rg_childmax = max; + pthread_mutex_unlock(&locks_mutex); + return old; +} + + +int +rg_inc_children(void) +{ + pthread_mutex_lock(&locks_mutex); + if (_rg_childmax && (_rg_childcnt >= _rg_childmax)) { + pthread_mutex_unlock(&locks_mutex); + return -1; + } + ++_rg_childcnt; + pthread_mutex_unlock(&locks_mutex); + return 0; +} + + +int +rg_dec_children(void) +{ + pthread_mutex_lock(&locks_mutex); + --_rg_childcnt; + if (_rg_childcnt < 0) { + assert(0); + _rg_childcnt = 0; + } + pthread_mutex_unlock(&locks_mutex); + return 0; +} + + +int rg_wait_threads(void) { pthread_mutex_lock(&locks_mutex); diff --git a/rgmanager/src/resources/service.sh b/rgmanager/src/resources/service.sh index b7c9e08..a21a66b 100755 --- a/rgmanager/src/resources/service.sh +++ b/rgmanager/src/resources/service.sh @@ -222,9 +222,10 @@ meta_data() - +