From: Noah Sanci <nsanci@redhat.com>
To: elfutils-devel@sourceware.org
Subject: PR: 25978
Date: Fri, 2 Jul 2021 14:24:14 -0400 [thread overview]
Message-ID: <CAJXA7qgf5vSzh4TF8SvxpxKt2x_J=hr74RhUGCrprsjYcVs=MQ@mail.gmail.com> (raw)
diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index 286c910a..06d03e72 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,9 @@
+2021-06-28 Noah Sanci <nsanci@redhat.com>
+
+ PR25978
+ * debuginfod.cxx: Added command line options
+ --fdcache-prefetch-fds/mbs and associated metrics/functionality.
+
2021-06-03 Frank Ch. Eigler <fche@redhat.com>
PR27863
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index 543044c6..431605aa 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -368,7 +368,13 @@ static const struct argp_option options[] =
{ "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number
of archive files to prefetch into fdcache.", 0 },
#define ARGP_KEY_FDCACHE_MINTMP 0x1004
{ "fdcache-mintmp", ARGP_KEY_FDCACHE_MINTMP, "NUM", 0, "Minimum
free space% on tmpdir.", 0 },
- { NULL, 0, NULL, 0, NULL, 0 }
+#define ARGP_KEY_FDCACHE_PREFETCH_MBS 0x1005
+ { "fdcache-prefetch-mbs", ARGP_KEY_FDCACHE_PREFETCH_MBS, "MB",
0,"Megabytes allocated to the \
+ prefetch cache.", 0},
+#define ARGP_KEY_FDCACHE_PREFETCH_FDS 0x1006
+ { "fdcache-prefetch-fds", ARGP_KEY_FDCACHE_PREFETCH_FDS, "NUM",
0,"Number of files allocated to the \
+ prefetch cache.", 0},
+ { NULL, 0, NULL, 0, NULL, 0 },
};
/* Short description of program. */
@@ -412,6 +418,8 @@ static long fdcache_fds;
static long fdcache_mbs;
static long fdcache_prefetch;
static long fdcache_mintmp;
+static long fdcache_prefetch_mbs;
+static long fdcache_prefetch_fds;
static string tmpdir;
static void set_metric(const string& key, double value);
@@ -538,10 +546,22 @@ parse_opt (int key, char *arg,
break;
case ARGP_KEY_FDCACHE_MINTMP:
fdcache_mintmp = atol (arg);
+ if( fdcache_mintmp > 100 || fdcache_mintmp < 0 )
+ argp_failure(state, 1, EINVAL, "fdcache mintmp percent");
break;
case ARGP_KEY_ARG:
source_paths.insert(string(arg));
break;
+ case ARGP_KEY_FDCACHE_PREFETCH_FDS:
+ fdcache_prefetch_fds = atol(arg);
+ if ( fdcache_prefetch_fds <= 0)
+ argp_failure(state, 1, EINVAL, "fdcache prefetch fds");
+ break;
+ case ARGP_KEY_FDCACHE_PREFETCH_MBS:
+ fdcache_prefetch_mbs = atol(arg);
+ if ( fdcache_prefetch_mbs <= 0)
+ argp_failure(state, 1, EINVAL, "fdcache prefetch mbs");
+ break;
// case 'h': argp_state_help (state, stderr,
ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
default: return ARGP_ERR_UNKNOWN;
}
@@ -1199,16 +1219,24 @@ private:
};
deque<fdcache_entry> lru; // @head: most recently used
long max_fds;
+ deque<fdcache_entry> prefetch; // prefetched
long max_mbs;
+ long max_prefetch_mbs;
+ long max_prefetch_fds;
public:
void set_metrics()
{
- double total_mb = 0.0;
+ double fdcache_mb = 0.0;
+ double prefetch_mb = 0.0;
for (auto i = lru.begin(); i < lru.end(); i++)
- total_mb += i->fd_size_mb;
- set_metric("fdcache_bytes", (int64_t)(total_mb*1024.0*1024.0));
- set_metric("fdcache_count", lru.size());
+ fdcache_mb += i->fd_size_mb;
+ for (auto j = prefetch.begin(); j < prefetch.end(); j++)
+ prefetch_mb += j->fd_size_mb;
+ set_metric("fdcache_bytes", fdcache_mb*1024.0*1024.0);
+ set_metric("fdcache_count", fdcache_mb);
+ set_metric("fdcache_prefetch_bytes", prefetch_mb*1024.0*1024.0);
+ set_metric("fdcache_prefetch_count", prefetch_mb);
}
void intern(const string& a, const string& b, string fd, off_t sz,
bool front_p)
@@ -1221,7 +1249,17 @@ public:
{
unlink (i->fd.c_str());
lru.erase(i);
- inc_metric("fdcache_op_count","op","dequeue");
+ inc_metric("fdcache_op_count","op","lru_dequeue");
+ break; // must not continue iterating
+ }
+ }
+ for (auto i = prefetch.begin(); i < prefetch.end(); i++) //
nuke preexisting copy in prefetch
+ {
+ if (i->archive == a && i->entry == b)
+ {
+ unlink (i->fd.c_str());
+ prefetch.erase(i);
+ inc_metric("fdcache_op_count","op","prefetch_dequeue");
break; // must not continue iterating
}
}
@@ -1229,13 +1267,13 @@ public:
fdcache_entry n = { a, b, fd, mb };
if (front_p)
{
- inc_metric("fdcache_op_count","op","enqueue_front");
+ inc_metric("fdcache_op_count","op","lru_enqueue");
lru.push_front(n);
}
else
{
- inc_metric("fdcache_op_count","op","enqueue_back");
- lru.push_back(n);
+ inc_metric("fdcache_op_count","op","prefetch_enqueue");
+ prefetch.push_front(n);
}
if (verbose > 3)
obatched(clog) << "fdcache interned a=" << a << " b=" << b
@@ -1248,10 +1286,10 @@ public:
{
inc_metric("fdcache_op_count","op","emerg-flush");
obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
- this->limit(0, 0); // emergency flush
+ this->limit(0, 0, 0, 0); // emergency flush
}
else if (front_p)
- this->limit(max_fds, max_mbs); // age cache if required
+ this->limit(max_fds, max_mbs, max_prefetch_fds,
max_prefetch_mbs); // age cache if required
}
int lookup(const string& a, const string& b)
@@ -1266,21 +1304,32 @@ public:
fdcache_entry n = *i;
lru.erase(i); // invalidates i, so no more iteration!
lru.push_front(n);
- inc_metric("fdcache_op_count","op","requeue_front");
+ inc_metric("fdcache_op_count","op","lru_requeue_front");
+ fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if
dup() fails; looks like cache miss
+ break;
+ }
+ }
+ for ( auto i = prefetch.begin(); fd == -1 && i < prefetch.end(); ++i)
+ {
+ if (i->archive == a && i->entry == b)
+ { // found it; take the entry from the prefetch deque to
the lru deque, since it has now been accessed.
+ fdcache_entry n = *i;
+ prefetch.erase(i);
+ lru.push_front(n);
+ inc_metric("fdcache_op_count","op","prefetch_access");
fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if
dup() fails; looks like cache miss
break;
}
}
}
-
if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
{
inc_metric("fdcache_op_count","op","emerg-flush");
obatched(clog) << "fdcache emergency flush for filling tmpdir";
- this->limit(0, 0); // emergency flush
+ this->limit(0, 0, 0, 0); // emergency flush
}
else if (fd >= 0)
- this->limit(max_fds, max_mbs); // age cache if required
+ this->limit(max_fds, max_mbs, max_prefetch_fds,
max_prefetch_mbs); // age cache if required
return fd;
}
@@ -1292,7 +1341,15 @@ public:
{
if (i->archive == a && i->entry == b)
{
- inc_metric("fdcache_op_count","op","probe_hit");
+ inc_metric("fdcache_op_count","op","lru_probe_hit");
+ return true;
+ }
+ }
+ for (auto i = prefetch.begin(); i < prefetch.end(); i++)
+ {
+ if (i->archive == a && i->entry == b)
+ {
+ inc_metric("fdcache_op_count","op","prefetch_probe_hit");
return true;
}
}
@@ -1306,19 +1363,30 @@ public:
for (auto i = lru.begin(); i < lru.end(); i++)
{
if (i->archive == a && i->entry == b)
- { // found it; move it to head of lru
+ { // found it; erase it from lru
fdcache_entry n = *i;
lru.erase(i); // invalidates i, so no more iteration!
- inc_metric("fdcache_op_count","op","clear");
+ inc_metric("fdcache_op_count","op","lru_clear");
unlink (n.fd.c_str());
set_metrics();
return;
}
}
+ for (auto i = prefetch.begin(); i < prefetch.end(); i++)
+ {
+ if (i->archive == a && i->entry == b)
+ { // found it; erase it from lru
+ fdcache_entry n = *i;
+ prefetch.erase(i); // invalidates i, so no more iteration!
+ inc_metric("fdcache_op_count","op","prefetch_clear");
+ unlink (n.fd.c_str());
+ set_metrics();
+ return;
+ }
+ }
}
-
- void limit(long maxfds, long maxmbs, bool metrics_p = true)
+ void limit(long maxfds, long maxmbs, long maxprefetchfds, long
maxprefetchmbs , bool metrics_p = true)
{
if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
obatched(clog) << "fdcache limited to maxfds=" << maxfds << "
maxmbs=" << maxmbs << endl;
@@ -1326,7 +1394,8 @@ public:
unique_lock<mutex> lock(fdcache_lock);
this->max_fds = maxfds;
this->max_mbs = maxmbs;
-
+ this->max_prefetch_fds = maxprefetchfds;
+ this->max_prefetch_mbs = maxprefetchmbs;
long total_fd = 0;
double total_mb = 0.0;
for (auto i = lru.begin(); i < lru.end(); i++)
@@ -1334,7 +1403,7 @@ public:
// accumulate totals from most recently used one going backward
total_fd ++;
total_mb += i->fd_size_mb;
- if (total_fd > max_fds || total_mb > max_mbs)
+ if (total_fd > this->max_fds || total_mb > this->max_mbs)
{
// found the cut here point!
@@ -1344,7 +1413,7 @@ public:
obatched(clog) << "fdcache evicted a=" <<
j->archive << " b=" << j->entry
<< " fd=" << j->fd << " mb=" <<
j->fd_size_mb << endl;
if (metrics_p)
- inc_metric("fdcache_op_count","op","evict");
+ inc_metric("fdcache_op_count","op","lru_evict");
unlink (j->fd.c_str());
}
@@ -1352,6 +1421,29 @@ public:
break;
}
}
+ total_fd = 0;
+ total_mb = 0.0;
+ for(auto i = prefetch.begin(); i < prefetch.end(); i++){
+ // accumulate totals from most recently used one going backward
+ total_fd ++;
+ total_mb += i->fd_size_mb;
+ if (total_fd > this->max_prefetch_fds || total_mb >
this->max_prefetch_mbs)
+ {
+ // found the cut here point!
+ for (auto j = i; j < prefetch.end(); j++) // close all
the fds from here on in
+ {
+ if (verbose > 3)
+ obatched(clog) << "fdcache evicted from prefetch
a=" << j->archive << " b=" << j->entry
+ << " fd=" << j->fd << " mb=" <<
j->fd_size_mb << endl;
+ if (metrics_p)
+ inc_metric("fdcache_op_count","op","prefetch_evict");
+ unlink (j->fd.c_str());
+ }
+
+ prefetch.erase(i, prefetch.end()); // erase the nodes generally
+ break;
+ }
+ }
if (metrics_p) set_metrics();
}
@@ -1360,7 +1452,7 @@ public:
{
// unlink any fdcache entries in $TMPDIR
// don't update metrics; those globals may be already destroyed
- limit(0, 0, false);
+ limit(0, 0, 0, 0, false);
}
};
static libarchive_fdcache fdcache;
@@ -1547,7 +1639,7 @@ handle_buildid_r_match (bool internal_req_p,
// responsible for unlinking it later.
fdcache.intern(b_source0, fn,
tmppath, archive_entry_size(e),
- false); // prefetched ones go to back of lru
+ false); // prefetched ones go to the prefetch cache
prefetch_count --;
close (fd); // we're not saving this fd to make a mhd-response from!
continue;
@@ -3293,8 +3385,8 @@ void groom()
sqlite3_db_release_memory(dbq); // ... for both connections
debuginfod_pool_groom(); // and release any debuginfod_client
objects we've been holding onto
- fdcache.limit(0,0); // release the fdcache contents
- fdcache.limit(fdcache_fds,fdcache_mbs); // restore status quo parameters
+ fdcache.limit(0,0,0,0); // release the fdcache contents
+ fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds,
fdcache_prefetch_mbs); // restore status quo parameters
clock_gettime (CLOCK_MONOTONIC, &ts_end);
double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec
- ts_start.tv_nsec)/1.e9;
@@ -3456,7 +3548,7 @@ main (int argc, char *argv[])
if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;
- fdcache.limit(fdcache_fds, fdcache_mbs);
+ fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds,
fdcache_prefetch_mbs);
(void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it
incidentally, ignore
(void) signal (SIGINT, signal_handler); // ^C
@@ -3600,6 +3692,9 @@ main (int argc, char *argv[])
obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
obatched(clog) << "fdcache tmpdir min% " << fdcache_mintmp << endl;
obatched(clog) << "groom time " << groom_s << endl;
+ obatched(clog) << "prefetch fds " << fdcache_prefetch_fds << endl;
+ obatched(clog) << "prefetch mbs " << fdcache_prefetch_mbs << endl;
+
if (scan_archives.size()>0)
{
obatched ob(clog);
diff --git a/doc/debuginfod.8 b/doc/debuginfod.8
index 1ba42cf6..8945eb9b 100644
--- a/doc/debuginfod.8
+++ b/doc/debuginfod.8
@@ -212,6 +212,16 @@ $TMPDIR or \fB/tmp\fP filesystem. This is
because that is where the
most recently used extracted files are kept. Grooming cleans this
cache.
+.TP
+.B "\-\-fdcache\-\-prefetch\-fds=NUM" "\-\-fdcache\-\-prefetch\-mbs=MB"
+Configure how many file descriptors (fds) and megabytes (mbs) are
+allocated to the prefetch portion of the fdcache. If unspecified,
+values of \fB\-\-prefetch\-fds\fP and \fB\-\-prefetch\-mbs\fP depend
+on concurrency of the system and on the available disk space on
+the $TMPDIR. Allocating more to the prefetch cache will improve
+performance in environments where different parts of several large
+archives are being accessed.
+
.TP
.B "\-\-fdcache\-mintmp=NUM"
Configure a disk space threshold for emergency flushing of the cache.
diff --git a/tests/ChangeLog b/tests/ChangeLog
index d8fa97fa..94201959 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,10 @@
+2021-06-28 Noah Sanci <nsanci@redhat.com>
+
+ PR25978
+ * run-debuginfod-find.sh: Test to ensure options
+ fdcache-prefetch-fds/mbs are set. Check that inc_metric works for lru
+ and prefetch cache metrics.
+
2021-06-16 Frank Ch. Eigler <fche@redhat.com>
* run-debuginfod-find.sh: Fix intermittent groom/stale failure,
diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh
index 456dc2f8..47e33424 100755
--- a/tests/run-debuginfod-find.sh
+++ b/tests/run-debuginfod-find.sh
@@ -119,12 +119,15 @@ wait_ready()
fi
}
+FDCACHE_FDS=50
+FDCACHE_MBS=190
+PREFETCH_FDS=10
+PREFETCH_MBS=120
# create a bogus .rpm file to evoke a metric-visible error
# Use a cyclic symlink instead of chmod 000 to make sure even root
# would see an error (running the testsuite under root is NOT encouraged).
ln -s R/nothing.rpm R/nothing.rpm
-
-env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS=
${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p
$PORT1 -t0 -g0 --fdcache-fds 1 --fdcache-mbs 2 --fdcache-mintmp 0 -Z
.tar.xz -Z .tar.bz2=bzcat -v R F Z L > vlog$PORT1 2>&1 &
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS=
${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p
$PORT1 -t0 -g0 --fdcache-mbs=$FDCACHE_MBS --fdcache-fds=$FDCACHE_FDS
--fdcache-prefetch-mbs=$PREFETCH_MBS
--fdcache-prefetch-fds=$PREFETCH_FDS --fdcache-mintmp 0 -Z .tar.xz -Z
.tar.bz2=bzcat -v R F Z L > vlog$PORT1 2>&1 &
PID1=$!
tempfiles vlog$PORT1
errfiles vlog$PORT1
@@ -470,6 +473,25 @@ archive_test
f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/h
egrep '(libc.error.*rhel7)|(bc1febfd03ca)|(f0aa15b8aba)' vlog$PORT1
+########################################################################
+## PR25978
+# Ensure that the fdcache options are working.
+grep "prefetch fds" vlog$PORT1
+grep "prefetch mbs" vlog$PORT1
+grep "fdcache fds" vlog$PORT1
+grep "fdcache mbs" vlog$PORT1
+
+wait_ready $PORT1 'fdcache_op_count{op="lru_enqueue"}' 28
+wait_ready $PORT1 'fdcache_op_count{op="lru_evict"}' 24
+wait_ready $PORT1 'fdcache_op_count{op="lru_probe_hit"}' 6
+wait_ready $PORT1 'fdcache_op_count{op="lru_requeue_front"}' 11
+wait_ready $PORT1 'fdcache_op_count{op="lru_probe_miss"}' 0
+wait_ready $PORT1 'fdcache_prefetch_bytes' 0
+wait_ready $PORT1 'fdcache_prefetch_count' 0
+wait_ready $PORT1 'fdcache_op_count{op="prefetch_access"}' 4
+wait_ready $PORT1 'fdcache_op_count{op="prefetch_enqueue"}' 4
+wait_ready $PORT1 'fdcache_op_count{op="prefetch_evict"}' 0
+
########################################################################
# Federation mode
fpaste: https://paste.centos.org/view/raw/c798096f
next reply other threads:[~2021-07-02 18:24 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-02 18:24 Noah Sanci [this message]
2021-07-08 13:07 ` Mark Wielaard
2021-07-08 13:12 ` Frank Ch. Eigler
2021-07-08 15:04 ` Mark Wielaard
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to='CAJXA7qgf5vSzh4TF8SvxpxKt2x_J=hr74RhUGCrprsjYcVs=MQ@mail.gmail.com' \
--to=nsanci@redhat.com \
--cc=elfutils-devel@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).