From 6351258d707337b69563d4be8effbb30fc42f784 Mon Sep 17 00:00:00 2001 From: Noah Sanci Date: Wed, 28 Jul 2021 14:46:05 -0400 Subject: [PATCH] debuginfod: PR27277 - Describe retrieved files when verbose There appear to exist use cases that intend to simply check for the existence of content in a debuginfod server, without actually downloading it. In HTTP land, the HEAD operation is the natural expression of this. Instead of implementing a HEAD/describe option, allow users, with enough verbosity, to print the HTTP response headers upon retrieving a file. E.g output: HTTP/1.1 200 OK Connection: Keep-Alive Content-Length: 2428240 Cache-Control: public Last-Modified: Sat, 15 May 2021 20:49:51 GMT Content-Type: application/octet-stream Date: Tue, 03 Aug 2021 18:50:36 GMT https://sourceware.org/bugzilla/show_bug.cgi?id=27277 Signed-off-by: Noah Sanci --- debuginfod/ChangeLog | 11 ++ debuginfod/debuginfod-client.c | 58 ++++++- doc/ChangeLog | 6 + doc/debuginfod-find.1 | 3 +- tests/ChangeLog | 7 + tests/Makefile.am | 3 +- tests/run-debuginfod-response-headers.sh | 186 +++++++++++++++++++++++ 7 files changed, 268 insertions(+), 6 deletions(-) create mode 100755 tests/run-debuginfod-response-headers.sh diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog index 530f7dc7..cb9e50c7 100644 --- a/debuginfod/ChangeLog +++ b/debuginfod/ChangeLog @@ -4,6 +4,17 @@ * debuginfod.cxx (handler_cb): Fix after_you unique_set key to the entire incoming URL. +2021-08-02 Noah Sanci + + PR27277 + * debuginfod-client.c (struct debuginfod_client): New field + winning_headers. + (struct handle_data): New field response_data. + (header_callback): Store received headers in response_data. + (debuginfod_query_server): Activate CURLOPT_HEADERFUNCTION. + Save winning response_data. + (debuginfod_end): free client winning headers. + 2021-07-26 Noah Sanci PR27982 diff --git a/debuginfod/debuginfod-client.c b/debuginfod/debuginfod-client.c index 7d4b220f..985a17c5 100644 --- a/debuginfod/debuginfod-client.c +++ b/debuginfod/debuginfod-client.c @@ -127,6 +127,7 @@ struct debuginfod_client timeout or other info gotten from environment variables, the handle data, etc. So those don't have to be reparsed and recreated on each request. */ + char * winning_headers; }; /* The cache_clean_interval_s file within the debuginfod cache specifies @@ -183,6 +184,8 @@ struct handle_data to the cache. Used to ensure that a file is not downloaded from multiple servers unnecessarily. */ CURL **target_handle; + /* Response http headers for this client handle, sent from the server */ + char *response_data; }; static size_t @@ -499,6 +502,33 @@ default_progressfn (debuginfod_client *c, long a, long b) } +static size_t +header_callback (char * buffer, size_t size, size_t numitems, void * userdata) +{ + if (size != 1) + return 0; + /* Temporary buffer for realloc */ + char *temp = NULL; + size_t userlen = 0; + if (*(char**)userdata == NULL) + { + temp = malloc(numitems+1); + if (temp == NULL) + return 0; + memset(temp, '\0', numitems+1); + } + else + { + userlen = strlen(*(char**)userdata); + temp = realloc(*(char**)userdata, userlen + numitems + 1); + if (temp == NULL) + return 0; + } + strncat(temp, buffer, numitems); + *(char**)userdata = temp; + return numitems; +} + /* Query each of the server URLs found in $DEBUGINFOD_URLS for the file with the specified build-id, type (debuginfo, executable or source) and filename. filename may be NULL. If found, return a file @@ -936,10 +966,13 @@ debuginfod_query_server (debuginfod_client *c, curl_easy_setopt (data[i].handle, CURLOPT_LOW_SPEED_LIMIT, 100 * 1024L); } + data[i].response_data = NULL; curl_easy_setopt(data[i].handle, CURLOPT_FILETIME, (long) 1); curl_easy_setopt(data[i].handle, CURLOPT_FOLLOWLOCATION, (long) 1); curl_easy_setopt(data[i].handle, CURLOPT_FAILONERROR, (long) 1); curl_easy_setopt(data[i].handle, CURLOPT_NOSIGNAL, (long) 1); + curl_easy_setopt(data[i].handle, CURLOPT_HEADERFUNCTION, header_callback); + curl_easy_setopt(data[i].handle, CURLOPT_HEADERDATA, (void *) &(data[i].response_data)); #if LIBCURL_VERSION_NUM >= 0x072a00 /* 7.42.0 */ curl_easy_setopt(data[i].handle, CURLOPT_PATH_AS_IS, (long) 1); #else @@ -961,6 +994,7 @@ debuginfod_query_server (debuginfod_client *c, int committed_to = -1; bool verbose_reported = false; struct timespec start_time, cur_time; + c->winning_headers = NULL; if ( maxtime > 0 && clock_gettime(CLOCK_MONOTONIC_RAW, &start_time) == -1) { rc = errno; @@ -995,8 +1029,18 @@ debuginfod_query_server (debuginfod_client *c, if (data[i].handle != target_handle) curl_multi_remove_handle(curlm, data[i].handle); else - committed_to = i; - } + { + committed_to = i; + if (c->winning_headers == NULL) + { + c->winning_headers = data[committed_to].response_data; + if (vfd >= 0) + dprintf(vfd, "\n%s", c->winning_headers); + data[committed_to].response_data = NULL; + } + + } + } if (vfd >= 0 && !verbose_reported && committed_to >= 0) { @@ -1161,10 +1205,10 @@ debuginfod_query_server (debuginfod_client *c, { char *effective_url = NULL; long resp_code = 500; - CURLcode ok1 = curl_easy_getinfo (target_handle, + CURLcode ok1 = curl_easy_getinfo (msg->easy_handle, CURLINFO_EFFECTIVE_URL, &effective_url); - CURLcode ok2 = curl_easy_getinfo (target_handle, + CURLcode ok2 = curl_easy_getinfo (msg->easy_handle, CURLINFO_RESPONSE_CODE, &resp_code); if(ok1 == CURLE_OK && ok2 == CURLE_OK && effective_url) @@ -1238,7 +1282,10 @@ debuginfod_query_server (debuginfod_client *c, { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ curl_easy_cleanup (data[i].handle); + free(data[i].response_data); } + free(c->winning_headers); + c->winning_headers = NULL; goto query_in_parallel; } else @@ -1281,6 +1328,7 @@ debuginfod_query_server (debuginfod_client *c, { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ curl_easy_cleanup (data[i].handle); + free (data[i].response_data); } for (int i = 0; i < num_urls; ++i) @@ -1304,6 +1352,7 @@ debuginfod_query_server (debuginfod_client *c, { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ curl_easy_cleanup (data[i].handle); + free (data[i].response_data); } unlink (target_cache_tmppath); @@ -1415,6 +1464,7 @@ debuginfod_end (debuginfod_client *client) curl_multi_cleanup (client->server_mhandle); curl_slist_free_all (client->headers); + free (client->winning_headers); free (client->url); free (client); } diff --git a/doc/ChangeLog b/doc/ChangeLog index 77d1d705..3f542e08 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -10,6 +10,12 @@ * Makefile.am: Updated to include debuginfod-client-config.7 * man3, man7: Symlinks for source tree man page testing. +2021-08-04 Noah Sanci + + PR27277 + * debuginfod-find.1: Increasing verbosity describes the downloaded + file. + 2021-07-26 Noah Sanci PR27982 diff --git a/doc/debuginfod-find.1 b/doc/debuginfod-find.1 index a61673f5..957ec7e7 100644 --- a/doc/debuginfod-find.1 +++ b/doc/debuginfod-find.1 @@ -110,7 +110,8 @@ l l. .TP .B "\-v" -Increase verbosity, including printing frequent download-progress messages. +Increase verbosity, including printing frequent download-progress messages +and printing the http response headers from the server. .SH "SECURITY" diff --git a/tests/ChangeLog b/tests/ChangeLog index 3bfd1ca2..f4aaf3ee 100644 --- a/tests/ChangeLog +++ b/tests/ChangeLog @@ -4,6 +4,13 @@ * backtrace.c (callback_verify): Check for pthread_kill as first frame. Change asserts to fprintf plus abort. +2021-08-02 Noah Sanci + + PR27277 + * run-debuginfod-response-headers.sh: Add a test to ensure that file descriptions + are accurate for files outside or within archives. + * Makefile.am: Add run-debuginfod-response-headers.sh to TESTS. + 2021-07-26 Noah Sanci PR27982 diff --git a/tests/Makefile.am b/tests/Makefile.am index 429649f4..99fa61ba 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -211,7 +211,7 @@ if DEBUGINFOD check_PROGRAMS += debuginfod_build_id_find # With the dummy delegation doesn't work if !DUMMY_LIBDEBUGINFOD -TESTS += run-debuginfod-find.sh +TESTS += run-debuginfod-find.sh run-debuginfod-response-headers.sh endif endif @@ -475,6 +475,7 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \ run-disasm-riscv64.sh \ testfile-riscv64-dis1.o.bz2 testfile-riscv64-dis1.expect.bz2 \ run-debuginfod-find.sh \ + run-debuginfod-response-headers.sh \ debuginfod-rpms/fedora30/hello2-1.0-2.src.rpm \ debuginfod-rpms/fedora30/hello2-1.0-2.x86_64.rpm \ debuginfod-rpms/fedora30/hello2-debuginfo-1.0-2.x86_64.rpm \ diff --git a/tests/run-debuginfod-response-headers.sh b/tests/run-debuginfod-response-headers.sh new file mode 100755 index 00000000..be1345fc --- /dev/null +++ b/tests/run-debuginfod-response-headers.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2019-2021 Red Hat, Inc. +# This file is part of elfutils. +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# elfutils is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. $srcdir/test-subr.sh # includes set -e + +# for test case debugging, uncomment: +set -x + +DB=${PWD}/.debuginfod_tmp.sqlite +tempfiles $DB +export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache + +while true; do + PORT1=`expr '(' $RANDOM % 1000 ')' + 9000` + ss -atn | fgrep ":$PORT1" || break +done +PID1=0 +PID2=0 +PID3=0 +PID4=0 + +cleanup() +{ + if [ $PID1 -ne 0 ]; then kill $PID1; wait $PID1; fi + if [ $PID2 -ne 0 ]; then kill $PID2; wait $PID2; fi + if [ $PID3 -ne 0 ]; then kill $PID3; wait $PID3; fi + if [ $PID4 -ne 0 ]; then kill $PID4; wait $PID4; fi + rm -rf F R D L Z ${PWD}/foobar ${PWD}/mocktree ${PWD}/.client_cache* ${PWD}/tmp* + exit_cleanup +} + +# clean up trash if we were aborted early +trap cleanup 0 1 2 3 5 9 15 + +errfiles_list= +err() { + echo ERROR REPORTS + for ports in $PORT1 $PORT2 $PORT3 + do + echo ERROR REPORT $port metrics + curl -s http://127.0.0.1:$port/metrics + echo + done + for x in $errfiles_list + do + echo ERROR REPORT "$x" + cat $x + echo + done + false # trigger set -e +} +trap err ERR + +errfiles() { + errfiles_list="$errfiles_list $*" +} + +wait_ready() +{ + port=$1; + what=$2; + value=$3; + timeout=20; + + echo "Wait $timeout seconds on $port for metric $what to change to $value" + while [ $timeout -gt 0 ]; do + mvalue="$(curl -s http://127.0.0.1:$port/metrics \ + | grep "$what" | awk '{print $NF}')" + if [ -z "$mvalue" ]; then mvalue=0; fi + echo "metric $what: $mvalue" + if [ "$mvalue" -eq "$value" ]; then + break; + fi + sleep 0.5; + ((timeout--)); + done; + + if [ $timeout -eq 0 ]; then + echo "metric $what never changed to $value on port $port" + err + fi +} + +# We want to run debuginfod in the background. We also want to start +# it with the same check/installcheck-sensitive LD_LIBRARY_PATH stuff +# that the testrun alias sets. But: we if we just use +# testrun .../debuginfod +# it runs in a subshell, with different pid, so not helpful. +# +# So we gather the LD_LIBRARY_PATH with this cunning trick: +ldpath=`testrun sh -c 'echo $LD_LIBRARY_PATH'` + +mkdir F R + +env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p $PORT1 -t0 -g0 -v R F > vlog$PORT1 2>&1 & +PID1=$! +tempfiles vlog$PORT1 +errfiles vlog$PORT1 +# Server must become ready +wait_ready $PORT1 'ready' 1 +export DEBUGINFOD_URLS=http://127.0.0.1:$PORT1/ # or without trailing / +# Check thread comm names +ps -q $PID1 -e -L -o '%p %c %a' | grep groom +ps -q $PID1 -e -L -o '%p %c %a' | grep scan +ps -q $PID1 -e -L -o '%p %c %a' | grep traverse + +# We use -t0 and -g0 here to turn off time-based scanning & grooming. +# For testing purposes, we just sic SIGUSR1 / SIGUSR2 at the process. + +######################################################################## + +# Compile a simple program, strip its debuginfo and save the build-id. +# Also move the debuginfo into another directory so that elfutils +# cannot find it without debuginfod. +echo "int main() { return 0; }" > ${PWD}/prog.c +tempfiles prog.c +# Create a subdirectory to confound source path names +mkdir foobar +gcc -Wl,--build-id -g -o prog ${PWD}/foobar///./../prog.c +testrun ${abs_top_builddir}/src/strip -g -f prog.debug ${PWD}/prog +BUILDID=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \ + -a prog | grep 'Build ID' | cut -d ' ' -f 7` + +mv prog F +mv prog.debug F +kill -USR1 $PID1 +# Wait till both files are in the index. +wait_ready $PORT1 'thread_work_total{role="traverse"}' 1 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 + +cp -rvp ${abs_srcdir}/debuginfod-rpms R +if [ "$zstd" = "false" ]; then # nuke the zstd fedora 31 ones + rm -vrf R/debuginfod-rpms/fedora31 +fi + +kill -USR1 $PID1 +# Wait till both files are in the index and scan/index fully finished +wait_ready $PORT1 'thread_work_total{role="traverse"}' 2 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 +# All rpms need to be in the index, except the dummy permission-000 one +rpms=$(find R -name \*rpm | grep -v nothing | wc -l) +wait_ready $PORT1 'scanned_files_total{source=".rpm archive"}' $rpms +kill -USR1 $PID1 # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs +# Wait till both files are in the index and scan/index fully finished +wait_ready $PORT1 'thread_work_total{role="traverse"}' 3 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 + +######################################################################## +## PR27277 +# Make a simple request to the debuginfod server and check debuginfod-find's vlog to see if +# the custom HTTP headers are received. +rm -rf $DEBUGINFOD_CACHE_PATH +env DEBUGINFOD_URLS="http://127.0.0.1:"$PORT1 LD_LIBRARY_PATH=$ldpath ${abs_top_builddir}/debuginfod/debuginfod-find\ + -vvv executable F/prog > vlog-find$PORT1.1 2>&1 +tempfiles vlog-find$PORT1.1 +cat vlog-find$PORT1.1 +#grep 'Content-Length: ' vlog-find$PORT1.1 +#grep 'Connection: ' vlog-find$PORT1.1 +#grep 'Cache-Control: ' vlog-find$PORT1.1 + +# Check to see if an executable file located in an archive prints the file's description and archive +env DEBUGINFOD_URLS="http://127.0.0.1:"$PORT1 LD_LIBRARY_PATH=$ldpath ${abs_top_builddir}/debuginfod/debuginfod-find\ + -vvv executable c36708a78618d597dee15d0dc989f093ca5f9120 > vlog-find$PORT1.2 2>&1 +tempfiles vlog-find$PORT1.2 +grep 'Content-Length: ' vlog-find$PORT1.2 +grep 'Connection: ' vlog-find$PORT1.2 +grep 'Cache-Control: ' vlog-find$PORT1.2 + -- 2.31.1