From a4e5d1bbd39d2448e5ae208a114e3a68d9f2c28a Mon Sep 17 00:00:00 2001
From: Ryan Goldberg <rgoldber@redhat.com>
Date: Mon, 8 Aug 2022 13:30:08 -0400
Subject: [PATCH] One-shot mode for server

When in one-shot mode, for the first
scan all of the found executables' paths
and buildids will be written to the provided
file descriptor in the given format.

Signed-off-by: Ryan Goldberg <rgoldber@redhat.com>
---
 configure.ac                     |   5 ++
 debuginfod/ChangeLog             |   9 +++
 debuginfod/Makefile.am           |   2 +-
 debuginfod/debuginfod.cxx        |  52 ++++++++++++++-
 debuginfod/debuginfod.h.in       |   9 +++
 doc/ChangeLog                    |   4 ++
 doc/debuginfod.8                 |  16 +++++
 tests/ChangeLog                  |   7 ++-
 tests/Makefile.am                |   5 +-
 tests/run-debuginfod-one-shot.sh | 105 +++++++++++++++++++++++++++++++
 10 files changed, 209 insertions(+), 5 deletions(-)
 create mode 100755 tests/run-debuginfod-one-shot.sh

diff --git a/configure.ac b/configure.ac
index 03b67a9d..aadff4ca 100644
--- a/configure.ac
+++ b/configure.ac
@@ -600,6 +600,11 @@ case "$ac_cv_search__obstack_free" in
 esac
 AC_SUBST([obstack_LIBS])
 
+AC_CHECK_LIB(json-c, json_tokener_parse, [
+  AC_DEFINE([ENABLE_ONE_SHOT], [1], [Define if the one-shot feature is enabled])
+  AC_SUBST(jsonc_LIBS, '-ljson-c')
+])
+
 dnl The directories with content.
 
 dnl Documentation.
diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index a8b5d7f4..b6fac996 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,12 @@
+2022-08-09  Ryan Goldberg <rgoldber@redhat.com>
+
+	* debuginfod.cxx (archive_classify): New global (one_shot_mtx).
+	(archive_classify, thread_main_fts_source_paths) In one-shot mode
+	for the first scan, output found buildids/paths to given file descriptor.
+	(parse_opt): Add "--one-shot" flag.
+	* debuginfod.h.in: Added in debuginfod_oneshot_header struct,
+	debuginfod_oneshot_header_magic
+
 2022-08-02  Josef Cejka <jcejka@suse.de>
 
 	* debuginfod.cxx (groom): Don't evaluate regex unless needed.
diff --git a/debuginfod/Makefile.am b/debuginfod/Makefile.am
index 435cb8a6..03cdeed0 100644
--- a/debuginfod/Makefile.am
+++ b/debuginfod/Makefile.am
@@ -70,7 +70,7 @@ bin_PROGRAMS += debuginfod-find
 endif
 
 debuginfod_SOURCES = debuginfod.cxx
-debuginfod_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(argp_LDADD) $(fts_LIBS) $(libmicrohttpd_LIBS) $(sqlite3_LIBS) $(libarchive_LIBS) -lpthread -ldl
+debuginfod_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(argp_LDADD) $(fts_LIBS) $(libmicrohttpd_LIBS) $(sqlite3_LIBS) $(libarchive_LIBS) $(jsonc_LIBS) -lpthread -ldl
 
 debuginfod_find_SOURCES = debuginfod-find.c
 debuginfod_find_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(argp_LDADD) $(fts_LIBS)
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index a089d0bd..04c5f635 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -54,10 +54,12 @@ extern "C" {
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/vfs.h>
-#include <unistd.h>
 #include <fcntl.h>
 #include <netdb.h>
 
+#ifdef ENABLE_ONE_SHOT
+#include <json-c/json.h>
+#endif
 
 /* If fts.h is included before config.h, its indirect inclusions may not
    give us the right LFS aliases of these functions, so map them manually.  */
@@ -387,6 +389,8 @@ static const struct argp_option options[] =
    { "passive", ARGP_KEY_PASSIVE, NULL, 0, "Do not scan or groom, read-only database.", 0 },
 #define ARGP_KEY_DISABLE_SOURCE_SCAN 0x1009
    { "disable-source-scan", ARGP_KEY_DISABLE_SOURCE_SCAN, NULL, 0, "Do not scan dwarf source info.", 0 },
+#define ARGP_KEY_ONE_SHOT 0x100a
+   { "one-shot", ARGP_KEY_ONE_SHOT, "NUM", 0, "For the first scan, output found buildids/paths to given file descriptor", 0 },
    { NULL, 0, NULL, 0, NULL, 0 },
   };
 
@@ -439,6 +443,8 @@ static unsigned forwarded_ttl_limit = 8;
 static bool scan_source_info = true;
 static string tmpdir;
 static bool passive_p = false;
+static int one_shot_fd = -1;
+static mutex one_shot_mtx;
 
 static void set_metric(const string& key, double value);
 // static void inc_metric(const string& key);
@@ -639,6 +645,16 @@ parse_opt (int key, char *arg,
         // other conflicting options tricky to check
         argp_failure(state, 1, EINVAL, "inconsistent options with passive mode");
       break;
+    case ARGP_KEY_ONE_SHOT:
+      one_shot_fd = atol(arg);
+      #ifdef ENABLE_ONE_SHOT
+        if (one_shot_fd < 0)
+          argp_failure(state, 1, EINVAL, "one-shot fd");
+      #else
+        argp_failure(state, 1, EINVAL, "Debuginfod in one-shot mode requires libjson-c");
+      #endif
+
+      break;
     case ARGP_KEY_DISABLE_SOURCE_SCAN:
       scan_source_info = false;
       break;
@@ -3110,6 +3126,31 @@ archive_classify (const string& rps, string& archive_extension,
                            << (debuginfo_p ? "D" : "")
                            << " sourcefiles=" << sourcefiles.size() << endl;
 
+          #ifdef ENABLE_ONE_SHOT
+          if(one_shot_fd >= 0 && executable_p && "" != fn && "" != buildid){
+            unique_lock<mutex> lock(one_shot_mtx);
+            struct debuginfod_oneshot_header hdr;
+            hdr.magic = debuginfod_oneshot_header_magic;
+            string json_path = "\"path\": \""+fn+"\"";
+            string json_buildid = " \"buildid\": \""+buildid+"\"";
+
+            // Converts the strings to a properly escaped JSON string
+            json_object *json = json_tokener_parse((
+              "{"+
+              json_path+","+
+              json_buildid+
+              "}"
+            ).c_str());
+	        const char* payload = json_object_to_json_string(json);
+
+          hdr.payload_len = strlen(payload)+1;
+          if( NULL == payload ||
+              sizeof(hdr)               != write(one_shot_fd, &hdr, sizeof(hdr)) ||
+              (ssize_t) hdr.payload_len != write(one_shot_fd, payload, hdr.payload_len) )
+              if (verbose) obatched(clog) << "There was an issue writing the path+buildid in one-shot mode" << endl;
+          json_object_put(json);
+         }
+         #endif
         }
       catch (const reportable_exception& e)
         {
@@ -3470,6 +3511,15 @@ thread_main_fts_source_paths (void* arg)
       scanq.done_idle(); // release the hounds
       if (interrupted) break;
 
+      #ifdef ENABLE_ONE_SHOT
+      // If doing a one-shot pass, close the pipe once the first scan is done
+      // Then resume the normal state
+      if(last_rescan != 0 && one_shot_fd >= 0){
+        if(one_shot_fd > 2) close(one_shot_fd);
+        one_shot_fd = -1;
+      }
+      #endif
+
       time_t now = time(NULL);
       bool rescan_now = false;
       if (last_rescan == 0) // at least one initial rescan is documented even for -t0
diff --git a/debuginfod/debuginfod.h.in b/debuginfod/debuginfod.h.in
index c358df4d..3d0c0d49 100644
--- a/debuginfod/debuginfod.h.in
+++ b/debuginfod/debuginfod.h.in
@@ -103,5 +103,14 @@ void debuginfod_end (debuginfod_client *client);
 }
 #endif
 
+/* The message format used by debuginfod one-shot mode */
+#include <sys/types.h>
+static const uint debuginfod_oneshot_header_magic = 0xdeb91f0d;
+struct debuginfod_oneshot_header{
+    uint magic;
+    size_t payload_len;
+    char payload[0];
+};
+
 
 #endif /* _DEBUGINFOD_CLIENT_H */
diff --git a/doc/ChangeLog b/doc/ChangeLog
index ceec2467..201a0bdb 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2022-08-09  Ryan Goldberg <rgoldber@redhat.com>
+
+	* debuginfod.8 (--one-shot): Document new flag & operation mode.
+
 2022-06-03  Michael Trapp <michael.trapp@sap.com>
 
 	* debuginfod.8 (--disable-source-scan): Document.
diff --git a/doc/debuginfod.8 b/doc/debuginfod.8
index 50fce7f5..a905a15d 100644
--- a/doc/debuginfod.8
+++ b/doc/debuginfod.8
@@ -279,6 +279,22 @@ Disable scan of the dwarf source info of debuginfo sections.
 If a setup has no access to source code, the source info is not
 required.
 
+.TP
+.B "\-\-one\-shot=FD"
+For the first scan, atomically write the found executables' buildids and
+canonicalized paths in JSON format to the given file descriptor FD.
+The format for each write is as follows:
+.TS
+tab(;) allbox;
+cf s s.
+Magic number (uint) = 0xdeb91f0d
+.T&
+l,al.
+JSON string length (size_t); JSON string (char[])
+.TE
+
+Once completed, FD will be closed and debuginfod will return to the normal state.
+
 .TP
 .B "\-v"
 Increase verbosity of logging to the standard error file descriptor.
diff --git a/tests/ChangeLog b/tests/ChangeLog
index d2952cc9..ee3521af 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,9 @@
+2022-08-09  Ryan Goldberg <rgoldber@redhat.com>
+
+	* run-debuginfod-one-shot.sh: New test.
+	* Makefile.am (TESTS): Add run-debuginfod-one-shot.sh.
+	(EXTRA_DIST): Add run-debuginfod-one-shot.sh
+
 2022-08-04  Sergei Trofimovich  <slyich@gmail.com>
 
 	* low_high_pc.c (handle_die): Drop redundant 'lx' suffix.
@@ -10,7 +16,6 @@
 
 	* run-readelf-Dd.sh: New test.
 	* Makefile.am (TESTS): Add run-readelf-Dd.sh.
-	(EXTRA_DIST): Likewise.
 
 2022-06-01  Mark Wielaard  <mark@klomp.org>
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 87988fb9..20dc2680 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -244,7 +244,8 @@ TESTS += run-debuginfod-dlopen.sh \
          run-debuginfod-x-forwarded-for.sh \
          run-debuginfod-response-headers.sh \
          run-debuginfod-extraction-passive.sh \
-	 run-debuginfod-webapi-concurrency.sh
+	 run-debuginfod-webapi-concurrency.sh \
+	 run-debuginfod-one-shot.sh
 endif
 if !OLD_LIBMICROHTTPD
 # Will crash on too old libmicrohttpd
@@ -252,7 +253,6 @@ if !OLD_LIBMICROHTTPD
 TESTS += run-debuginfod-federation-metrics.sh
 endif
 endif
-
 EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
 	     run-show-die-info.sh run-get-files.sh run-get-lines.sh \
 	     run-next-files.sh run-next-lines.sh testfile-only-debug-line.bz2 \
@@ -549,6 +549,7 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
 	     run-debuginfod-response-headers.sh \
              run-debuginfod-extraction-passive.sh \
              run-debuginfod-webapi-concurrency.sh \
+			 run-debuginfod-one-shot.sh \
 	     debuginfod-rpms/fedora30/hello2-1.0-2.src.rpm \
 	     debuginfod-rpms/fedora30/hello2-1.0-2.x86_64.rpm \
 	     debuginfod-rpms/fedora30/hello2-debuginfo-1.0-2.x86_64.rpm \
diff --git a/tests/run-debuginfod-one-shot.sh b/tests/run-debuginfod-one-shot.sh
new file mode 100755
index 00000000..185f2e27
--- /dev/null
+++ b/tests/run-debuginfod-one-shot.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2019-2021 Red Hat, Inc.
+# This file is part of elfutils.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# elfutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. $srcdir/debuginfod-subr.sh
+
+DB=${PWD}/.debuginfod_tmp.sqlite
+tempfiles $DB
+
+binoutput="one-shot-output.bin"
+tempfiles $binoutput
+exec 3<>$binoutput # Open FD=3 for RW
+
+# This variable is essential and ensures no time-race for claiming ports occurs
+# set base to a unique multiple of 100 not used in any other 'run-debuginfod-*' test
+base=13100
+get_ports
+mkdir R
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -R \
+    -d $DB -p $PORT1 -t0 -g0 R > vlog$PORT1 2>&1 --one-shot 3 &
+PID1=$!
+tempfiles vlog$PORT1
+errfiles vlog$PORT1
+
+########################################################################
+cp -rvp ${abs_srcdir}/debuginfod-rpms/rhel7 R
+
+correct_output=("/usr/local/bin/hello2:bc1febfd03ca05e030f0d205f7659db29f8a4b30" \
+                "/usr/local/bin/hello:f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2")
+MAGIC="0xdeb91f0d"
+
+cat << EoF > parseData.cxx
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include "${abs_top_builddir}/debuginfod/debuginfod.h"
+#include <stdio.h>
+#include <json-c/json.h>
+
+int main(){
+    int fd = open("$binoutput", 0);
+    if(fd < 0) return 1;
+
+    struct debuginfod_oneshot_header hdr;
+
+    int hdr_sze;
+    while(0 != (hdr_sze = read(fd, &hdr, sizeof(hdr)))){
+        char* payload  = (char*)malloc(hdr.payload_len);
+        if(hdr_sze   != sizeof(hdr)     ||
+           hdr.magic != $MAGIC          ||
+           (ssize_t)hdr.payload_len  != read(fd, payload, hdr.payload_len))
+            return 1;
+        json_object *json = json_tokener_parse(payload);
+        json_object *json_field;
+        if(!json_object_object_get_ex(json, "path", &json_field)) return 1;
+        printf("%s:",json_object_get_string(json_field));
+        if(!json_object_object_get_ex(json, "buildid", &json_field)) return 1;
+        printf("%s\n",json_object_get_string(json_field));
+        free(payload);
+        json_object_put(json);
+    }
+    close(fd);
+    return 0;
+}
+EoF
+tempfiles parseData.cxx parseData.o tempfiles parseData
+g++ -o  parseData -l json-c parseData.cxx || { echo "Can't compile test program"; exit 77; }; # Skip on compilation faliure (probably missing libjson-c.so)
+
+########################################################################
+# Server must become ready with R fully scanned and indexed
+wait_ready $PORT1 'ready' 1
+wait_ready $PORT1 'thread_work_total{role="traverse"}' 1
+wait_ready $PORT1 'thread_work_pending{role="scan"}' 0
+wait_ready $PORT1 'thread_busy{role="scan"}' 0
+
+exec 3>&- # close fd
+tempfiles output.txt
+# Check the header format is correct
+testrun ./parseData > output.txt
+
+cat output.txt
+
+# Check that each of the output lines corresponds to one of the correct outputs
+while read -r line; do
+    printf '%s\0' "${correct_output[@]}" | grep -Fxqz -- $line;
+done < output.txt
+
+kill $PID1
+wait $PID1
+PID1=0
+exit 0
-- 
2.37.1

