Re: [PATCH v2] PR12331: Introduce stap options --sysroot and --sysenv

[Wade Farnsworth <wade_farnsworth@mentor.com>]

Hi all,

Anyone have a chance to review this yet?

Thanks in advance!

-Wade

Wade Farnsworth wrote:
> This adds new command-line options to facilitate different file locations at
> compile-time versus run-time when generating stap kernel modules for a remote
> system:
>
> 1. --sysroot=DIR
>     Specifies a separate filesystem for the remote system on the local
>     system.  The following stap functionalities will make use of this:
>
>     a. When compiling process and library probes with a remote filesystem path
>        (e.g. process("/bin/foo") with --sysroot=/bar/baz).  In this case symbols
>        will be taken from the local filesystem (/bar/baz/bin/foo), but the
>        resulting .ko will still contain paths on the remote filesystem
>        (/bin/foo).
>
>     b. When passing a kernel release with -r (not a full path), the kernel
>        build directory will be expected in the sysroot.
>
>     c. When DWARF debug info is contained in a separate file from the
>        executable, the sysroot is used to find the debug info files.
>
>     d. All calls to find_executable() search the path passed in the
>        argument env_path relative to the sysroot.  For example if
>        PATH=/bin/foo, and --sysroot=/bar/baz is provided, then
>        find_executable() called with env_path == "PATH" will search
>        /bar/baz/bin/foo.
>
>     e. stap attempts to detect if a path in a probe would be outside of the
>        sysroot and errors appropriately.  This situation may occur, for
>        example, if the path contains several ".." directories.
>
> 2. --sysenv=VAR=VALUE
>     Specifies a different environment variable for the remote system as opposed
>     to the local one, and that the value on the remote system should be used.
>     Currently only valid env_path arguments to find_executable() are handled
>     (i.e. PATH, LD_LIBRARY_PATH).  If --sysroot is provided and --sysenv
>     is omitted, then the local environment relative to the sysroot will
>     be used.
>
> These options are disabled for systemtap clients.
>
> Signed-off-by: Wade Farnsworth<wade_farnsworth@mentor.com>
> ---
>
> Josh, I believe I have implemented all of your comments on the previous version
> of this patch.  Any further suggestions are welcome!
>
> -v2: Implemented several improvements per Josh Stone's comment.
>
>   cmdline.cxx       |    2 +
>   cmdline.h         |    2 +
>   dwflpp.cxx        |    4 +-
>   hash.cxx          |    3 +-
>   session.cxx       |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
>   session.h         |    3 ++
>   setupdwfl.cxx     |   39 ++++++++++++++++++++++++++----
>   setupdwfl.h       |    3 +-
>   tapset-itrace.cxx |    5 +++-
>   tapset-utrace.cxx |    5 +++-
>   tapsets.cxx       |   64 ++++++++++++++++++++++++++++++++++++++------------
>   translate.cxx     |    6 ++--
>   util.cxx          |   28 +++++++++++++++++----
>   util.h            |    3 ++
>   14 files changed, 197 insertions(+), 37 deletions(-)
>
> diff --git a/cmdline.cxx b/cmdline.cxx
> index 88e02f9..881a5c3 100644
> --- a/cmdline.cxx
> +++ b/cmdline.cxx
> @@ -50,5 +50,7 @@ struct option stap_long_options[] = {
>     { "privilege", 1,&stap_long_opt, LONG_OPT_PRIVILEGE },
>     { "suppress-handler-errors", 0,&stap_long_opt, LONG_OPT_SUPPRESS_HANDLER_ERRORS },
>     { "modinfo", 1,&stap_long_opt, LONG_OPT_MODINFO },
> +  { "sysroot", 1,&stap_long_opt, LONG_OPT_SYSROOT },
> +  { "sysenv", 1,&stap_long_opt, LONG_OPT_SYSENV },
>     { NULL, 0, NULL, 0 }
>   };
> diff --git a/cmdline.h b/cmdline.h
> index 3229a7b..680f290 100644
> --- a/cmdline.h
> +++ b/cmdline.h
> @@ -44,6 +44,8 @@ extern "C" {
>   #define LONG_OPT_PRIVILEGE 28
>   #define LONG_OPT_SUPPRESS_HANDLER_ERRORS 29
>   #define LONG_OPT_MODINFO 30
> +#define LONG_OPT_SYSROOT 31
> +#define LONG_OPT_SYSENV 32
>
>   // NB: when adding new options, consider very carefully whether they
>   // should be restricted from stap clients (after --client-options)!
> diff --git a/dwflpp.cxx b/dwflpp.cxx
> index 4d78187..17fd5c6 100644
> --- a/dwflpp.cxx
> +++ b/dwflpp.cxx
> @@ -321,7 +321,7 @@ dwflpp::setup_kernel(const string&  name, systemtap_session&  s, bool debuginfo_n
>       {
>         if (debuginfo_needed) {
>           // Suggest a likely kernel dir to find debuginfo rpm for
> -        string dir = string("/lib/modules/" + sess.kernel_release );
> +        string dir = string(sess.sysroot + "/lib/modules/" + sess.kernel_release );
>           find_debug_rpms(sess, dir.c_str());
>         }
>         throw semantic_error (_F("missing %s kernel/module debuginfo under '%s'",
> @@ -359,7 +359,7 @@ dwflpp::setup_kernel(const vector<string>  &names, bool debuginfo_needed)
>       {
>         if (debuginfo_needed) {
>           // Suggest a likely kernel dir to find debuginfo rpm for
> -        string dir = string("/lib/modules/" + sess.kernel_release );
> +        string dir = string(sess.sysroot + "/lib/modules/" + sess.kernel_release );
>           find_debug_rpms(sess, dir.c_str());
>         }
>         throw semantic_error (_F("missing %s kernel/module debuginfo under '%s'",
> diff --git a/hash.cxx b/hash.cxx
> index c435a03..69334f5 100644
> --- a/hash.cxx
> +++ b/hash.cxx
> @@ -133,6 +133,7 @@ void create_hash_log(const string&type_str, const string&parms, const string&
>   static const hash&
>   get_base_hash (systemtap_session&  s)
>   {
> +  map<string, char*>  dummy;
>     if (s.base_hash)
>       return *s.base_hash;
>
> @@ -158,7 +159,7 @@ get_base_hash (systemtap_session&  s)
>     // Hash compiler path, size, and mtime.  We're just going to assume
>     // we'll be using gcc. XXX: getting kbuild to spit out out would be
>     // better, especially since this is fooled by ccache.
> -  h.add_path("Compiler ", find_executable("gcc"));
> +  h.add_path("Compiler ", find_executable("gcc", "", dummy));
>
>     // Hash the systemtap size and mtime.  We could use VERSION/DATE,
>     // but when developing systemtap that doesn't work well (since you
> diff --git a/session.cxx b/session.cxx
> index 24c329a..2c5be33 100644
> --- a/session.cxx
> +++ b/session.cxx
> @@ -20,6 +20,7 @@
>   #include "util.h"
>   #include "cmdline.h"
>   #include "git_version.h"
> +#include "setupdwfl.h"
>
>   #include<cerrno>
>   #include<cstdlib>
> @@ -86,6 +87,8 @@ systemtap_session::systemtap_session ():
>     kernel_release = string (buf.release);
>     release = kernel_release;
>     kernel_build_tree = "/lib/modules/" + kernel_release + "/build";
> +  sysroot = "";
> +  update_release_sysroot = false;
>     architecture = machine = normalize_machine(buf.machine);
>
>     for (unsigned i=0; i<5; i++) perpass_verbose[i]=0;
> @@ -533,6 +536,13 @@ systemtap_session::usage (int exitcode)
>       "              yes,no,ask,<timeout value>\n"
>       "   --dump-probe-types\n"
>       "              show a list of available probe types.\n"
> +    "   --sysroot=DIR\n"
> +    "              specify sysroot directory where executables are located.\n"
> +    "   --sysenv=VAR=VALUE\n"
> +    "              provide an alternate value for an environment variable\n"
> +    "              where the value on a remote system differs.  Path\n"
> +    "              variables (e.g. PATH, LD_LIBRARY_PATH) are assumed to be\n"
> +    "              relatve to the sysroot.\n"
>       , compatible.c_str())<<  endl
>     ;
>
> @@ -1097,6 +1107,56 @@ systemtap_session::parse_cmdline (int argc, char * const argv [])
>                 modinfos.push_back (string(optarg));
>                 break;
>
> +            case LONG_OPT_SYSROOT:
> +              if (client_options) {
> +                  cerr<<  _F("ERROR: %s invalid with %s", "--sysroot", "--client-options")<<  endl;
> +                  return 1;
> +              } else {
> +                  const char *spath = canonicalize_file_name (optarg);
> +                  if (spath == NULL) {
> +                      cerr<<  _F("ERROR: %s is an invalid directory for --sysroot", optarg)<<  endl;
> +                      return 1;
> +                  }
> +
> +                  sysroot = string(spath);
> +                  if (sysroot[sysroot.size() - 1] != '/')
> +                      sysroot.append("/");
> +
> +                  if (update_release_sysroot)
> +                      kernel_build_tree = sysroot + kernel_build_tree;
> +
> +                  debuginfo_path_insert_sysroot(sysroot);
> +
> +                  break;
> +              }
> +            case LONG_OPT_SYSENV:
> +              if (client_options) {
> +                  cerr<<  _F("ERROR: %s invalid with %s", "--sysenv", "--client-options")<<  endl;
> +                  return 1;
> +              } else {
> +                  string sysenv_str = optarg;
> +                  string value_str;
> +		  char * value;
> +                  size_t pos;
> +                  if (sysroot.empty()) {
> +                      cerr<<  "ERROR: --sysenv must follow --sysroot"<<  endl;
> +                      return 1;
> +                  }
> +
> +                  pos = sysenv_str.find("=");
> +                  if (pos == string::npos) {
> +                      cerr<<  _F("ERROR: %s is an invalid argument for --sysenv", optarg)<<  endl;
> +                      return 1;
> +                  }
> +
> +                  value_str = sysenv_str.substr(pos + 1);
> +		  value = new char[value_str.size()+1];
> +		  strcpy(value, value_str.c_str());
> +                  sysenv[sysenv_str.substr(0, pos)] = value;
> +
> +                  break;
> +              }
> +
>               default:
>                 // NOTREACHED unless one added a getopt option but not a corresponding switch/case:
>                 cerr<<  _F("Unhandled long argument id %d", stap_long_opt)<<  endl;
> @@ -1387,8 +1447,13 @@ systemtap_session::setup_kernel_release (const char* kstr)
>     else
>       {
>         kernel_release = string (kstr);
> -      if (!kernel_release.empty())
> +      if (!kernel_release.empty()) {
>           kernel_build_tree = "/lib/modules/" + kernel_release + "/build";
> +        if (sysroot.empty())
> +          update_release_sysroot = true;
> +        else
> +          kernel_build_tree = sysroot + kernel_build_tree;
> +      }
>
>         // PR10745
>         // Let's not look for the kernel_source_tree; it's definitely
> diff --git a/session.h b/session.h
> index e128b2f..b5b8d82 100644
> --- a/session.h
> +++ b/session.h
> @@ -140,6 +140,9 @@ public:
>     std::string kernel_base_release;
>     std::string kernel_build_tree;
>     std::string kernel_source_tree;
> +  std::string sysroot;
> +  std::map<std::string,char *>  sysenv;
> +  bool update_release_sysroot;
>     std::map<std::string,std::string>  kernel_config;
>     std::set<std::string>  kernel_exports;
>     std::string machine;
> diff --git a/setupdwfl.cxx b/setupdwfl.cxx
> index 82bc42a..fbffaf0 100644
> --- a/setupdwfl.cxx
> +++ b/setupdwfl.cxx
> @@ -39,14 +39,14 @@ extern "C" {
>   // XXX: also consider adding $HOME/.debug/ for perf build-id-cache
>   static const char *debuginfo_path_arr = "+:.debug:/usr/lib/debug:/var/cache/abrt-di/usr/lib/debug:build";
>   static const char *debuginfo_env_arr = getenv("SYSTEMTAP_DEBUGINFO_PATH");
> -static const char *debuginfo_path = (debuginfo_env_arr ?: debuginfo_path_arr);
> +static char *debuginfo_path = (char *)(debuginfo_env_arr ?: debuginfo_path_arr);
>
>   // NB: kernel_build_tree doesn't enter into this, as it's for
>   // kernel-side modules only.
>   // XXX: also consider adding $HOME/.debug/ for perf build-id-cache
>   static const char *debuginfo_usr_path_arr = "+:.debug:/usr/lib/debug:/var/cache/abrt-di/usr/lib/debug";
> -static const char *debuginfo_usr_path = (debuginfo_env_arr
> -					 ?: debuginfo_usr_path_arr);
> +static char *debuginfo_usr_path = (char *)(debuginfo_env_arr
> +					   ?: debuginfo_usr_path_arr);
>
>   // A pointer to the current systemtap session for use only by a few
>   // dwfl calls. DO NOT rely on this, as it is cleared after use.
> @@ -146,7 +146,10 @@ setup_mod_deps()
>       }
>     else
>       {
> -      modulesdep = "/lib/modules/";
> +      string sysroot = "";
> +      if (current_session_for_find_debuginfo)
> +        sysroot = current_session_for_find_debuginfo->sysroot;
> +      modulesdep = sysroot + "/lib/modules/";
>         modulesdep += elfutils_kernel_path;
>         modulesdep += "/modules.dep";
>       }
> @@ -280,6 +283,30 @@ setup_dwfl_report_kernel_p(const char* modname, const char* filename)
>       }
>   }
>
> +static char * path_insert_sysroot(string sysroot, string path)
> +{
> +  char * path_new;
> +  size_t pos = 1;
> +  if (path[0] == '/')
> +    path.replace(0, 1, sysroot);
> +  while (true) {
> +    pos = path.find(":/", pos);
> +    if (pos == string::npos)
> +      break;
> +    path.replace(pos, 2, ":" + sysroot);
> +    ++pos;
> +  }
> +  path_new = new char[path.size()+1];
> +  strcpy (path_new, path.c_str());
> +  return path_new;
> +}
> +
> +void debuginfo_path_insert_sysroot(string sysroot)
> +{
> +  debuginfo_path = path_insert_sysroot(sysroot, debuginfo_path);
> +  debuginfo_usr_path = path_insert_sysroot(sysroot, debuginfo_usr_path);
> +}
> +
>   static DwflPtr
>   setup_dwfl_kernel (unsigned *modules_found, systemtap_session&s)
>   {
> @@ -295,7 +322,7 @@ setup_dwfl_kernel (unsigned *modules_found, systemtap_session&s)
>     // no way to set the dwfl_callback.debuginfo_path and always
>     // passs the plain kernel_release here.  So instead we have to
>     // hard-code this magic here.
> -  if (s.kernel_build_tree == string("/lib/modules/"
> +  if (s.kernel_build_tree == string(s.sysroot + "/lib/modules/"
>   				    + s.kernel_release
>   				    + "/build"))
>       elfutils_kernel_path = s.kernel_release;
> @@ -413,7 +440,7 @@ setup_dwfl_kernel(const std::set<std::string>  &names,
>   }
>
>   DwflPtr
> -setup_dwfl_user(const std::string&name)
> +setup_dwfl_user(const std::string&name, systemtap_session&s)
>   {
>     if (user_dwfl != NULL
>         &&  user_modset.size() == 1
> diff --git a/setupdwfl.h b/setupdwfl.h
> index 48f9105..4686574 100644
> --- a/setupdwfl.h
> +++ b/setupdwfl.h
> @@ -50,7 +50,7 @@ DwflPtr setup_dwfl_kernel(const std::set<std::string>  &names,
>   			  unsigned *found,
>   			  systemtap_session&s);
>
> -DwflPtr setup_dwfl_user(const std::string&name);
> +DwflPtr setup_dwfl_user(const std::string&name, systemtap_session&s);
>   DwflPtr setup_dwfl_user(std::vector<std::string>::const_iterator&begin,
>   		        const std::vector<std::string>::const_iterator&end,
>   		        bool all_needed, systemtap_session&s);
> @@ -69,5 +69,6 @@ int internal_find_debuginfo (Dwfl_Module *mod,
>   int execute_abrt_action_install_debuginfo_to_abrt_cache (std::string hex);
>   std::string get_kernel_build_id (systemtap_session&s);
>   int download_kernel_debuginfo (systemtap_session&s, std::string hex);
> +void debuginfo_path_insert_sysroot(std::string sysroot);
>
>   #endif
> diff --git a/tapset-itrace.cxx b/tapset-itrace.cxx
> index 1a9bf67..0e623fe 100644
> --- a/tapset-itrace.cxx
> +++ b/tapset-itrace.cxx
> @@ -114,8 +114,11 @@ struct itrace_builder: public derived_probe_builder
>       // If we have a path, we need to validate it.
>       if (has_path)
>         {
> -        path = find_executable (path);
> +        size_t pos;
> +        path = find_executable (path, sess.sysroot, sess.sysenv);
>           sess.unwindsym_modules.insert (path);
> +        if (!sess.sysroot.empty()&&  ((pos = path.find(sess.sysroot)) != string::npos))
> +          path.replace(pos, sess.sysroot.length(), "/");
>         }
>
>       finished_results.push_back(new itrace_derived_probe(sess, base, location,
> diff --git a/tapset-utrace.cxx b/tapset-utrace.cxx
> index 64ba900..133d42a 100644
> --- a/tapset-utrace.cxx
> +++ b/tapset-utrace.cxx
> @@ -679,8 +679,11 @@ struct utrace_builder: public derived_probe_builder
>         }
>       else if (has_path)
>         {
> -        path = find_executable (path);
> +        size_t pos;
> +        path = find_executable (path, sess.sysroot, sess.sysenv);
>           sess.unwindsym_modules.insert (path);
> +	if (!sess.sysroot.empty()&&  ((pos = path.find(sess.sysroot)) != string::npos))
> +	  path.replace(pos, sess.sysroot.length(), "/");
>         }
>       else if (has_pid)
>         {
> diff --git a/tapsets.cxx b/tapsets.cxx
> index cb56170..6cdb8e1 100644
> --- a/tapsets.cxx
> +++ b/tapsets.cxx
> @@ -584,13 +584,14 @@ base_query::base_query(dwflpp&  dw, literal_map_t const&  params):
>         has_statement = get_number_param(params, TOK_STATEMENT, statement_num_val);
>
>         if (has_process)
> -        module_val = find_executable (module_val);
> +        module_val = find_executable (module_val, sess.sysroot, sess.sysenv);
>         if (has_library)
>           {
>             if (! contains_glob_chars (library_name))
>               {
>                 path = module_val;
> -              module_val = find_executable (library_name, "LD_LIBRARY_PATH");
> +              module_val = find_executable (library_name, sess.sysroot,
> +                                            sess.sysenv, "LD_LIBRARY_PATH");
>               }
>             else
>               path = library_name;
> @@ -1137,7 +1138,7 @@ dwarf_query::add_probe_point(const string&  dw_funcname,
>   {
>     string reloc_section; // base section for relocation purposes
>     Dwarf_Addr reloc_addr; // relocated
> -  const string&  module = dw.module_name; // "kernel" or other
> +  string&  module = dw.module_name; // "kernel" or other
>     string funcname = dw_funcname;
>
>     assert (! has_absolute); // already handled in dwarf_builder::build()
> @@ -1188,6 +1189,14 @@ dwarf_query::add_probe_point(const string&  dw_funcname,
>
>         if (has_process)
>           {
> +	  if (!sess.sysroot.empty())
> +            {
> +              size_t pos;
> +              if ((pos = module.find(sess.sysroot)) != string::npos)
> +                module.replace(pos, sess.sysroot.length(), "/");
> +              if ((pos = path.find(sess.sysroot)) != string::npos)
> +                path.replace(pos, sess.sysroot.length(), "/");
> +            }
>             results.push_back (new uprobe_derived_probe(funcname, filename, line,
>                                                         module, reloc_section, addr, reloc_addr,
>                                                         *this, scope_die));
> @@ -2051,7 +2060,8 @@ query_one_library (const char *library, dwflpp&  dw,
>   {
>     if (dw.function_name_matches_pattern(library, user_lib))
>       {
> -      string library_path = find_executable (library, "LD_LIBRARY_PATH");
> +      string library_path = find_executable (library, "", dw.sess.sysenv,
> +                                             "LD_LIBRARY_PATH");
>         probe_point* specific_loc = new probe_point(*base_loc);
>         specific_loc->optional = true;
>         vector<probe_point::component*>  derived_comps;
> @@ -3906,7 +3916,7 @@ void dwarf_cast_expanding_visitor::visit_cast_op (cast_op* e)
>   	    }
>   	  else
>   	    {
> -	      module = find_executable (module); // canonicalize it
> +	      module = find_executable (module, "", s.sysenv); // canonicalize it
>   	      dw = db.get_user_dw(s, module);
>   	    }
>   	}
> @@ -6546,13 +6556,14 @@ dwarf_builder::build(systemtap_session&  sess,
>       }
>     else if (get_param (parameters, TOK_PROCESS, module_name) || has_null_param(parameters, TOK_PROCESS))
>         {
> +      module_name = sess.sysroot + module_name;
>         if(has_null_param(filled_parameters, TOK_PROCESS))
>           {
>             wordexp_t words;
>             int rc = wordexp(sess.cmd.c_str(),&words, WRDE_NOCMD|WRDE_UNDEF);
>             if(rc || words.we_wordc<= 0)
>               throw semantic_error(_("unspecified process probe is invalid without a -c COMMAND"));
> -          module_name = words.we_wordv[0];
> +          module_name = sess.sysroot + words.we_wordv[0];
>             filled_parameters[TOK_PROCESS] = new literal_string(module_name);// this needs to be used in place of the blank map
>             // in the case of TOK_MARK we need to modify locations as well
>             if(location->components[0]->functor==TOK_PROCESS&&
> @@ -6605,9 +6616,15 @@ dwarf_builder::build(systemtap_session&  sess,
>                     if (sess.verbose>  1)
>                       clog<<  _F("Expanded process(\"%s\") to process(\"%s\")",
>                                  module_name.c_str(), eglobbed.c_str())<<  endl;
> +                  if (!sess.sysroot.empty())
> +                    {
> +                      size_t pos;
> +                      if ((pos = eglobbed.find(sess.sysroot)) != string::npos)
> +                        eglobbed.replace(pos, sess.sysroot.length(), "/");
> +                    }
>
>                     probe_point::component* ppc = new probe_point::component (TOK_PROCESS,
> -                                                                            new literal_string (eglobbed));
> +                                                    new literal_string (eglobbed));
>                     ppc->tok = location->components[0]->tok; // overwrite [0] slot, pattern matched above
>                     pp->components[0] = ppc;
>
> @@ -6633,7 +6650,7 @@ dwarf_builder::build(systemtap_session&  sess,
>
>         // PR13338: unquote glob results
>         module_name = unescape_glob_chars (module_name);
> -      user_path = find_executable (module_name); // canonicalize it
> +      user_path = find_executable (module_name, "", sess.sysenv); // canonicalize it
>
>         // if the executable starts with "#!", we look for the interpreter of the script
>         {
> @@ -6679,12 +6696,13 @@ dwarf_builder::build(systemtap_session&  sess,
>                       if (p3 != string::npos)
>                       {
>                          string env_path = path.substr(p3);
> -                       user_path = find_executable (env_path);
> +                       user_path = find_executable (env_path, sess.sysroot,
> +                                                    sess.sysenv);
>                       }
>                   }
>                   else
>                   {
> -                   user_path = find_executable (path);
> +                  user_path = find_executable (path, sess.sysroot, sess.sysenv);
>                   }
>
>                   struct stat st;
> @@ -6705,8 +6723,12 @@ dwarf_builder::build(systemtap_session&  sess,
>
>                     // synthesize a new probe_point, with the expanded string
>                     probe_point *pp = new probe_point (*location);
> +                  string user_path_tgt = user_path;
> +                  size_t pos;
> +                  if (!sess.sysroot.empty()&&  ((pos = path.find(sess.sysroot)) != string::npos))
> +                    user_path_tgt.replace(pos, sess.sysroot.length(), "/");
>                     probe_point::component* ppc = new probe_point::component (TOK_PROCESS,
> -                                                                            new literal_string (user_path.c_str()));
> +                                                                            new literal_string (user_path_tgt.c_str()));
>                     ppc->tok = location->components[0]->tok; // overwrite [0] slot, pattern matched above
>                     pp->components[0] = ppc;
>
> @@ -6725,7 +6747,8 @@ dwarf_builder::build(systemtap_session&  sess,
>
>         if(get_param (parameters, TOK_LIBRARY, user_lib)
>   	&&  user_lib.length()&&  ! contains_glob_chars (user_lib))
> -	module_name = find_executable (user_lib, "LD_LIBRARY_PATH");
> +	module_name = find_executable (user_lib, sess.sysroot, sess.sysenv,
> +                                       "LD_LIBRARY_PATH");
>         else
>   	module_name = user_path; // canonicalize it
>
> @@ -8189,7 +8212,7 @@ struct kprobe_builder: public derived_probe_builder
>
>
>   void
> -kprobe_builder::build(systemtap_session&,
> +kprobe_builder::build(systemtap_session&  sess,
>   		      probe * base,
>   		      probe_point * location,
>   		      literal_map_t const&  parameters,
> @@ -8212,9 +8235,20 @@ kprobe_builder::build(systemtap_session&,
>     has_library = get_param (parameters, TOK_LIBRARY, library);
>
>     if (has_path)
> -    path = find_executable (path);
> +    {
> +      size_t pos;
> +      path = find_executable (path, sess.sysroot, sess.sysenv);
> +      if (!sess.sysroot.empty()&&  ((pos = path.find(sess.sysroot)) != string::npos))
> +        path.replace(pos, sess.sysroot.length(), "/");
> +    }
>     if (has_library)
> -    library = find_executable (library, "LD_LIBRARY_PATH");
> +    {
> +      size_t pos;
> +      library = find_executable (library, sess.sysroot, sess.sysenv,
> +                                 "LD_LIBRARY_PATH");
> +      if (!sess.sysroot.empty()&&  ((pos = library.find(sess.sysroot)) != string::npos))
> +        library.replace(pos, sess.sysroot.length(), "/");
> +    }
>
>     if (has_function_str)
>       {
> diff --git a/translate.cxx b/translate.cxx
> index 97847a8..70c4751 100644
> --- a/translate.cxx
> +++ b/translate.cxx
> @@ -6242,10 +6242,10 @@ add_unwindsym_vdso (systemtap_session&s)
>     // having the BUILDDIR we need to do a deep search (the specific
>     // arch name dir in the kernel build tree is unknown).
>     string vdso_dir;
> -  if (s.kernel_build_tree == string("/lib/modules/"
> +  if (s.kernel_build_tree == string(s.sysroot + "/lib/modules/"
>   				    + s.kernel_release
>   				    + "/build"))
> -    vdso_dir = "/lib/modules/" + s.kernel_release + "/vdso";
> +    vdso_dir = s.sysroot + "/lib/modules/" + s.kernel_release + "/vdso";
>     else
>       vdso_dir = s.kernel_build_tree + "/arch/";
>
> @@ -6353,7 +6353,7 @@ emit_symbol_data (systemtap_session&  s)
>         string modname = *it;
>         assert (modname.length() != 0);
>         if (! is_user_module (modname)) continue;
> -      DwflPtr dwfl_ptr = setup_dwfl_user (modname);
> +      DwflPtr dwfl_ptr = setup_dwfl_user (modname, s);
>         Dwfl *dwfl = dwfl_ptr.get()->dwfl;
>         if (dwfl != NULL) // tolerate missing data; will warn below
>           {
> diff --git a/util.cxx b/util.cxx
> index 93cf12a..4158f0d 100644
> --- a/util.cxx
> +++ b/util.cxx
> @@ -345,7 +345,8 @@ tokenize_cxx(const string&  str, vector<string>&  tokens)
>   // same policy as execvp().  A program name not containing a slash
>   // will be searched along the $PATH.
>
> -string find_executable(const string&  name, const string&  env_path)
> +string find_executable(const string&  name, const string&  sysroot,
> +		       map<string, char*>&  sysenv, const string&  env_path)
>   {
>     string retpath;
>
> @@ -356,11 +357,15 @@ string find_executable(const string&  name, const string&  env_path)
>
>     if (name.find('/') != string::npos) // slash in the path already?
>       {
> -      retpath = name;
> +      retpath = sysroot + name;
>       }
>     else // Nope, search $PATH.
>       {
> -      char *path = getenv(env_path.c_str());
> +      char *path;
> +      if (sysenv.count(env_path) != 0)
> +        path = sysenv[env_path];
> +      else
> +        path = getenv(env_path.c_str());
>         if (path)
>           {
>             // Split PATH up.
> @@ -370,7 +375,7 @@ string find_executable(const string&  name, const string&  env_path)
>             // Search the path looking for the first executable of the right name.
>             for (vector<string>::iterator i = dirs.begin(); i != dirs.end(); i++)
>               {
> -              string fname = *i + "/" + name;
> +              string fname = sysroot + *i + "/" + name;
>                 const char *f = fname.c_str();
>
>                 // Look for a normal executable file.
> @@ -389,13 +394,24 @@ string find_executable(const string&  name, const string&  env_path)
>     // Could not find the program on the $PATH.  We'll just fall back to
>     // the unqualified name, which our caller will probably fail with.
>     if (retpath == "")
> -    retpath = name;
> +    retpath = sysroot + name;
>
>     // Canonicalize the path name.
>     char *cf = canonicalize_file_name (retpath.c_str());
>     if (cf)
>       {
> -      retpath = string(cf);
> +      string scf = string(cf);
> +      if (sysroot.empty())
> +        retpath = scf;
> +      else {
> +        int pos = scf.find(sysroot);
> +        if (pos == 0)
> +          retpath = scf;
> +        else {
> +          cerr<<  _F("ERROR: file %s not in sysroot %s", cf, sysroot.c_str())<<  endl;
> +          exit(1);
> +        }
> +      }
>         free (cf);
>       }
>
> diff --git a/util.h b/util.h
> index e576036..44e41e4 100644
> --- a/util.h
> +++ b/util.h
> @@ -12,6 +12,7 @@
>   #include<cctype>
>   #include<set>
>   #include<iomanip>
> +#include<map>
>   extern "C" {
>   #include<libintl.h>
>   #include<locale.h>
> @@ -51,6 +52,8 @@ void tokenize(const std::string&  str, std::vector<std::string>&  tokens,
>   	      const std::string&  delimiters);
>   void tokenize_cxx(const std::string&  str, std::vector<std::string>&  tokens);
>   std::string find_executable(const std::string&  name,
> +			    const std::string&  sysroot,
> +			    std::map<std::string,char *>&  sysenv,
>   			    const std::string&  env_path = "PATH");
>   const std::string cmdstr_quoted(const std::string&  cmd);
>   const std::string cmdstr_join(const std::vector<std::string>&  cmds);