RFC V3 [1/2] test-in-container

RFC V3 [1/2] test-in-container

[DJ Delorie]

Just reposting this part, the nss example didn't change, less mailing list spam...

	* Makefile (testroot.pristine): New rules to initialize the
	test-in-container "testroot".
	* Makerules (all-testsuite): Add tests-container.
	* Rules (tests): Add tests-container.
	(binaries-all-tests): Likewise.
	(tests-container): New, run these tests in the testroot container.
	* support/Makefile (others): Add test-container, xmkdirp,
	and links-dso-program.
	* support/links-dso-program-c.c: New.
	* support/links-dso-program.cc: New.
	* support/test-container.c: New.
	* support/mkdirp.c: New.
	* support/support.h: Add mkdirp() prototype.

Changes since V2:

Updated Rules to run test-container itself with the right environment
variables to run against the just-build library.  Tested with
F26->riscv cross and F26->glibc2.27 "native" (despite being native,
test-container had 2.27 symbols that F26 didn't provide, so... valid
test :)

Moved mkdir_p() intto xmkdirp.c as xmkdirp(), it takes a mode now too,
and a const char *.

Use copy_file_range() instead of read/write.

Link test-container against libsupport (it's really REALLY a test
program now :)

Use error() instead of printf/perror.

Exit with EXIT_UNSUPPORTED throughout.


diff --git a/Makefile b/Makefile
index bea4e27f8d..3fd19917df 100644
--- a/Makefile
+++ b/Makefile
@@ -297,6 +297,46 @@ define summarize-tests
 @! egrep -q -v '^(X?PASS|XFAIL|UNSUPPORTED):' $(objpfx)$1
 endef
 
+# The intention here is to do ONE install of our build into the
+# testroot.pristine/ directory, then rsync (internal to
+# support/test-container) that to testroot.root/ at the start of each
+# test.  That way we can promise each test a "clean" install, without
+# having to do the install for each test.
+#
+# In addition, we have to copy some files into this root in addition
+# to what glibc installs.  For example, many tests require /bin/sh be
+# present, and any shared objects that /bin/sh depends on.  We also
+# build a "test" program in either C or (if available) C++ just so we
+# can copy in any shared objects that GCC-compiled programs depend on.
+
+$(tests-container) $(addsuffix /tests,$(subdirs)) : $(objpfx)testroot.pristine/ready.ts
+$(objpfx)testroot.pristine/ready.ts :
+	test -d $(objpfx)testroot.pristine || \
+	  mkdir $(objpfx)testroot.pristine
+	# We need a working /bin/sh for some of the tests.
+	test -d $(objpfx)testroot.pristine/bin || \
+	  mkdir $(objpfx)testroot.pristine/bin
+	$(test-wrapper) cp /bin/sh $(objpfx)testroot.pristine/bin/sh
+	# Copy these DSOs first so we can overwrite them with our own.
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)testroot.pristine/bin/sh \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)support/links-dso-program \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	$(MAKE) install DESTDIR=$(objpfx)testroot.pristine
+	touch $(objpfx)testroot.pristine/ready.ts
+
 tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
diff --git a/Makerules b/Makerules
index b2c2724fcb..21367503d6 100644
--- a/Makerules
+++ b/Makerules
@@ -1372,7 +1372,7 @@ xcheck: xtests
 # The only difference between MODULE_NAME=testsuite and MODULE_NAME=nonlib is
 # that almost all internal declarations from config.h, libc-symbols.h, and
 # include/*.h are not available to 'testsuite' code, but are to 'nonlib' code.
-all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras))
+all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(tests-container))
 ifneq (,$(all-testsuite))
 cpp-srcs-left = $(all-testsuite)
 lib := testsuite
diff --git a/Rules b/Rules
index 706c8a749d..aa75afdb4c 100644
--- a/Rules
+++ b/Rules
@@ -130,12 +130,12 @@ others: $(py-const)
 
 ifeq ($(run-built-tests),no)
 tests: $(addprefix $(objpfx),$(filter-out $(tests-unsupported), \
-                                          $(tests) $(tests-internal)) \
+                                          $(tests) $(tests-internal) $(tests-container)) \
 			     $(test-srcs)) $(tests-special) \
 			     $(tests-printers-programs)
 xtests: tests $(xtests-special)
 else
-tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \
+tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) $(tests-container:%=$(objpfx)%.out) \
        $(tests-special) $(tests-printers-out)
 xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special)
 endif
@@ -149,7 +149,7 @@ tests-expected = $(tests) $(tests-internal) $(tests-printers)
 endif
 tests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
-	  $(sort $(tests-expected) $(tests-special-notdir:.out=)) \
+	  $(sort $(tests-expected) $(tests-special-notdir:.out=) $(tests-container)) \
 	  > $(objpfx)subdir-tests.sum
 xtests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
@@ -158,7 +158,7 @@ xtests:
 
 ifeq ($(build-programs),yes)
 binaries-all-notests = $(others) $(sysdep-others)
-binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs)
+binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) $(tests-container)
 binaries-all = $(binaries-all-notests) $(binaries-all-tests)
 binaries-static-notests = $(others-static)
 binaries-static-tests = $(tests-static) $(xtests-static)
@@ -248,6 +248,16 @@ $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
 	$(make-test-out) > $@; \
 	$(evaluate-test)
 
+
+# Any tests that require an isolated container (chroot) in which to
+# run, should be added to tests-container.
+$(tests-container:%=$(objpfx)%.out): $(objpfx)%.out : $(if $(wildcard $(objpfx)%.files),$(objpfx)%.files,/dev/null) $(objpfx)%
+	$(test-wrapper-env) $(run-program-env) $(run-via-rtld-prefix) \
+	  $(common-objpfx)support/test-container env $(run-program-env) \
+	  $(host-test-program-cmd) $($*-ARGS) > $@; \
+	$(evaluate-test)
+
+
 # tests-unsupported lists tests that we will not try to build at all in
 # this configuration.  Note this runs every time because it does not
 # actually create its target.  The dependency on Makefile is meant to
diff --git a/support/Makefile b/support/Makefile
index 1bda81e55e..9dc02ef356 100644
--- a/support/Makefile
+++ b/support/Makefile
@@ -81,6 +81,7 @@ libsupport-routines = \
   xmalloc \
   xmemstream \
   xmkdir \
+  xmkdirp \
   xmmap \
   xmprotect \
   xmunmap \
@@ -145,6 +146,31 @@ ifeq ($(build-shared),yes)
 libsupport-inhibit-o += .o
 endif
 
+others: $(objpfx)test-container $(objpfx)links-dso-program
+
+LDLIBS-test-container = $(libsupport)
+
+CFLAGS-test-container.c = \
+		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
+		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
+		-DINSTDIR_PATH=\"${prefix}\" \
+		-DLIBDIR_PATH=\"${libdir}\"
+
+others += test-container
+others-noinstall += test-container
+
+$(objpfx)test-container : $(libsupport)
+
+# This exists only so we can guess which OS DSOs we need to copy into
+# the testing container.
+ifeq (,$(CXX))
+$(objpfx)links-dso-program : $(objpfx)links-dso-program-c.o
+	$(LINK.o) -o $@ $^
+else
+$(objpfx)links-dso-program : $(objpfx)links-dso-program.o
+	$(LINK.o) -o $@ $^ -lstdc++
+endif
+
 tests = \
   README-testing \
   tst-support-namespace \
diff --git a/support/links-dso-program-c.c b/support/links-dso-program-c.c
new file mode 100644
index 0000000000..c1f64fbbac
--- /dev/null
+++ b/support/links-dso-program-c.c
@@ -0,0 +1,4 @@
+int
+main() {
+  return 0;
+}
diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
new file mode 100644
index 0000000000..c1f64fbbac
--- /dev/null
+++ b/support/links-dso-program.cc
@@ -0,0 +1,4 @@
+int
+main() {
+  return 0;
+}
diff --git a/support/support.h b/support/support.h
index bc5827ed87..bf3b50ca25 100644
--- a/support/support.h
+++ b/support/support.h
@@ -70,6 +70,9 @@ char *xasprintf (const char *format, ...)
 char *xstrdup (const char *);
 char *xstrndup (const char *, size_t);
 
+/* Equivalent of "mkdir -p".  */
+int xmkdirp (const char *, int);
+
 __END_DECLS
 
 #endif /* SUPPORT_H */
diff --git a/support/test-container.c b/support/test-container.c
new file mode 100644
index 0000000000..e08905f108
--- /dev/null
+++ b/support/test-container.c
@@ -0,0 +1,979 @@
+/* Run a test case in an isolated namespace.
+   Copyright (C) 2017-2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define __USE_LARGEFILE64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sched.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <sys/sysmacros.h>
+#include <ctype.h>
+#include <utime.h>
+#include <errno.h>
+#include <error.h>
+
+#include <support/support.h>
+#include "test-driver.h"
+
+#ifdef SRCDIR_PATH
+const char *srcdir = SRCDIR_PATH;
+#else
+# error please -DSRCDIR_PATH=something in the Makefile
+#endif
+
+#ifdef OBJDIR_PATH
+const char *objdir = OBJDIR_PATH;
+#else
+# error please -DOBJDIR_PATH=something in the Makefile
+#endif
+
+#ifdef INSTDIR_PATH
+const char *instdir = INSTDIR_PATH;
+#else
+# error please -DINSTDIR_PATH=something in the Makefile
+#endif
+
+#ifdef LIBDIR_PATH
+const char *libdir = LIBDIR_PATH;
+#else
+# error please -DLIBDIR_PATH=something in the Makefile
+#endif
+
+int verbose = 0;
+
+/* Running a test in a container is tricky.  There are two main
+   categories of things to do:
+
+   1. "Once" actions, like setting up the container and doing an
+      install into it.
+
+   2. "Per-test" actions, like copying in support files and
+      configuring the container.
+
+
+   "Once" actions:
+
+   * mkdir $buildroot/testroot.pristine/
+   * install into it
+   * rsync to $buildroot/testroot.root/
+
+   "Per-test" actions:
+   * maybe rsync to $buildroot/testroot.root/
+   * copy support files and test binary
+   * chroot/unshare
+   * set up any mounts (like /proc)
+
+   Magic files:
+
+   For test $srcdir/foo/mytest.c we look for $srcdir/foo/mytest.root and, if found...
+
+   * mytest.root/ is rsync'd into container
+   * mytest.root/preclean.txt causes fresh rsync (with delete) before test if present
+   * mytest.root/files.txt has a list of files to copy - TBD
+     ($B/ or $S/ for build and source directories)
+       syntax:
+         # comment
+         mv FILE FILE
+	 cp FILE FILE
+	 rm FILE
+	 FILE must start with $B/, $S/, $I/, $L/, or /
+	  (expands to build dir, source dir, install dir, library dir
+	   (in container), or container's root)
+   * mytest.root/postclean.txt causes fresh rsync (with delete) after test if present
+
+   Note that $srcdir/foo/mytest.files may be used instead of a
+   files.txt in the sysroot, if there is no other reason for a sysroot.
+
+   Design goals:
+
+   * independent of other packages which may not be installed (like
+     rsync or Docker, or even "cp")
+
+   * Simple, easy to review code (i.e. prefer simple naive code over
+     complex efficient code)
+
+   TBD:
+
+   * The current implementation is not parallel-make-safe, as one test
+     could be modifying the chroot while another is running against
+     it.
+
+*/
+
+/*--------------------------------------------------*/
+/* Utility Functions */
+
+/* Temporarily concatenate multiple strings into one.  Allows up to 10
+   temporary results; use strdup () if you need them to be
+   permanent.  */
+
+static char *
+concat (const char *str, ...)
+{
+  /* Assume initialized to NULL/zero.  */
+  static char *bufs[10];
+  static size_t buflens[10];
+  static int bufn = 0;
+  int n;
+  size_t len;
+  va_list ap, ap2;
+  char *cp;
+  char *next;
+
+  va_start (ap, str);
+  va_copy (ap2, ap);
+
+  n = bufn;
+  bufn = (bufn + 1) % 10;
+  len = strlen (str);
+
+  while ((next = va_arg (ap, char *)) != NULL)
+    len = len + strlen (next);
+
+  va_end (ap);
+
+  if (bufs[n] == NULL)
+    {
+      bufs[n] = malloc (len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+  else if (buflens[n] < len + 1)
+    {
+      bufs[n] = realloc (bufs[n], len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+
+  strcpy (bufs[n], str);
+  cp = strchr (bufs[n], '\0');
+  while ((next = va_arg (ap2, char *)) != NULL)
+    {
+      strcpy (cp, next);
+      cp = strchr (cp, '\0');
+    }
+  *cp = 0;
+  va_end (ap2);
+
+  return bufs[n];
+}
+
+/* Try to mount SRC onto DEST.  */
+
+static void
+trymount (const char *src, const char *dest)
+{
+  if (mount (src, dest, "", MS_BIND, NULL) < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "can't mount %s onto %s\n", src, dest);
+}
+
+/* Special case of above for devices like /dev/zero where we have to
+   mount a device over a device, not a directory over a directory.  */
+
+static void
+devmount (const char *new_root_path, const char *which)
+{
+  int fd;
+  fd = open (concat (new_root_path, "/dev/", which, NULL),
+	     O_CREAT | O_TRUNC | O_RDWR, 0777);
+  close (fd);
+
+  trymount (concat ("/dev/", which, NULL),
+	    concat (new_root_path, "/dev/", which, NULL));
+}
+
+/* Returns true if the string "looks like" an environement variable
+   being set.  */
+
+static int
+is_env_setting (const char *a)
+{
+  int count_name = 0;
+
+  while (*a)
+    {
+      if (isalnum (*a) || *a == '_')
+	++count_name;
+      else if (*a == '=' && count_name > 0)
+	return 1;
+      else
+	return 0;
+      ++a;
+    }
+  return 0;
+}
+
+/* Break the_line into words and store in the_words.  Max nwords,
+   returns actual count.  */
+static int
+tokenize (char *the_line, char **the_words, int nwords)
+{
+  int rv = 0;
+
+  while (nwords > 0)
+    {
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && isspace (*the_line))
+	++the_line;
+
+      /* End of line?  */
+      if (*the_line == 0)
+	return rv;
+
+      /* THE_LINE points to a non-whitespace character, so we have a
+	 word.  */
+      *the_words = the_line;
+      ++the_words;
+      nwords--;
+      ++rv;
+
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && ! isspace (*the_line))
+	++the_line;
+
+      /* We now point at the trailing NUL *or* some whitespace.  */
+      if (*the_line == 0)
+	return rv;
+
+      /* It was whitespace, skip and keep tokenizing.  */
+      *the_line++ = 0;
+    }
+
+  /* We get here if we filled the words buffer.  */
+  return rv;
+}
+
+/*--------------------------------------------------*/
+/* mini-RSYNC implementation.  Optimize later.      */
+
+/* Set this to 1 if you need to debug the rsync function.  */
+#define RTRACE 0
+
+/* A few routines for an "rsync buffer" which stores the paths we're
+   working on.  We continuously grow and shrink the paths in each
+   buffer so there's lot of re-use.  */
+
+/* We rely on "initialized to zero" to set these up.  */
+typedef struct
+{
+  char *buf;
+  size_t len;
+  size_t size;
+} path_buf;
+
+static path_buf spath, dpath;
+
+static void
+r_setup (char *path, path_buf * pb)
+{
+  size_t len = strlen (path);
+  if (pb->buf == NULL || pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      if (pb->buf == NULL)
+	pb->buf = (char *) malloc (sz);
+      else
+	pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	error (EXIT_UNSUPPORTED, ENOMEM,
+	       "Out of memory while rsyncing\n");
+
+      pb->size = sz;
+    }
+  strcpy (pb->buf, path);
+  pb->len = len;
+}
+
+static void
+r_append (const char *path, path_buf * pb)
+{
+  size_t len = strlen (path) + pb->len;
+  if (pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	error (EXIT_UNSUPPORTED, ENOMEM,
+	       "Out of memory while rsyncing\n");
+
+      pb->size = sz;
+    }
+  strcpy (pb->buf + pb->len, path);
+  pb->len = len;
+}
+
+static int
+file_exists (char *path)
+{
+  struct stat st;
+  if (lstat (path, &st) == 0)
+    return 1;
+  return 0;
+}
+
+static void
+recursive_remove (char *path)
+{
+  pid_t child;
+  int status;
+  /* FIXME: re-implement without external dependencies at some point.
+     Fortunately, this runs outside the container.  */
+
+  child = fork();
+
+  switch (child) {
+  case -1:
+    error (EXIT_UNSUPPORTED, errno, "fork");
+  case 0:
+    /* Child.  */
+    execlp ("rm", "rm", "-rf", path, NULL);
+  default:
+    /* Parent.  */
+    waitpid (child, &status, 0);
+    break;
+  }
+}
+
+/* Used for both rsync and the files.txt "cp" command.  */
+
+static void
+copy_one_file (const char *sname, const char *dname)
+{
+  int sfd, dfd;
+  struct stat st;
+  struct utimbuf times;
+
+  sfd = open (sname, O_RDONLY);
+  if (sfd < 0)
+      error (EXIT_UNSUPPORTED, errno,
+	     "unable to open %s for reading\n", sname);
+
+  if (stat (sname, &st) < 0)
+      error (EXIT_UNSUPPORTED, errno,
+	     "unable to stat %s\n", sname);
+
+  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+  if (dfd < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "unable to open %s for writing\n", dname);
+
+  if (copy_file_range (sfd, 0, dfd, 0, st.st_size, 0) != st.st_size)
+    error (EXIT_UNSUPPORTED, errno,
+	   "cannot copy file %s to %s\n", sname, dname);
+
+  close (sfd);
+  close (dfd);
+
+  chmod (dname, st.st_mode & 0777);
+
+  times.actime = st.st_atime;
+  times.modtime = st.st_mtime;
+  utime (dname, &times);
+}
+
+/* We don't check *everything* about the two files to see if a copy is
+   needed, just the minimum to make sure we get the latest copy. */
+static int
+need_sync (char *ap, char *bp, struct stat *a, struct stat *b)
+{
+  if ((a->st_mode & S_IFMT) != (b->st_mode & S_IFMT))
+    return 1;
+
+  if (S_ISLNK (a->st_mode))
+    {
+      int rv;
+      char *al, *bl;
+
+      if (a->st_size != b->st_size)
+	return 1;
+
+      al = (char *) malloc (a->st_size + 1);
+      bl = (char *) malloc (b->st_size + 1);
+      readlink (ap, al, a->st_size + 1);
+      readlink (bp, bl, b->st_size + 1);
+      al[a->st_size] = 0;
+      bl[b->st_size] = 0;
+      rv = strcmp (al, bl);
+      free (al);
+      free (bl);
+      if (rv == 0)
+	return 0; /* links are same */
+      return 1; /* links differ */
+    }
+
+#if RTRACE
+  if (a->st_size != b->st_size)
+    printf ("SIZE\n");
+  if ((a->st_mode & 0777) != (b->st_mode & 0777))
+    printf ("MODE\n");
+  if (a->st_mtime != b->st_mtime)
+    printf ("TIME\n");
+#endif
+
+  if (a->st_size == b->st_size
+      && ((a->st_mode & 0777) == (b->st_mode & 0777))
+      && a->st_mtime == b->st_mtime)
+    return 0;
+
+  return 1;
+}
+
+static void
+rsync_1 (path_buf * src, path_buf * dest, int and_delete)
+{
+  DIR *dir;
+  struct dirent *de;
+  struct stat s, d;
+
+  r_append ("/", src);
+  r_append ("/", dest);
+#if RTRACE
+  printf ("sync %s to %s %s\n", src->buf, dest->buf,
+	  and_delete ? "and delete" : "");
+#endif
+
+  size_t staillen = src->len;
+
+  size_t dtaillen = dest->len;
+
+  dir = opendir (src->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      if (lstat (src->buf, &s) != 0)
+	error (EXIT_UNSUPPORTED, errno,
+	       "%s obtained by readdir, but stat failed.\n", src->buf);
+
+      // It's OK if this one fails, since we know the file might be missing.
+      lstat (dest->buf, &d);
+
+      if (! need_sync (src->buf, dest->buf, &s, &d))
+	{
+	  if (S_ISDIR (s.st_mode))
+	    rsync_1 (src, dest, and_delete);
+	  continue;
+	}
+
+      if (d.st_mode != ~0)
+	switch (d.st_mode & S_IFMT)
+	  {
+	  case S_IFDIR:
+	    if (!S_ISDIR (s.st_mode))
+	      {
+#if RTRACE
+		printf ("-D %s\n", dest->buf);
+#endif
+		recursive_remove (dest->buf);
+	      }
+	    break;
+
+	  default:
+#if RTRACE
+	    printf ("-F %s\n", dest->buf);
+#endif
+	    unlink (dest->buf);
+	    break;
+	  }
+
+      switch (s.st_mode & S_IFMT)
+	{
+	case S_IFREG:
+#if RTRACE
+	  printf ("+F %s\n", dest->buf);
+#endif
+	  copy_one_file (src->buf, dest->buf);
+	  break;
+
+	case S_IFDIR:
+#if RTRACE
+	  printf ("+D %s\n", dest->buf);
+#endif
+	  mkdir (dest->buf, (s.st_mode & 0777) | 0700);
+	  rsync_1 (src, dest, and_delete);
+	  break;
+
+	case S_IFLNK:
+	  {
+	    char *lp = (char *) malloc (s.st_size + 1);
+#if RTRACE
+	    printf ("+L %s\n", dest->buf);
+#endif
+	    readlink (src->buf, lp, s.st_size + 1);
+	    lp[s.st_size] = 0;
+	    symlink (lp, dest->buf);
+	    free (lp);
+	    break;
+	  }
+
+	default:
+	  break;
+	}
+    }
+
+  closedir (dir);
+  src->len = staillen;
+  src->buf[staillen] = 0;
+  dest->len = dtaillen;
+  dest->buf[dtaillen] = 0;
+
+  if (!and_delete)
+    return;
+
+  /* The rest of this function removes any files/directories in DEST
+     that do not exist in SRC.  This is triggered as part of a
+     preclean or postsclean step.  */
+
+  dir = opendir (dest->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      
+      if (lstat (dest->buf, &d) != 0)
+	error (EXIT_UNSUPPORTED, errno,
+	       "%s obtained by readdir, but stat failed.\n", dest->buf);
+
+      if (s.st_mode == ~0)
+	{
+	  /* dest exists and src doesn't, clean it.  */
+	  switch (d.st_mode & S_IFMT)
+	    {
+	    case S_IFDIR:
+	      if (!S_ISDIR (s.st_mode))
+		{
+#if RTRACE
+		  printf ("-D %s\n", dest->buf);
+#endif
+		  recursive_remove (dest->buf);
+		}
+	      break;
+
+	    default:
+#if RTRACE
+	      printf ("-F %s\n", dest->buf);
+#endif
+	      unlink (dest->buf);
+	      break;
+	    }
+	}
+    }
+
+  closedir (dir);
+}
+
+static void
+rsync (char *src, char *dest, int and_delete)
+{
+  r_setup (src, &spath);
+  r_setup (dest, &dpath);
+
+  rsync_1 (&spath, &dpath, and_delete);
+}
+
+/*--------------------------------------------------*/
+/* Main */
+
+int
+main (int argc, char **argv)
+{
+  pid_t child;
+  char *pristine_root_path;
+  char *new_root_path;
+  char *new_cwd_path;
+  char *new_objdir_path;
+  char *new_srcdir_path;
+  char **new_child_proc;
+  char *command_root;
+  char *command_base;
+  char *so_base;
+  int do_postclean = 0;
+
+  uid_t original_uid;
+  gid_t original_gid;
+  int UMAP;
+  int GMAP;
+  char tmp[100];
+  struct stat st;
+  int lock_fd;
+
+  setbuf (stdout, NULL);
+
+  // The command line we're expecting looks like this:
+  // env <set some vars> ld.so <library path> test-binary
+
+  // We need to peel off any "env" or "ld.so" portion of the command
+  // line, and keep track of which env vars we should preserve and
+  // which we drop.
+
+  if (argc < 2)
+    {
+      fprintf (stderr, "Usage: containerize <program to run> <args...>\n");
+      exit (1);
+    }
+
+  if (strcmp (argv[1], "-v") == 0)
+    {
+      verbose = 1;
+      ++argv;
+      --argc;
+    }
+
+  if (strcmp (argv[1], "env") == 0)
+    {
+      ++argv;
+      --argc;
+      while (is_env_setting (argv[1]))
+	{
+	  // List variables we do NOT want to propogate.
+#if 0
+	  // until we discover why locale/iconv tests don't
+	  // work against an installed tree...
+	  if (memcmp (argv[1], "GCONV_PATH=", 11)
+	      && memcmp (argv[1], "LOCPATH=", 8))
+#endif
+	    {
+	      // Need to keep these.  Note that putenv stores a
+	      // pointer to our argv.
+	      putenv (argv[1]);
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  if (strncmp (argv[1], concat (objdir, "/elf/ld-linux-", NULL),
+	       strlen (objdir) + 14) == 0)
+    {
+      ++argv;
+      --argc;
+      while (argv[1][0] == '-')
+	{
+	  if (strcmp (argv[1], "--library-path") == 0)
+	    {
+	      ++argv;
+	      --argc;
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  pristine_root_path = strdup (concat (objdir, "/testroot.pristine", NULL));
+  new_root_path = strdup (concat (objdir, "/testroot.root", NULL));
+  new_cwd_path = get_current_dir_name ();
+  new_child_proc = argv + 1;
+
+  lock_fd = open(concat (pristine_root_path, "/lock.fd", NULL),
+		 O_CREAT | O_TRUNC | O_RDWR, 0666);
+  if (lock_fd < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "Cannot create testroot lock.\n");
+
+  while (flock (lock_fd, LOCK_EX) != 0)
+    {
+      if (errno != EINTR)
+	error (EXIT_UNSUPPORTED, errno, "Cannot lock testroot.\n");
+    }
+
+  xmkdirp (new_root_path, 0755);
+
+  /* We look for extra setup info in a subdir in the same spot as the
+     test, with the same name but a ".root" extension.  This is that
+     directory.  We try to look in the source tree if the path we're
+     given refers to the build tree, but we rely on the path to be
+     absolute.  This is what the glibc makefiles do.  */
+  command_root = concat (argv[1], ".root", NULL);
+  if (strncmp (command_root, objdir, strlen (objdir)) == 0
+      && command_root[strlen (objdir)] == '/')
+    command_root = concat (srcdir, argv[1] + strlen (objdir), ".root", NULL);
+  command_root = strdup (command_root);
+
+  /* This cuts off the ".root" we appended above.  */
+  command_base = strdup (command_root);
+  command_base[strlen (command_base) - 5] = 0;
+
+  /* Shared object base directory.  */
+  so_base = strdup (argv[1]);
+  if (strrchr (so_base, '/') != NULL)
+    strrchr (so_base, '/')[1] = 0;
+
+  if (file_exists (concat (command_root, "/postclean.txt", NULL)))
+    do_postclean = 1;
+
+  rsync (pristine_root_path, new_root_path,
+	 1 || file_exists (concat (command_root, "/preclean.txt", NULL)));
+
+  if (stat (command_root, &st) >= 0
+      && S_ISDIR (st.st_mode))
+    rsync (command_root, new_root_path, 0);
+
+  new_objdir_path = strdup (concat (new_root_path, objdir, NULL));
+  new_srcdir_path = strdup (concat (new_root_path, srcdir, NULL));
+
+  /* new_cwd_path starts with '/' so no "/" needed between the two.  */
+  xmkdirp (concat (new_root_path, new_cwd_path, NULL), 0755);
+  xmkdirp (new_srcdir_path, 0755);
+  xmkdirp (new_objdir_path, 0755);
+
+  original_uid = getuid ();
+  original_gid = getgid ();
+
+  /* Handle the cp/mv/rm "script" here.  */
+  {
+    char *the_line = NULL;
+    size_t line_len = 0;
+    char *fname = concat (command_root, "/files.txt", NULL);
+    char *the_words[3];
+    FILE *f = fopen (fname, "r");
+
+    if (verbose && f)
+      fprintf (stderr, "reading %s\n", fname);
+
+    if (f == NULL)
+      {
+	/* Try foo.files instead of foo.root/files.txt, as a shortcut.  */
+	fname = concat (command_base, ".files", NULL);
+	f = fopen (fname, "r");
+	if (verbose && f)
+	  fprintf (stderr, "reading %s\n", fname);
+      }
+
+#if 0
+    /* I don't want to add this until we know we need it, but here it
+       is...  */
+    if (f == NULL)
+      {
+	/* Look for a Makefile-generated one also.  */
+	fname = concat (argv[1], ".files", NULL);
+	f = fopen (fname, "r");
+      }
+#endif
+
+    /* This is where we "interpret" the mini-script which is <test>.files.  */
+    if (f != NULL)
+      {
+	while (getline (&the_line, &line_len, f) > 0)
+	  {
+	    int nt = tokenize (the_line, the_words, 3);
+	    int i;
+
+	    for (i = 1; i < nt; ++i)
+	      {
+		if (memcmp (the_words[i], "$B/", 3) == 0)
+		  the_words[i] = concat (objdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$S/", 3) == 0)
+		  the_words[i] = concat (srcdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$I/", 3) == 0)
+		  the_words[i] = concat (new_root_path, instdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$L/", 3) == 0)
+		  the_words[i] = concat (new_root_path, libdir, the_words[i] + 2, NULL);
+		else if (the_words[i][0] == '/')
+		  the_words[i] = concat (new_root_path, the_words[i], NULL);
+	      }
+
+	    if (nt == 3 && the_words[2][strlen (the_words[2]) - 1] == '/')
+	      {
+		char *r = strrchr (the_words[1], '/');
+		if (r)
+		  the_words[2] = concat (the_words[2], r + 1, NULL);
+		else
+		  the_words[2] = concat (the_words[2], the_words[1], NULL);
+	      }
+
+	    if (nt == 2 && strcmp (the_words[0], "so") == 0)
+	      {
+		the_words[2] = concat (new_root_path, libdir, "/", the_words[1], NULL);
+		the_words[1] = concat (so_base, the_words[1], NULL);
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "cp") == 0)
+	      {
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "mv") == 0)
+	      {
+		rename (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "chmod") == 0)
+	      {
+		long int m;
+		m = strtol (the_words[1], NULL, 0);
+		chmod (the_words[2], m);
+	      }
+	    else if (nt == 2 && strcmp (the_words[0], "rm") == 0)
+	      {
+		unlink (the_words[1]);
+	      }
+	    else if (nt > 0 && the_words[0][0] != '#')
+	      {
+		printf ("\033[31minvalid [%s]\033[0m\n", the_words[0]);
+	      }
+	  }
+	fclose (f);
+      }
+  }
+
+  // The unshare here gives us our own spaces and capabilities.
+  if (unshare (CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS) < 0)
+    error (EXIT_UNSUPPORTED, errno, "unable to unshare user/fs, ");
+
+  /* Some systems, by default, all mounts leak out of the namespace.  */
+  if (mount ("none", "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0)
+    error (EXIT_UNSUPPORTED, errno, "could not create a private mount namespace\n");
+
+  trymount (srcdir, new_srcdir_path);
+  trymount (objdir, new_objdir_path);
+
+  xmkdirp (concat (new_root_path, "/dev", NULL), 0755);
+  devmount (new_root_path, "null");
+  devmount (new_root_path, "zero");
+  devmount (new_root_path, "urandom");
+
+  // We're done with the "old" root, switch to the new one.
+  if (chroot (new_root_path) < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "Can't chroot to %s - ", new_root_path);
+
+  if (chdir (new_cwd_path) < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "Can't cd to new %s - ", new_cwd_path);
+
+  /* To complete the containerization, we need to fork () at least
+     once.  We can't exec, nor can we somehow link the new child to
+     our parent.  So we run the child and propogate it's exit status
+     up. */
+  child = fork ();
+  if (child < 0)
+    error (EXIT_UNSUPPORTED, errno, "fork");
+  else if (child > 0)
+    {
+      /* Parent.  */
+      int status;
+      waitpid (child, &status, 0);
+
+      /* There's a bit of magic here, since the buildroot is mounted
+	 in our space, the paths are still valid, and since the mounts
+	 aren't recursive, it sees *only* the built root, not anything
+	 we would normally se if we rsync'd to "/" like mounted /dev
+	 files.  */
+      if (do_postclean)
+	  rsync (pristine_root_path, new_root_path, 1);
+
+      if (WIFEXITED (status))
+	exit (WEXITSTATUS (status));
+
+      if (WIFSIGNALED (status))
+	{
+	  printf ("%%SIGNALLED%%\n");
+	  exit (77);
+	}
+
+      printf ("%%EXITERROR%%\n");
+      exit (78);
+    }
+
+  /* The rest is the child process, which is now PID 1 and "in" the
+     new root.  */
+
+  mkdir ("/tmp", 0755);
+
+  // Now that we're pid 1 (effectively "root") we can mount /proc
+  mkdir ("/proc", 0777);
+  if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "Unable to mount /proc: ");
+
+  // We map our original UID to the same UID in the container so we
+  // can own our own files normally
+  UMAP = open ("/proc/self/uid_map", O_WRONLY);
+  if (UMAP < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "can't write to /proc/self/uid_map\n");
+
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_uid, (long long) original_uid);
+  write (UMAP, tmp, strlen (tmp));
+  close (UMAP);
+
+  // We must disable setgroups () before we can map our groups, else we
+  // get EPERM.
+  GMAP = open ("/proc/self/setgroups", O_WRONLY);
+  if (GMAP >= 0)
+    {
+      /* We support kernels old enough to not have this.  */
+      write (GMAP, "deny\n", 5);
+      close (GMAP);
+    }
+
+  // We map our original GID to the same GID in the container so we
+  // can own our own files normally
+  GMAP = open ("/proc/self/gid_map", O_WRONLY);
+  if (GMAP < 0)
+    error (EXIT_UNSUPPORTED, errno,
+	   "can't write to /proc/self/gid_map\n");
+
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_gid, (long long) original_gid);
+  write (GMAP, tmp, strlen (tmp));
+  close (GMAP);
+
+  // Now run the child
+  execvp (new_child_proc[0], new_child_proc);
+
+  // Or don't run the child?
+  error (EXIT_UNSUPPORTED, errno,
+	 "Unable to exec %s\n", new_child_proc[0]);
+
+  // Because gcc won't know error() never returns...
+  exit (EXIT_UNSUPPORTED);
+}

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> Moved mkdir_p() intto xmkdirp.c as xmkdirp(), it takes a mode now too,
> and a const char *.

The file is missing in the patch.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> +CFLAGS-test-container.c = \
> +		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
> +		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
> +		-DINSTDIR_PATH=\"${prefix}\" \
> +		-DLIBDIR_PATH=\"${libdir}\"


I think OBJDIR_PATH could be $(objdir).  It's also odd to see make 
variables expanded with ${…}.

Could you add variables (maybe support_srcdir, support_objdir, 
support_prefix, support_libdir) to a separate file under support/ and 
add matching declarations, perhaps in support/support.h?  This looks 
generally useful to me.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[DJ Delorie]

Florian Weimer <fweimer@redhat.com> writes:
> On 03/02/2018 12:05 AM, DJ Delorie wrote:
>> Moved mkdir_p() intto xmkdirp.c as xmkdirp(), it takes a mode now too,
>> and a const char *.
>
> The file is missing in the patch.

Sigh, sorry...

/* Error-checking replacement for "mkdir -p".
   Copyright (C) 2018 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <support/support.h>
#include <support/check.h>

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

/* Equivalent of "mkdir -p".  */

int
xmkdirp (const char *path, int mode)
{
  struct stat s;
  const char *slash_p;
  int rv;

  if (path[0] == 0)
    return 0;

  if (stat (path, &s) == 0)
    {
      if (S_ISDIR (s.st_mode))
	return 0;
      errno = EEXIST;
      FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);
    }

  slash_p = strrchr (path, '/');
  if (slash_p)
    {
      while (slash_p > path && slash_p[-1] == '/')
	--slash_p;
      if (slash_p > path)
	{
	  char *parent = xstrndup (path, slash_p - path);
	  rv = xmkdirp (parent, mode);
	  free (parent);
	}
    }

  rv = mkdir (path, mode);
  if (rv != 0)
    FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);

  return 0;
}

Re: RFC V3 [1/2] test-in-container

[DJ Delorie]

Florian Weimer <fweimer@redhat.com> writes:
> On 03/02/2018 12:05 AM, DJ Delorie wrote:
>> +CFLAGS-test-container.c = \
>> +		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
>> +		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
>> +		-DINSTDIR_PATH=\"${prefix}\" \
>> +		-DLIBDIR_PATH=\"${libdir}\"
>
>
> I think OBJDIR_PATH could be $(objdir).

That would put the testroot itself under support/

There's nothing under support/ needed by test-container.c, but many
things in the root (like elf/*) that are.

And they need to be absolute (resolved) as symlinks might not work
inside the container, where only part of the filesystem exists and $PWD
may differ.

>  It's also odd to see make variables expanded with ${…}.

Fixed.

> Could you add variables (maybe support_srcdir, support_objdir, 
> support_prefix, support_libdir) to a separate file under support/ and 
> add matching declarations, perhaps in support/support.h?  This looks 
> generally useful to me.

Useful to what?  And is this usefulness based on them being pointers to
the support subdirectory, or the toplevel build directory?

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

* DJ Delorie:

> Florian Weimer <fweimer@redhat.com> writes:
>> On 03/02/2018 12:05 AM, DJ Delorie wrote:
>>> +CFLAGS-test-container.c = \
>>> +		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
>>> +		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
>>> +		-DINSTDIR_PATH=\"${prefix}\" \
>>> +		-DLIBDIR_PATH=\"${libdir}\"
>>
>>
>> I think OBJDIR_PATH could be $(objdir).
>
> That would put the testroot itself under support/

Are you sure?  $(objdir) is used to define $(common-objpfx), after
all.

> There's nothing under support/ needed by test-container.c, but many
> things in the root (like elf/*) that are.
>
> And they need to be absolute (resolved) as symlinks might not work
> inside the container, where only part of the filesystem exists and $PWD
> may differ.

You could bind mount in the container with the same path as outside
the container.

>> Could you add variables (maybe support_srcdir, support_objdir, 
>> support_prefix, support_libdir) to a separate file under support/ and 
>> add matching declarations, perhaps in support/support.h?  This looks 
>> generally useful to me.
>
> Useful to what?  And is this usefulness based on them being pointers to
> the support subdirectory, or the toplevel build directory?

Useful for launching statically linked programs, for example.
Currently, we need to pass in $(common-objpfx) from Makefile.  See
tst-getdate-ENV in time/Makefile for an example.  Or the construct
around tst-support_record_failure-2.out in support/Makefile.

Re: RFC V3 [1/2] test-in-container

[DJ Delorie]

Florian Weimer <fw@deneb.enyo.de> writes:
>> That would put the testroot itself under support/
>
> Are you sure?  $(objdir) is used to define $(common-objpfx), after
> all.

The testroot is created by the toplevel Makefile, but test-container is
run from many different subdirectories.  It needs to know what the
toplevel directory is.  When building test-container in the support
subdir, $(objdir) points so the subdir:

gcc test-container.c ... -DOBJDIR_PATH=\"`cd /envy/dj/tools/upstream/glibc.testroot.build/support//..; pwd`\"

> You could bind mount in the container with the same path as outside
> the container.

I do.

Also, all the paths used inside the container need to work regardless of
what the current directory is.

> Useful for launching statically linked programs, for example.
> Currently, we need to pass in $(common-objpfx) from Makefile.  See
> tst-getdate-ENV in time/Makefile for an example.  Or the construct
> around tst-support_record_failure-2.out in support/Makefile.

Some of those cases could be candidates for conversion to
test-in-container, though, since the data files could be placed in their
"default" locations.

But sure, I could split those out.  Or we could split them out later as
part of integrating them into whatever needs them.  Personally, I'd
rather do it in conjunction with an actual use case, so it can be
tested.

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 06/25/2018 11:48 PM, DJ Delorie wrote:
>    slash_p = strrchr (path, '/');
>    if (slash_p)

No implicit comparison against NULL, please.

>      {
>        while (slash_p > path && slash_p[-1] == '/')
> 	--slash_p;
>        if (slash_p > path)
> 	{
> 	  char *parent = xstrndup (path, slash_p - path);
> 	  rv = xmkdirp (parent, mode);
> 	  free (parent);
> 	}
>      }
> 
>    rv = mkdir (path, mode);
>    if (rv != 0)
>      FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);

The recursion is a bit awkward, but I think it will give a reasonable 
error code because of the preceding stat check (which is repeated many 
times unnecessarily).  But for test support code, that's fine, I think.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> diff --git a/support/links-dso-program-c.c b/support/links-dso-program-c.c
> new file mode 100644
> index 0000000000..c1f64fbbac
> --- /dev/null
> +++ b/support/links-dso-program-c.c
> @@ -0,0 +1,4 @@
> +int
> +main() {
> +  return 0;
> +}
> diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
> new file mode 100644
> index 0000000000..c1f64fbbac
> --- /dev/null
> +++ b/support/links-dso-program.cc
> @@ -0,0 +1,4 @@
> +int
> +main() {
> +  return 0;
> +}

Style: missing space before (), and the C version uses an old-style 
function definition.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> +$(tests-container) $(addsuffix /tests,$(subdirs)) : $(objpfx)testroot.pristine/ready.ts
> +$(objpfx)testroot.pristine/ready.ts :

ready.ts is just as stamp file, right?  We use “stamp” for those elsewhere.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> +# The intention here is to do ONE install of our build into the
> +# testroot.pristine/ directory, then rsync (internal to
> +# support/test-container) that to testroot.root/ at the start of each
> +# test.  That way we can promise each test a "clean" install, without
> +# having to do the install for each test.
> +#
> +# In addition, we have to copy some files into this root in addition
> +# to what glibc installs.  For example, many tests require /bin/sh be
> +# present, and any shared objects that /bin/sh depends on.  We also
> +# build a "test" program in either C or (if available) C++ just so we
> +# can copy in any shared objects that GCC-compiled programs depend on.

I think we need something to run these tests outside the Makefile 
framework, similar to testrun.sh.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> +  // The unshare here gives us our own spaces and capabilities.
> +  if (unshare (CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS) < 0)
> +    error (EXIT_UNSUPPORTED, errno, "unable to unshare user/fs, ");
> +
> +  /* Some systems, by default, all mounts leak out of the namespace.  */
> +  if (mount ("none", "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0)
> +    error (EXIT_UNSUPPORTED, errno, "could not create a private mount namespace\n");

error writes error messages to standard error, where they aren't 
captured in the .out file.  You need to use something like 
FAIL_UNSUPPORTED or printf.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 03/02/2018 12:05 AM, DJ Delorie wrote:
> +  sfd = open (sname, O_RDONLY);
> +  if (sfd < 0)
> +      error (EXIT_UNSUPPORTED, errno,
> +	     "unable to open %s for reading\n", sname);
> +
> +  if (stat (sname, &st) < 0)
> +      error (EXIT_UNSUPPORTED, errno,
> +	     "unable to stat %s\n", sname);

Should this be fstat64?

> +  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
> +  if (dfd < 0)
> +    error (EXIT_UNSUPPORTED, errno,
> +	   "unable to open %s for writing\n", dname);

EXIT_UNSUPPORTED does not appear to be correct here.  These should be 
hard failures IMHO.

Thanks,
Florian

Re: RFC V3 [1/2] test-in-container

[DJ Delorie]

Florian Weimer <fweimer@redhat.com> writes:
> I think we need something to run these tests outside the Makefile 
> framework, similar to testrun.sh.

Mostly, you can do ./testrun.sh support/test-container ./some-program

There's overhead in $srcdir/Rules to handle all the other environment
variables needed for various tests, but you have that as a manual step
the old way too.

Otherwise, if you happen to be running a test with a container setup,
it autodetects that.  So this should work:

$ ./testrun.sh support/test-container nss/tst-nss-test3

where it finds nss/tst-nss-test3/* and sets up the container for you.

Re: RFC V3 [1/2] test-in-container

[DJ Delorie]

Florian Weimer <fweimer@redhat.com> writes:
> On 03/02/2018 12:05 AM, DJ Delorie wrote:
>> +  sfd = open (sname, O_RDONLY);
>> +  if (sfd < 0)
>> +      error (EXIT_UNSUPPORTED, errno,
>> +	     "unable to open %s for reading\n", sname);
>> +
>> +  if (stat (sname, &st) < 0)
>> +      error (EXIT_UNSUPPORTED, errno,
>> +	     "unable to stat %s\n", sname);
>
> Should this be fstat64?

Probably :-)

>> +  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
>> +  if (dfd < 0)
>> +    error (EXIT_UNSUPPORTED, errno,
>> +	   "unable to open %s for writing\n", dname);
>
> EXIT_UNSUPPORTED does not appear to be correct here.  These should be 
> hard failures IMHO.

Are there good guidelines for dealing with failures in the test harness
itself, rather than errors resulting from the actual thing being tested?
Or do we lump them all as "fail" and let the developers sort it out?

RFC V4 test-in-container

[DJ Delorie]

	* Makefile (testroot.pristine): New rules to initialize the
	test-in-container "testroot".
	* Makerules (all-testsuite): Add tests-container.
	* Rules (tests): Add tests-container.
	(binaries-all-tests): Likewise.
	(tests-container): New, run these tests in the testroot container.
	* support/Makefile (others): Add test-container, xmkdirp,
	and links-dso-program.
	* support/links-dso-program-c.c: New.
	* support/links-dso-program.cc: New.
	* support/test-container.c: New.
	* support/mkdirp.c: New.
	* support/support_paths.c: New.
	* support/support.h: Add mkdirp() and support paths prototypes.

	* nss/tst-nss-test3.c: Convert to test-in-container.
	* nss/tst-nss-test3.root/: New.

Changes since V3:

	Style: parens, space-after-periods

	paths moved to support_paths.c

	make NULL comparisons explicit

	xmkdirp returns void - all errors are handled internally

	ready.ts -> install.stamp

	error -> FAIL_UNSUPPORTED

	stat -> fstat in one case.

I used FAIL_UNSUPPORTED for cases where the test case isn't responsible,
like running out of memory or not being able to enter the container.  I
used FAIL_EXIT1 for cases where the test case was indirectly
responsible, like asking to have a non-existing file copied in.

diff --git a/Makefile b/Makefile
index bea4e27f8d..d3e49fe426 100644
--- a/Makefile
+++ b/Makefile
@@ -297,6 +297,46 @@ define summarize-tests
 @! egrep -q -v '^(X?PASS|XFAIL|UNSUPPORTED):' $(objpfx)$1
 endef
 
+# The intention here is to do ONE install of our build into the
+# testroot.pristine/ directory, then rsync (internal to
+# support/test-container) that to testroot.root/ at the start of each
+# test.  That way we can promise each test a "clean" install, without
+# having to do the install for each test.
+#
+# In addition, we have to copy some files into this root in addition
+# to what glibc installs.  For example, many tests require /bin/sh be
+# present, and any shared objects that /bin/sh depends on.  We also
+# build a "test" program in either C or (if available) C++ just so we
+# can copy in any shared objects that GCC-compiled programs depend on.
+
+$(tests-container) $(addsuffix /tests,$(subdirs)) : $(objpfx)testroot.pristine/install.stamp
+$(objpfx)testroot.pristine/install.stamp :
+	test -d $(objpfx)testroot.pristine || \
+	  mkdir $(objpfx)testroot.pristine
+	# We need a working /bin/sh for some of the tests.
+	test -d $(objpfx)testroot.pristine/bin || \
+	  mkdir $(objpfx)testroot.pristine/bin
+	$(test-wrapper) cp /bin/sh $(objpfx)testroot.pristine/bin/sh
+	# Copy these DSOs first so we can overwrite them with our own.
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)testroot.pristine/bin/sh \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)support/links-dso-program \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	$(MAKE) install DESTDIR=$(objpfx)testroot.pristine
+	touch $(objpfx)testroot.pristine/install.stamp
+
 tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
diff --git a/Makerules b/Makerules
index a10a0b4d70..64dab056ea 100644
--- a/Makerules
+++ b/Makerules
@@ -1369,7 +1369,7 @@ xcheck: xtests
 # The only difference between MODULE_NAME=testsuite and MODULE_NAME=nonlib is
 # that almost all internal declarations from config.h, libc-symbols.h, and
 # include/*.h are not available to 'testsuite' code, but are to 'nonlib' code.
-all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras))
+all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(tests-container))
 ifneq (,$(all-testsuite))
 cpp-srcs-left = $(all-testsuite)
 lib := testsuite
diff --git a/Rules b/Rules
index 706c8a749d..aa75afdb4c 100644
--- a/Rules
+++ b/Rules
@@ -130,12 +130,12 @@ others: $(py-const)
 
 ifeq ($(run-built-tests),no)
 tests: $(addprefix $(objpfx),$(filter-out $(tests-unsupported), \
-                                          $(tests) $(tests-internal)) \
+                                          $(tests) $(tests-internal) $(tests-container)) \
 			     $(test-srcs)) $(tests-special) \
 			     $(tests-printers-programs)
 xtests: tests $(xtests-special)
 else
-tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \
+tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) $(tests-container:%=$(objpfx)%.out) \
        $(tests-special) $(tests-printers-out)
 xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special)
 endif
@@ -149,7 +149,7 @@ tests-expected = $(tests) $(tests-internal) $(tests-printers)
 endif
 tests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
-	  $(sort $(tests-expected) $(tests-special-notdir:.out=)) \
+	  $(sort $(tests-expected) $(tests-special-notdir:.out=) $(tests-container)) \
 	  > $(objpfx)subdir-tests.sum
 xtests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
@@ -158,7 +158,7 @@ xtests:
 
 ifeq ($(build-programs),yes)
 binaries-all-notests = $(others) $(sysdep-others)
-binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs)
+binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) $(tests-container)
 binaries-all = $(binaries-all-notests) $(binaries-all-tests)
 binaries-static-notests = $(others-static)
 binaries-static-tests = $(tests-static) $(xtests-static)
@@ -248,6 +248,16 @@ $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
 	$(make-test-out) > $@; \
 	$(evaluate-test)
 
+
+# Any tests that require an isolated container (chroot) in which to
+# run, should be added to tests-container.
+$(tests-container:%=$(objpfx)%.out): $(objpfx)%.out : $(if $(wildcard $(objpfx)%.files),$(objpfx)%.files,/dev/null) $(objpfx)%
+	$(test-wrapper-env) $(run-program-env) $(run-via-rtld-prefix) \
+	  $(common-objpfx)support/test-container env $(run-program-env) \
+	  $(host-test-program-cmd) $($*-ARGS) > $@; \
+	$(evaluate-test)
+
+
 # tests-unsupported lists tests that we will not try to build at all in
 # this configuration.  Note this runs every time because it does not
 # actually create its target.  The dependency on Makefile is meant to
diff --git a/nss/Makefile b/nss/Makefile
index a5cd2aacae..60a28a1519 100644
--- a/nss/Makefile
+++ b/nss/Makefile
@@ -55,11 +55,13 @@ tests-internal		= tst-field
 tests			= test-netdb test-digits-dots tst-nss-getpwent bug17079 \
 			  tst-nss-test1 \
 			  tst-nss-test2 \
-			  tst-nss-test3 \
 			  tst-nss-test4 \
 			  tst-nss-test5
 xtests			= bug-erange
 
+tests-container = \
+			  tst-nss-test3
+
 # Tests which need libdl
 ifeq (yes,$(build-shared))
 tests += tst-nss-files-hosts-erange
diff --git a/nss/tst-nss-test3.c b/nss/tst-nss-test3.c
index d9d708ae7b..4112231778 100644
--- a/nss/tst-nss-test3.c
+++ b/nss/tst-nss-test3.c
@@ -107,7 +107,11 @@ do_test (void)
   int i;
   struct group *g = NULL;
 
-  __nss_configure_lookup ("group", "test1");
+/* Previously we used __nss_configure_lookup to isolate the test
+   from the host environment and to get it to lookup from our new
+   test1 NSS service module, but now this test is run in a different
+   root filesystem via the test-container support and we directly
+   configure the use of the test1 NSS service.  */
 
   setgrent ();
 
diff --git a/nss/tst-nss-test3.root/etc/nsswitch.conf b/nss/tst-nss-test3.root/etc/nsswitch.conf
new file mode 100644
index 0000000000..5e08fe5eea
--- /dev/null
+++ b/nss/tst-nss-test3.root/etc/nsswitch.conf
@@ -0,0 +1 @@
+group	test1
diff --git a/nss/tst-nss-test3.root/files.txt b/nss/tst-nss-test3.root/files.txt
new file mode 100644
index 0000000000..a10beb1e6c
--- /dev/null
+++ b/nss/tst-nss-test3.root/files.txt
@@ -0,0 +1,2 @@
+cp $B/nss/libnss_test1.so $L/libnss_test1.so.2
+cp $B/nss/libnss_test2.so $L/libnss_test2.so.2
diff --git a/support/Makefile b/support/Makefile
index 652d2cdf69..9fa8552180 100644
--- a/support/Makefile
+++ b/support/Makefile
@@ -53,6 +53,7 @@ libsupport-routines = \
   support_format_netent \
   support_isolate_in_subprocess \
   support_openpty \
+  support_paths \
   support_quote_blob \
   support_record_failure \
   support_run_diff \
@@ -84,6 +85,7 @@ libsupport-routines = \
   xmalloc \
   xmemstream \
   xmkdir \
+  xmkdirp \
   xmmap \
   xmprotect \
   xmunmap \
@@ -151,6 +153,31 @@ ifeq ($(build-shared),yes)
 libsupport-inhibit-o += .o
 endif
 
+CFLAGS-support_paths.c = \
+		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
+		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
+		-DINSTDIR_PATH=\"$(prefix)\" \
+		-DLIBDIR_PATH=\"$(libdir)\"
+
+others: $(objpfx)test-container $(objpfx)links-dso-program
+
+LDLIBS-test-container = $(libsupport)
+
+others += test-container
+others-noinstall += test-container
+
+$(objpfx)test-container : $(libsupport)
+
+# This exists only so we can guess which OS DSOs we need to copy into
+# the testing container.
+ifeq (,$(CXX))
+$(objpfx)links-dso-program : $(objpfx)links-dso-program-c.o
+	$(LINK.o) -o $@ $^
+else
+$(objpfx)links-dso-program : $(objpfx)links-dso-program.o
+	$(LINK.o) -o $@ $^ -lstdc++
+endif
+
 tests = \
   README-testing \
   tst-support-namespace \
diff --git a/support/links-dso-program-c.c b/support/links-dso-program-c.c
new file mode 100644
index 0000000000..398ec675a0
--- /dev/null
+++ b/support/links-dso-program-c.c
@@ -0,0 +1,5 @@
+int
+main (void)
+{
+  return 0;
+}
diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
new file mode 100644
index 0000000000..398ec675a0
--- /dev/null
+++ b/support/links-dso-program.cc
@@ -0,0 +1,5 @@
+int
+main (void)
+{
+  return 0;
+}
diff --git a/support/support.h b/support/support.h
index b61fe0735c..192b9e1424 100644
--- a/support/support.h
+++ b/support/support.h
@@ -76,6 +76,19 @@ char *xasprintf (const char *format, ...)
 char *xstrdup (const char *);
 char *xstrndup (const char *, size_t);
 
+/* Equivalent of "mkdir -p".  */
+void xmkdirp (const char *, int);
+
+/* These point to the TOP of the source/build tree, not your (or
+   support's) subdirectory.  */
+extern const char support_srcdir_root[];
+extern const char support_objdir_root[];
+
+/* Corresponds to the --prefix= passed to configure.  */
+extern const char support_install_prefix[];
+/* Corresponds to the install's lib/ or lib64/ directory.  */
+extern const char support_libdir_prefix[];
+
 __END_DECLS
 
 #endif /* SUPPORT_H */
diff --git a/support/support_paths.c b/support/support_paths.c
new file mode 100644
index 0000000000..a1c22315bd
--- /dev/null
+++ b/support/support_paths.c
@@ -0,0 +1,51 @@
+/* Various paths that might be needed.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/support.h>
+#include <support/check.h>
+
+/* The idea here is to make various makefile-level paths available to
+   support programs, as canonicalized absolute paths.  */
+
+/* These point to the TOP of the source/build tree, not your (or
+   support's) subdirectory.  */
+#ifdef SRCDIR_PATH
+const char support_srcdir_root[] = SRCDIR_PATH;
+#else
+# error please -DSRCDIR_PATH=something in the Makefile
+#endif
+
+#ifdef OBJDIR_PATH
+const char support_objdir_root[] = OBJDIR_PATH;
+#else
+# error please -DOBJDIR_PATH=something in the Makefile
+#endif
+
+#ifdef INSTDIR_PATH
+/* Corresponds to the --prefix= passed to configure.  */
+const char support_install_prefix[] = INSTDIR_PATH;
+#else
+# error please -DINSTDIR_PATH=something in the Makefile
+#endif
+
+#ifdef LIBDIR_PATH
+/* Corresponds to the install's lib/ or lib64/ directory.  */
+const char support_libdir_prefix[] = LIBDIR_PATH;
+#else
+# error please -DLIBDIR_PATH=something in the Makefile
+#endif
diff --git a/support/test-container.c b/support/test-container.c
new file mode 100644
index 0000000000..cb232a1baa
--- /dev/null
+++ b/support/test-container.c
@@ -0,0 +1,940 @@
+/* Run a test case in an isolated namespace.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define __USE_LARGEFILE64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sched.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <sys/sysmacros.h>
+#include <ctype.h>
+#include <utime.h>
+#include <errno.h>
+#include <error.h>
+
+#include <support/support.h>
+#include "check.h"
+#include "test-driver.h"
+
+int verbose = 0;
+
+/* Running a test in a container is tricky.  There are two main
+   categories of things to do:
+
+   1. "Once" actions, like setting up the container and doing an
+      install into it.
+
+   2. "Per-test" actions, like copying in support files and
+      configuring the container.
+
+
+   "Once" actions:
+
+   * mkdir $buildroot/testroot.pristine/
+   * install into it
+   * rsync to $buildroot/testroot.root/
+
+   "Per-test" actions:
+   * maybe rsync to $buildroot/testroot.root/
+   * copy support files and test binary
+   * chroot/unshare
+   * set up any mounts (like /proc)
+
+   Magic files:
+
+   For test $srcdir/foo/mytest.c we look for $srcdir/foo/mytest.root and, if found...
+
+   * mytest.root/ is rsync'd into container
+   * mytest.root/preclean.txt causes fresh rsync (with delete) before test if present
+   * mytest.root/files.txt has a list of files to copy - TBD
+     ($B/ or $S/ for build and source directories)
+       syntax:
+         # comment
+         mv FILE FILE
+	 cp FILE FILE
+	 rm FILE
+	 FILE must start with $B/, $S/, $I/, $L/, or /
+	  (expands to build dir, source dir, install dir, library dir
+	   (in container), or container's root)
+   * mytest.root/postclean.txt causes fresh rsync (with delete) after test if present
+
+   Note that $srcdir/foo/mytest.files may be used instead of a
+   files.txt in the sysroot, if there is no other reason for a sysroot.
+
+   Design goals:
+
+   * independent of other packages which may not be installed (like
+     rsync or Docker, or even "cp")
+
+   * Simple, easy to review code (i.e. prefer simple naive code over
+     complex efficient code)
+
+   TBD:
+
+   * The current implementation is not parallel-make-safe, as one test
+     could be modifying the chroot while another is running against
+     it.
+
+*/
+
+/*--------------------------------------------------*/
+/* Utility Functions */
+
+/* Temporarily concatenate multiple strings into one.  Allows up to 10
+   temporary results; use strdup () if you need them to be
+   permanent.  */
+
+static char *
+concat (const char *str, ...)
+{
+  /* Assume initialized to NULL/zero.  */
+  static char *bufs[10];
+  static size_t buflens[10];
+  static int bufn = 0;
+  int n;
+  size_t len;
+  va_list ap, ap2;
+  char *cp;
+  char *next;
+
+  va_start (ap, str);
+  va_copy (ap2, ap);
+
+  n = bufn;
+  bufn = (bufn + 1) % 10;
+  len = strlen (str);
+
+  while ((next = va_arg (ap, char *)) != NULL)
+    len = len + strlen (next);
+
+  va_end (ap);
+
+  if (bufs[n] == NULL)
+    {
+      bufs[n] = malloc (len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+  else if (buflens[n] < len + 1)
+    {
+      bufs[n] = realloc (bufs[n], len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+
+  strcpy (bufs[n], str);
+  cp = strchr (bufs[n], '\0');
+  while ((next = va_arg (ap2, char *)) != NULL)
+    {
+      strcpy (cp, next);
+      cp = strchr (cp, '\0');
+    }
+  *cp = 0;
+  va_end (ap2);
+
+  return bufs[n];
+}
+
+/* Try to mount SRC onto DEST.  */
+
+static void
+trymount (const char *src, const char *dest)
+{
+  if (mount (src, dest, "", MS_BIND, NULL) < 0)
+    FAIL_UNSUPPORTED ("can't mount %s onto %s\n", src, dest);
+}
+
+/* Special case of above for devices like /dev/zero where we have to
+   mount a device over a device, not a directory over a directory.  */
+
+static void
+devmount (const char *new_root_path, const char *which)
+{
+  int fd;
+  fd = open (concat (new_root_path, "/dev/", which, NULL),
+	     O_CREAT | O_TRUNC | O_RDWR, 0777);
+  close (fd);
+
+  trymount (concat ("/dev/", which, NULL),
+	    concat (new_root_path, "/dev/", which, NULL));
+}
+
+/* Returns true if the string "looks like" an environement variable
+   being set.  */
+
+static int
+is_env_setting (const char *a)
+{
+  int count_name = 0;
+
+  while (*a)
+    {
+      if (isalnum (*a) || *a == '_')
+	++count_name;
+      else if (*a == '=' && count_name > 0)
+	return 1;
+      else
+	return 0;
+      ++a;
+    }
+  return 0;
+}
+
+/* Break the_line into words and store in the_words.  Max nwords,
+   returns actual count.  */
+static int
+tokenize (char *the_line, char **the_words, int nwords)
+{
+  int rv = 0;
+
+  while (nwords > 0)
+    {
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && isspace (*the_line))
+	++the_line;
+
+      /* End of line?  */
+      if (*the_line == 0)
+	return rv;
+
+      /* THE_LINE points to a non-whitespace character, so we have a
+	 word.  */
+      *the_words = the_line;
+      ++the_words;
+      nwords--;
+      ++rv;
+
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && ! isspace (*the_line))
+	++the_line;
+
+      /* We now point at the trailing NUL *or* some whitespace.  */
+      if (*the_line == 0)
+	return rv;
+
+      /* It was whitespace, skip and keep tokenizing.  */
+      *the_line++ = 0;
+    }
+
+  /* We get here if we filled the words buffer.  */
+  return rv;
+}
+
+/*--------------------------------------------------*/
+/* mini-RSYNC implementation.  Optimize later.      */
+
+/* Set this to 1 if you need to debug the rsync function.  */
+#define RTRACE 0
+
+/* A few routines for an "rsync buffer" which stores the paths we're
+   working on.  We continuously grow and shrink the paths in each
+   buffer so there's lot of re-use.  */
+
+/* We rely on "initialized to zero" to set these up.  */
+typedef struct
+{
+  char *buf;
+  size_t len;
+  size_t size;
+} path_buf;
+
+static path_buf spath, dpath;
+
+static void
+r_setup (char *path, path_buf * pb)
+{
+  size_t len = strlen (path);
+  if (pb->buf == NULL || pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      if (pb->buf == NULL)
+	pb->buf = (char *) malloc (sz);
+      else
+	pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	FAIL_UNSUPPORTED ("Out of memory while rsyncing\n");
+
+      pb->size = sz;
+    }
+  strcpy (pb->buf, path);
+  pb->len = len;
+}
+
+static void
+r_append (const char *path, path_buf * pb)
+{
+  size_t len = strlen (path) + pb->len;
+  if (pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	FAIL_UNSUPPORTED ("Out of memory while rsyncing\n");
+
+      pb->size = sz;
+    }
+  strcpy (pb->buf + pb->len, path);
+  pb->len = len;
+}
+
+static int
+file_exists (char *path)
+{
+  struct stat st;
+  if (lstat (path, &st) == 0)
+    return 1;
+  return 0;
+}
+
+static void
+recursive_remove (char *path)
+{
+  pid_t child;
+  int status;
+  /* FIXME: re-implement without external dependencies at some point.
+     Fortunately, this runs outside the container.  */
+
+  child = fork ();
+
+  switch (child) {
+  case -1:
+    FAIL_UNSUPPORTED ("Unable to fork");
+  case 0:
+    /* Child.  */
+    execlp ("rm", "rm", "-rf", path, NULL);
+  default:
+    /* Parent.  */
+    waitpid (child, &status, 0);
+    break;
+  }
+}
+
+/* Used for both rsync and the files.txt "cp" command.  */
+
+static void
+copy_one_file (const char *sname, const char *dname)
+{
+  int sfd, dfd;
+  struct stat st;
+  struct utimbuf times;
+
+  sfd = open (sname, O_RDONLY);
+  if (sfd < 0)
+    FAIL_EXIT1 ("unable to open %s for reading\n", sname);
+
+  if (fstat (sfd, &st) < 0)
+    FAIL_EXIT1 ("unable to fstat %s\n", sname);
+
+  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+  if (dfd < 0)
+    FAIL_EXIT1 ("unable to open %s for writing\n", dname);
+
+  if (copy_file_range (sfd, 0, dfd, 0, st.st_size, 0) != st.st_size)
+    FAIL_EXIT1 ("cannot copy file %s to %s\n", sname, dname);
+
+  close (sfd);
+  close (dfd);
+
+  chmod (dname, st.st_mode & 0777);
+
+  times.actime = st.st_atime;
+  times.modtime = st.st_mtime;
+  utime (dname, &times);
+}
+
+/* We don't check *everything* about the two files to see if a copy is
+   needed, just the minimum to make sure we get the latest copy.  */
+static int
+need_sync (char *ap, char *bp, struct stat *a, struct stat *b)
+{
+  if ((a->st_mode & S_IFMT) != (b->st_mode & S_IFMT))
+    return 1;
+
+  if (S_ISLNK (a->st_mode))
+    {
+      int rv;
+      char *al, *bl;
+
+      if (a->st_size != b->st_size)
+	return 1;
+
+      al = (char *) malloc (a->st_size + 1);
+      bl = (char *) malloc (b->st_size + 1);
+      readlink (ap, al, a->st_size + 1);
+      readlink (bp, bl, b->st_size + 1);
+      al[a->st_size] = 0;
+      bl[b->st_size] = 0;
+      rv = strcmp (al, bl);
+      free (al);
+      free (bl);
+      if (rv == 0)
+	return 0; /* links are same */
+      return 1; /* links differ */
+    }
+
+#if RTRACE
+  if (a->st_size != b->st_size)
+    printf ("SIZE\n");
+  if ((a->st_mode & 0777) != (b->st_mode & 0777))
+    printf ("MODE\n");
+  if (a->st_mtime != b->st_mtime)
+    printf ("TIME\n");
+#endif
+
+  if (a->st_size == b->st_size
+      && ((a->st_mode & 0777) == (b->st_mode & 0777))
+      && a->st_mtime == b->st_mtime)
+    return 0;
+
+  return 1;
+}
+
+static void
+rsync_1 (path_buf * src, path_buf * dest, int and_delete)
+{
+  DIR *dir;
+  struct dirent *de;
+  struct stat s, d;
+
+  r_append ("/", src);
+  r_append ("/", dest);
+#if RTRACE
+  printf ("sync %s to %s %s\n", src->buf, dest->buf,
+	  and_delete ? "and delete" : "");
+#endif
+
+  size_t staillen = src->len;
+
+  size_t dtaillen = dest->len;
+
+  dir = opendir (src->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      if (lstat (src->buf, &s) != 0)
+	FAIL_EXIT1 ("%s obtained by readdir, but stat failed.\n", src->buf);
+
+      // It's OK if this one fails, since we know the file might be missing.
+      lstat (dest->buf, &d);
+
+      if (! need_sync (src->buf, dest->buf, &s, &d))
+	{
+	  if (S_ISDIR (s.st_mode))
+	    rsync_1 (src, dest, and_delete);
+	  continue;
+	}
+
+      if (d.st_mode != ~0)
+	switch (d.st_mode & S_IFMT)
+	  {
+	  case S_IFDIR:
+	    if (!S_ISDIR (s.st_mode))
+	      {
+#if RTRACE
+		printf ("-D %s\n", dest->buf);
+#endif
+		recursive_remove (dest->buf);
+	      }
+	    break;
+
+	  default:
+#if RTRACE
+	    printf ("-F %s\n", dest->buf);
+#endif
+	    unlink (dest->buf);
+	    break;
+	  }
+
+      switch (s.st_mode & S_IFMT)
+	{
+	case S_IFREG:
+#if RTRACE
+	  printf ("+F %s\n", dest->buf);
+#endif
+	  copy_one_file (src->buf, dest->buf);
+	  break;
+
+	case S_IFDIR:
+#if RTRACE
+	  printf ("+D %s\n", dest->buf);
+#endif
+	  mkdir (dest->buf, (s.st_mode & 0777) | 0700);
+	  rsync_1 (src, dest, and_delete);
+	  break;
+
+	case S_IFLNK:
+	  {
+	    char *lp = (char *) malloc (s.st_size + 1);
+#if RTRACE
+	    printf ("+L %s\n", dest->buf);
+#endif
+	    readlink (src->buf, lp, s.st_size + 1);
+	    lp[s.st_size] = 0;
+	    symlink (lp, dest->buf);
+	    free (lp);
+	    break;
+	  }
+
+	default:
+	  break;
+	}
+    }
+
+  closedir (dir);
+  src->len = staillen;
+  src->buf[staillen] = 0;
+  dest->len = dtaillen;
+  dest->buf[dtaillen] = 0;
+
+  if (!and_delete)
+    return;
+
+  /* The rest of this function removes any files/directories in DEST
+     that do not exist in SRC.  This is triggered as part of a
+     preclean or postsclean step.  */
+
+  dir = opendir (dest->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      
+      if (lstat (dest->buf, &d) != 0)
+	FAIL_EXIT1 ("%s obtained by readdir, but stat failed.\n", dest->buf);
+
+      if (s.st_mode == ~0)
+	{
+	  /* dest exists and src doesn't, clean it.  */
+	  switch (d.st_mode & S_IFMT)
+	    {
+	    case S_IFDIR:
+	      if (!S_ISDIR (s.st_mode))
+		{
+#if RTRACE
+		  printf ("-D %s\n", dest->buf);
+#endif
+		  recursive_remove (dest->buf);
+		}
+	      break;
+
+	    default:
+#if RTRACE
+	      printf ("-F %s\n", dest->buf);
+#endif
+	      unlink (dest->buf);
+	      break;
+	    }
+	}
+    }
+
+  closedir (dir);
+}
+
+static void
+rsync (char *src, char *dest, int and_delete)
+{
+  r_setup (src, &spath);
+  r_setup (dest, &dpath);
+
+  rsync_1 (&spath, &dpath, and_delete);
+}
+
+/*--------------------------------------------------*/
+/* Main */
+
+int
+main (int argc, char **argv)
+{
+  pid_t child;
+  char *pristine_root_path;
+  char *new_root_path;
+  char *new_cwd_path;
+  char *new_objdir_path;
+  char *new_srcdir_path;
+  char **new_child_proc;
+  char *command_root;
+  char *command_base;
+  char *so_base;
+  int do_postclean = 0;
+
+  uid_t original_uid;
+  gid_t original_gid;
+  int UMAP;
+  int GMAP;
+  char tmp[100];
+  struct stat st;
+  int lock_fd;
+
+  setbuf (stdout, NULL);
+
+  // The command line we're expecting looks like this:
+  // env <set some vars> ld.so <library path> test-binary
+
+  // We need to peel off any "env" or "ld.so" portion of the command
+  // line, and keep track of which env vars we should preserve and
+  // which we drop.
+
+  if (argc < 2)
+    {
+      fprintf (stderr, "Usage: containerize <program to run> <args...>\n");
+      exit (1);
+    }
+
+  if (strcmp (argv[1], "-v") == 0)
+    {
+      verbose = 1;
+      ++argv;
+      --argc;
+    }
+
+  if (strcmp (argv[1], "env") == 0)
+    {
+      ++argv;
+      --argc;
+      while (is_env_setting (argv[1]))
+	{
+	  // List variables we do NOT want to propogate.
+#if 0
+	  // until we discover why locale/iconv tests don't
+	  // work against an installed tree...
+	  if (memcmp (argv[1], "GCONV_PATH=", 11)
+	      && memcmp (argv[1], "LOCPATH=", 8))
+#endif
+	    {
+	      // Need to keep these.  Note that putenv stores a
+	      // pointer to our argv.
+	      putenv (argv[1]);
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  if (strncmp (argv[1], concat (support_objdir_root, "/elf/ld-linux-", NULL),
+	       strlen (support_objdir_root) + 14) == 0)
+    {
+      ++argv;
+      --argc;
+      while (argv[1][0] == '-')
+	{
+	  if (strcmp (argv[1], "--library-path") == 0)
+	    {
+	      ++argv;
+	      --argc;
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  pristine_root_path = strdup (concat (support_objdir_root, "/testroot.pristine", NULL));
+  new_root_path = strdup (concat (support_objdir_root, "/testroot.root", NULL));
+  new_cwd_path = get_current_dir_name ();
+  new_child_proc = argv + 1;
+
+  lock_fd = open (concat (pristine_root_path, "/lock.fd", NULL),
+		 O_CREAT | O_TRUNC | O_RDWR, 0666);
+  if (lock_fd < 0)
+    FAIL_UNSUPPORTED ("Cannot create testroot lock.\n");
+
+  while (flock (lock_fd, LOCK_EX) != 0)
+    {
+      if (errno != EINTR)
+	FAIL_UNSUPPORTED ("Cannot lock testroot.\n");
+    }
+
+  xmkdirp (new_root_path, 0755);
+
+  /* We look for extra setup info in a subdir in the same spot as the
+     test, with the same name but a ".root" extension.  This is that
+     directory.  We try to look in the source tree if the path we're
+     given refers to the build tree, but we rely on the path to be
+     absolute.  This is what the glibc makefiles do.  */
+  command_root = concat (argv[1], ".root", NULL);
+  if (strncmp (command_root, support_objdir_root, strlen (support_objdir_root)) == 0
+      && command_root[strlen (support_objdir_root)] == '/')
+    command_root = concat (support_srcdir_root, argv[1] + strlen (support_objdir_root), ".root", NULL);
+  command_root = strdup (command_root);
+
+  /* This cuts off the ".root" we appended above.  */
+  command_base = strdup (command_root);
+  command_base[strlen (command_base) - 5] = 0;
+
+  /* Shared object base directory.  */
+  so_base = strdup (argv[1]);
+  if (strrchr (so_base, '/') != NULL)
+    strrchr (so_base, '/')[1] = 0;
+
+  if (file_exists (concat (command_root, "/postclean.txt", NULL)))
+    do_postclean = 1;
+
+  rsync (pristine_root_path, new_root_path,
+	 1 || file_exists (concat (command_root, "/preclean.txt", NULL)));
+
+  if (stat (command_root, &st) >= 0
+      && S_ISDIR (st.st_mode))
+    rsync (command_root, new_root_path, 0);
+
+  new_objdir_path = strdup (concat (new_root_path, support_objdir_root, NULL));
+  new_srcdir_path = strdup (concat (new_root_path, support_srcdir_root, NULL));
+
+  /* new_cwd_path starts with '/' so no "/" needed between the two.  */
+  xmkdirp (concat (new_root_path, new_cwd_path, NULL), 0755);
+  xmkdirp (new_srcdir_path, 0755);
+  xmkdirp (new_objdir_path, 0755);
+
+  original_uid = getuid ();
+  original_gid = getgid ();
+
+  /* Handle the cp/mv/rm "script" here.  */
+  {
+    char *the_line = NULL;
+    size_t line_len = 0;
+    char *fname = concat (command_root, "/files.txt", NULL);
+    char *the_words[3];
+    FILE *f = fopen (fname, "r");
+
+    if (verbose && f)
+      fprintf (stderr, "reading %s\n", fname);
+
+    if (f == NULL)
+      {
+	/* Try foo.files instead of foo.root/files.txt, as a shortcut.  */
+	fname = concat (command_base, ".files", NULL);
+	f = fopen (fname, "r");
+	if (verbose && f)
+	  fprintf (stderr, "reading %s\n", fname);
+      }
+
+#if 0
+    /* I don't want to add this until we know we need it, but here it
+       is...  */
+    if (f == NULL)
+      {
+	/* Look for a Makefile-generated one also.  */
+	fname = concat (argv[1], ".files", NULL);
+	f = fopen (fname, "r");
+      }
+#endif
+
+    /* This is where we "interpret" the mini-script which is <test>.files.  */
+    if (f != NULL)
+      {
+	while (getline (&the_line, &line_len, f) > 0)
+	  {
+	    int nt = tokenize (the_line, the_words, 3);
+	    int i;
+
+	    for (i = 1; i < nt; ++i)
+	      {
+		if (memcmp (the_words[i], "$B/", 3) == 0)
+		  the_words[i] = concat (support_objdir_root, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$S/", 3) == 0)
+		  the_words[i] = concat (support_srcdir_root, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$I/", 3) == 0)
+		  the_words[i] = concat (new_root_path, support_install_prefix, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$L/", 3) == 0)
+		  the_words[i] = concat (new_root_path, support_libdir_prefix, the_words[i] + 2, NULL);
+		else if (the_words[i][0] == '/')
+		  the_words[i] = concat (new_root_path, the_words[i], NULL);
+	      }
+
+	    if (nt == 3 && the_words[2][strlen (the_words[2]) - 1] == '/')
+	      {
+		char *r = strrchr (the_words[1], '/');
+		if (r)
+		  the_words[2] = concat (the_words[2], r + 1, NULL);
+		else
+		  the_words[2] = concat (the_words[2], the_words[1], NULL);
+	      }
+
+	    if (nt == 2 && strcmp (the_words[0], "so") == 0)
+	      {
+		the_words[2] = concat (new_root_path, support_libdir_prefix, "/", the_words[1], NULL);
+		the_words[1] = concat (so_base, the_words[1], NULL);
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "cp") == 0)
+	      {
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "mv") == 0)
+	      {
+		rename (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "chmod") == 0)
+	      {
+		long int m;
+		m = strtol (the_words[1], NULL, 0);
+		chmod (the_words[2], m);
+	      }
+	    else if (nt == 2 && strcmp (the_words[0], "rm") == 0)
+	      {
+		unlink (the_words[1]);
+	      }
+	    else if (nt > 0 && the_words[0][0] != '#')
+	      {
+		printf ("\033[31minvalid [%s]\033[0m\n", the_words[0]);
+	      }
+	  }
+	fclose (f);
+      }
+  }
+
+  // The unshare here gives us our own spaces and capabilities.
+  if (unshare (CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS) < 0)
+    FAIL_UNSUPPORTED ("unable to unshare user/fs, ");
+
+  /* Some systems, by default, all mounts leak out of the namespace.  */
+  if (mount ("none", "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0)
+    FAIL_UNSUPPORTED ("could not create a private mount namespace\n");
+
+  trymount (support_srcdir_root, new_srcdir_path);
+  trymount (support_objdir_root, new_objdir_path);
+
+  xmkdirp (concat (new_root_path, "/dev", NULL), 0755);
+  devmount (new_root_path, "null");
+  devmount (new_root_path, "zero");
+  devmount (new_root_path, "urandom");
+
+  // We're done with the "old" root, switch to the new one.
+  if (chroot (new_root_path) < 0)
+    FAIL_UNSUPPORTED ("Can't chroot to %s - ", new_root_path);
+
+  if (chdir (new_cwd_path) < 0)
+    FAIL_UNSUPPORTED ("Can't cd to new %s - ", new_cwd_path);
+
+  /* To complete the containerization, we need to fork () at least
+     once.  We can't exec, nor can we somehow link the new child to
+     our parent.  So we run the child and propogate it's exit status
+     up.  */
+  child = fork ();
+  if (child < 0)
+    FAIL_UNSUPPORTED ("Unable to fork");
+  else if (child > 0)
+    {
+      /* Parent.  */
+      int status;
+      waitpid (child, &status, 0);
+
+      /* There's a bit of magic here, since the buildroot is mounted
+	 in our space, the paths are still valid, and since the mounts
+	 aren't recursive, it sees *only* the built root, not anything
+	 we would normally se if we rsync'd to "/" like mounted /dev
+	 files.  */
+      if (do_postclean)
+	  rsync (pristine_root_path, new_root_path, 1);
+
+      if (WIFEXITED (status))
+	exit (WEXITSTATUS (status));
+
+      if (WIFSIGNALED (status))
+	{
+	  printf ("%%SIGNALLED%%\n");
+	  exit (77);
+	}
+
+      printf ("%%EXITERROR%%\n");
+      exit (78);
+    }
+
+  /* The rest is the child process, which is now PID 1 and "in" the
+     new root.  */
+
+  mkdir ("/tmp", 0755);
+
+  // Now that we're pid 1 (effectively "root") we can mount /proc
+  mkdir ("/proc", 0777);
+  if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
+    FAIL_UNSUPPORTED ("Unable to mount /proc: ");
+
+  // We map our original UID to the same UID in the container so we
+  // can own our own files normally
+  UMAP = open ("/proc/self/uid_map", O_WRONLY);
+  if (UMAP < 0)
+    FAIL_UNSUPPORTED ("can't write to /proc/self/uid_map\n");
+
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_uid, (long long) original_uid);
+  write (UMAP, tmp, strlen (tmp));
+  close (UMAP);
+
+  // We must disable setgroups () before we can map our groups, else we
+  // get EPERM.
+  GMAP = open ("/proc/self/setgroups", O_WRONLY);
+  if (GMAP >= 0)
+    {
+      /* We support kernels old enough to not have this.  */
+      write (GMAP, "deny\n", 5);
+      close (GMAP);
+    }
+
+  // We map our original GID to the same GID in the container so we
+  // can own our own files normally
+  GMAP = open ("/proc/self/gid_map", O_WRONLY);
+  if (GMAP < 0)
+    FAIL_UNSUPPORTED ("can't write to /proc/self/gid_map\n");
+
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_gid, (long long) original_gid);
+  write (GMAP, tmp, strlen (tmp));
+  close (GMAP);
+
+  // Now run the child
+  execvp (new_child_proc[0], new_child_proc);
+
+  // Or don't run the child?
+  FAIL_EXIT1 ("Unable to exec %s\n", new_child_proc[0]);
+
+  // Because gcc won't know error () never returns...
+  exit (EXIT_UNSUPPORTED);
+}
diff --git a/support/xmkdirp.c b/support/xmkdirp.c
new file mode 100644
index 0000000000..a15fddea48
--- /dev/null
+++ b/support/xmkdirp.c
@@ -0,0 +1,67 @@
+/* Error-checking replacement for "mkdir -p".
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/support.h>
+#include <support/check.h>
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+/* Equivalent of "mkdir -p".  Any failures cause FAIL_EXIT1 so no
+   return code is needed.  */
+
+void
+xmkdirp (const char *path, int mode)
+{
+  struct stat s;
+  const char *slash_p;
+  int rv;
+
+  if (path[0] == 0)
+    return;
+
+  if (stat (path, &s) == 0)
+    {
+      if (S_ISDIR (s.st_mode))
+	return;
+      errno = EEXIST;
+      FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);
+    }
+
+  slash_p = strrchr (path, '/');
+  if (slash_p != NULL)
+    {
+      while (slash_p > path && slash_p[-1] == '/')
+	--slash_p;
+      if (slash_p > path)
+	{
+	  char *parent = xstrndup (path, slash_p - path);
+	  xmkdirp (parent, mode);
+	  free (parent);
+	}
+    }
+
+  rv = mkdir (path, mode);
+  if (rv != 0)
+    FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);
+
+  return;
+}

Re: RFC V3 [1/2] test-in-container

[Florian Weimer]

On 06/28/2018 11:07 PM, DJ Delorie wrote:
> Are there good guidelines for dealing with failures in the test harness
> itself, rather than errors resulting from the actual thing being tested?
> Or do we lump them all as "fail" and let the developers sort it out?

We treat it as FAIL if we cannot attribute it to a valid, expected 
environmental limitation (such as a kernel which is too old, but still 
supported).  I don't think there is any other way to do this.

Thanks,
Florian

Re: RFC V4 test-in-container

[Joseph Myers]

On Thu, 28 Jun 2018, DJ Delorie wrote:

> +	# We need a working /bin/sh for some of the tests.
> +	test -d $(objpfx)testroot.pristine/bin || \
> +	  mkdir $(objpfx)testroot.pristine/bin
> +	$(test-wrapper) cp /bin/sh $(objpfx)testroot.pristine/bin/sh
> +	# Copy these DSOs first so we can overwrite them with our own.
> +	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
> +			        $(objpfx)testroot.pristine/bin/sh \

I remain concerned about copying /bin/sh rather than having a local 
sh-substitute.  Can using the newly built dynamic linker to trace what's 
required by /bin/sh really work even in as simple a case as testing i386 
glibc on an x86_64 system, where /bin/sh is a 64-bit binary?

-- 
Joseph S. Myers
joseph@codesourcery.com

Re: RFC V4 test-in-container

[DJ Delorie]

Joseph Myers <joseph@codesourcery.com> writes:
> I remain concerned about copying /bin/sh rather than having a local 
> sh-substitute.  Can using the newly built dynamic linker to trace what's 
> required by /bin/sh really work even in as simple a case as testing i386 
> glibc on an x86_64 system, where /bin/sh is a 64-bit binary?

I can write a /bin/sh substitute as long as none of our tests test *it*
and not just the calls that require it ;-)

I think one caveat is that we need to decide where and how to document
it so that future test writers don't assume full /bin/sh functionality.

But I suspect it would work; the scriptlet copies libc.so from the host
system as well as everything else.  If it's the "wrong" libc.so (/lib vs
/lib64) we won't override it with our own.  It would only be a
"sufficiently reliable coincidence" though.

Re: RFC V4 test-in-container

[Joseph Myers]

On Fri, 29 Jun 2018, DJ Delorie wrote:

> 
> Joseph Myers <joseph@codesourcery.com> writes:
> > I remain concerned about copying /bin/sh rather than having a local 
> > sh-substitute.  Can using the newly built dynamic linker to trace what's 
> > required by /bin/sh really work even in as simple a case as testing i386 
> > glibc on an x86_64 system, where /bin/sh is a 64-bit binary?
> 
> I can write a /bin/sh substitute as long as none of our tests test *it*
> and not just the calls that require it ;-)

I think that's the right thing to do - avoid any dependence on host 
binaries or libraries beyond the libgcc_s / libstdc++ etc. shared 
libraries, *for the ABI for which glibc is built*, on which the compiler 
automatically introduces dependencies.

-- 
Joseph S. Myers
joseph@codesourcery.com

RFC V5 test-in-container

[DJ Delorie]

Joseph Myers <joseph@codesourcery.com> writes:
>> I can write a /bin/sh substitute
>
> I think that's the right thing to do.

I tested by running all of posix/ and libio/ tests inside the container,
decided to also add /bin/echo and /bin/true since they were (1) used,
and (2) trivial.  Let the slippery slope begin :-)

Changes since V4:

	add shell-container, echo-container, true-container helper programs

	add $($*-ENV) to test-container rules

Add test-in-container infrastructure

	* Makefile (testroot.pristine): New rules to initialize the
	test-in-container "testroot".
	* Makerules (all-testsuite): Add tests-container.
	* Rules (tests): Add tests-container.
	(binaries-all-tests): Likewise.
	(tests-container): New, run these tests in the testroot container.
	* support/Makefile (others): Add *-container, support_paths.c,
	xmkdirp, and links-dso-program.
	* support/links-dso-program-c.c: New.
	* support/links-dso-program.cc: New.
	* support/test-container.c: New.
	* support/shell-container.c: New.
	* support/echo-container.c: New.
	* support/true-container.c: New.
	* support/mkdirp.c: New.
	* support/support_paths.c: New.
	* support/support.h: Add mkdirp() and support paths prototypes.

* nss/tst-nss-test3.c: Convert to test-in-container.
* nss/tst-nss-test3.root/: New.

diff --git a/Makefile b/Makefile
index bea4e27f8d..806b61ad83 100644
--- a/Makefile
+++ b/Makefile
@@ -297,6 +297,48 @@ define summarize-tests
 @! egrep -q -v '^(X?PASS|XFAIL|UNSUPPORTED):' $(objpfx)$1
 endef
 
+# The intention here is to do ONE install of our build into the
+# testroot.pristine/ directory, then rsync (internal to
+# support/test-container) that to testroot.root/ at the start of each
+# test.  That way we can promise each test a "clean" install, without
+# having to do the install for each test.
+#
+# In addition, we have to copy some files into this root in addition
+# to what glibc installs.  For example, many tests require /bin/sh be
+# present, and any shared objects that /bin/sh depends on.  We also
+# build a "test" program in either C or (if available) C++ just so we
+# can copy in any shared objects that GCC-compiled programs depend on.
+
+$(tests-container) $(addsuffix /tests,$(subdirs)) : $(objpfx)testroot.pristine/install.stamp
+$(objpfx)testroot.pristine/install.stamp :
+	test -d $(objpfx)testroot.pristine || \
+	  mkdir $(objpfx)testroot.pristine
+	# We need a working /bin/sh for some of the tests.
+	test -d $(objpfx)testroot.pristine/bin || \
+	  mkdir $(objpfx)testroot.pristine/bin
+	cp $(objpfx)support/shell-container $(objpfx)testroot.pristine/bin/sh
+	cp $(objpfx)support/echo-container $(objpfx)testroot.pristine/bin/echo
+	cp $(objpfx)support/true-container $(objpfx)testroot.pristine/bin/true
+	# Copy these DSOs first so we can overwrite them with our own.
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)testroot.pristine/bin/sh \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)support/links-dso-program \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	$(MAKE) install DESTDIR=$(objpfx)testroot.pristine
+	touch $(objpfx)testroot.pristine/install.stamp
+
 tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
diff --git a/Makerules b/Makerules
index a10a0b4d70..64dab056ea 100644
--- a/Makerules
+++ b/Makerules
@@ -1369,7 +1369,7 @@ xcheck: xtests
 # The only difference between MODULE_NAME=testsuite and MODULE_NAME=nonlib is
 # that almost all internal declarations from config.h, libc-symbols.h, and
 # include/*.h are not available to 'testsuite' code, but are to 'nonlib' code.
-all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras))
+all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(tests-container))
 ifneq (,$(all-testsuite))
 cpp-srcs-left = $(all-testsuite)
 lib := testsuite
diff --git a/Rules b/Rules
index 706c8a749d..c3fd2bb0cb 100644
--- a/Rules
+++ b/Rules
@@ -130,12 +130,12 @@ others: $(py-const)
 
 ifeq ($(run-built-tests),no)
 tests: $(addprefix $(objpfx),$(filter-out $(tests-unsupported), \
-                                          $(tests) $(tests-internal)) \
+                                          $(tests) $(tests-internal) $(tests-container)) \
 			     $(test-srcs)) $(tests-special) \
 			     $(tests-printers-programs)
 xtests: tests $(xtests-special)
 else
-tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \
+tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) $(tests-container:%=$(objpfx)%.out) \
        $(tests-special) $(tests-printers-out)
 xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special)
 endif
@@ -149,7 +149,7 @@ tests-expected = $(tests) $(tests-internal) $(tests-printers)
 endif
 tests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
-	  $(sort $(tests-expected) $(tests-special-notdir:.out=)) \
+	  $(sort $(tests-expected) $(tests-special-notdir:.out=) $(tests-container)) \
 	  > $(objpfx)subdir-tests.sum
 xtests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
@@ -158,7 +158,7 @@ xtests:
 
 ifeq ($(build-programs),yes)
 binaries-all-notests = $(others) $(sysdep-others)
-binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs)
+binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) $(tests-container)
 binaries-all = $(binaries-all-notests) $(binaries-all-tests)
 binaries-static-notests = $(others-static)
 binaries-static-tests = $(tests-static) $(xtests-static)
@@ -248,6 +248,16 @@ $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
 	$(make-test-out) > $@; \
 	$(evaluate-test)
 
+
+# Any tests that require an isolated container (chroot) in which to
+# run, should be added to tests-container.
+$(tests-container:%=$(objpfx)%.out): $(objpfx)%.out : $(if $(wildcard $(objpfx)%.files),$(objpfx)%.files,/dev/null) $(objpfx)%
+	$(test-wrapper-env) $(run-program-env) $(run-via-rtld-prefix) \
+	  $(common-objpfx)support/test-container env $(run-program-env) $($*-ENV) \
+	  $(host-test-program-cmd) $($*-ARGS) > $@; \
+	$(evaluate-test)
+
+
 # tests-unsupported lists tests that we will not try to build at all in
 # this configuration.  Note this runs every time because it does not
 # actually create its target.  The dependency on Makefile is meant to
diff --git a/nss/Makefile b/nss/Makefile
index a5cd2aacae..60a28a1519 100644
--- a/nss/Makefile
+++ b/nss/Makefile
@@ -55,11 +55,13 @@ tests-internal		= tst-field
 tests			= test-netdb test-digits-dots tst-nss-getpwent bug17079 \
 			  tst-nss-test1 \
 			  tst-nss-test2 \
-			  tst-nss-test3 \
 			  tst-nss-test4 \
 			  tst-nss-test5
 xtests			= bug-erange
 
+tests-container = \
+			  tst-nss-test3
+
 # Tests which need libdl
 ifeq (yes,$(build-shared))
 tests += tst-nss-files-hosts-erange
diff --git a/nss/tst-nss-test3.c b/nss/tst-nss-test3.c
index d9d708ae7b..4112231778 100644
--- a/nss/tst-nss-test3.c
+++ b/nss/tst-nss-test3.c
@@ -107,7 +107,11 @@ do_test (void)
   int i;
   struct group *g = NULL;
 
-  __nss_configure_lookup ("group", "test1");
+/* Previously we used __nss_configure_lookup to isolate the test
+   from the host environment and to get it to lookup from our new
+   test1 NSS service module, but now this test is run in a different
+   root filesystem via the test-container support and we directly
+   configure the use of the test1 NSS service.  */
 
   setgrent ();
 
diff --git a/nss/tst-nss-test3.root/etc/nsswitch.conf b/nss/tst-nss-test3.root/etc/nsswitch.conf
new file mode 100644
index 0000000000..5e08fe5eea
--- /dev/null
+++ b/nss/tst-nss-test3.root/etc/nsswitch.conf
@@ -0,0 +1 @@
+group	test1
diff --git a/nss/tst-nss-test3.root/files.txt b/nss/tst-nss-test3.root/files.txt
new file mode 100644
index 0000000000..a10beb1e6c
--- /dev/null
+++ b/nss/tst-nss-test3.root/files.txt
@@ -0,0 +1,2 @@
+cp $B/nss/libnss_test1.so $L/libnss_test1.so.2
+cp $B/nss/libnss_test2.so $L/libnss_test2.so.2
diff --git a/support/Makefile b/support/Makefile
index 652d2cdf69..a6b52fb6f8 100644
--- a/support/Makefile
+++ b/support/Makefile
@@ -53,6 +53,7 @@ libsupport-routines = \
   support_format_netent \
   support_isolate_in_subprocess \
   support_openpty \
+  support_paths \
   support_quote_blob \
   support_record_failure \
   support_run_diff \
@@ -84,6 +85,7 @@ libsupport-routines = \
   xmalloc \
   xmemstream \
   xmkdir \
+  xmkdirp \
   xmmap \
   xmprotect \
   xmunmap \
@@ -151,6 +153,42 @@ ifeq ($(build-shared),yes)
 libsupport-inhibit-o += .o
 endif
 
+CFLAGS-support_paths.c = \
+		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
+		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
+		-DINSTDIR_PATH=\"$(prefix)\" \
+		-DLIBDIR_PATH=\"$(libdir)\"
+
+others: \
+	$(objpfx)test-container \
+	$(objpfx)links-dso-program \
+	$(objpfx)shell-container \
+	$(objpfx)echo-container \
+	$(objpfx)true-container
+
+LDLIBS-test-container = $(libsupport)
+
+others += test-container
+others-noinstall += test-container
+
+others += shell-container echo-container true-container
+others-noinstall += shell-container echo-container true-container
+
+$(objpfx)test-container : $(libsupport)
+$(objpfx)shell-container : $(libsupport)
+$(objpfx)echo-container : $(libsupport)
+$(objpfx)true-container : $(libsupport)
+
+# This exists only so we can guess which OS DSOs we need to copy into
+# the testing container.
+ifeq (,$(CXX))
+$(objpfx)links-dso-program : $(objpfx)links-dso-program-c.o
+	$(LINK.o) -o $@ $^
+else
+$(objpfx)links-dso-program : $(objpfx)links-dso-program.o
+	$(LINK.o) -o $@ $^ -lstdc++
+endif
+
 tests = \
   README-testing \
   tst-support-namespace \
diff --git a/support/echo-container.c b/support/echo-container.c
new file mode 100644
index 0000000000..9f8d78f510
--- /dev/null
+++ b/support/echo-container.c
@@ -0,0 +1,34 @@
+/* Minimal /bin/echo for in-container use.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <stdio.h>
+
+int
+main (int argc, const char **argv)
+{
+  int i;
+
+  for (i=1; i<argc; i++)
+    {
+      if (i > 1)
+	putchar (' ');
+      fputs (argv[i], stdout);
+    }
+  putchar ('\n');
+  return 0;
+}
diff --git a/support/links-dso-program-c.c b/support/links-dso-program-c.c
new file mode 100644
index 0000000000..398ec675a0
--- /dev/null
+++ b/support/links-dso-program-c.c
@@ -0,0 +1,5 @@
+int
+main (void)
+{
+  return 0;
+}
diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
new file mode 100644
index 0000000000..398ec675a0
--- /dev/null
+++ b/support/links-dso-program.cc
@@ -0,0 +1,5 @@
+int
+main (void)
+{
+  return 0;
+}
diff --git a/support/shell-container.c b/support/shell-container.c
new file mode 100644
index 0000000000..7354655418
--- /dev/null
+++ b/support/shell-container.c
@@ -0,0 +1,359 @@
+/* Minimal /bin/sh for in-container use.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define __USE_LARGEFILE64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sched.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <sys/sysmacros.h>
+#include <ctype.h>
+#include <utime.h>
+#include <errno.h>
+#include <error.h>
+
+#include <support/support.h>
+
+/* Design considerations
+
+ General rule: optimize for developer time, not run time.
+
+ Specifically:
+
+ * Don't worry about slow algorithms
+ * Don't worry about free'ing memory
+ * Don't implement anything the testsuite doesn't need.
+
+*/
+
+#define DEBUG 0
+#define dprintf if (DEBUG) fprintf
+
+static int
+true_func (char **argv)
+{
+  return 0;
+}
+
+static int
+echo_func (char **argv)
+{
+  int i;
+
+  for (i=0; argv[i]; i++)
+    {
+      if (i > 0)
+	putchar (' ');
+      fputs(argv[i], stdout);
+    }
+  putchar ('\n');
+
+  return 0;
+}
+
+static int
+copy_func (char **argv)
+{
+  char *sname = argv[0];
+  char *dname = argv[1];
+  int sfd, dfd;
+  struct stat st;
+
+  sfd = open (sname, O_RDONLY);
+  if (sfd < 0)
+    {
+      dprintf (stderr, "unable to open %s for reading\n", sname);
+      return 1;
+    }
+
+  if (fstat (sfd, &st) < 0)
+    {
+      dprintf (stderr, "unable to fstat %s\n", sname);
+      return 1;
+    }
+
+  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+  if (dfd < 0)
+    {
+      dprintf (stderr, "unable to open %s for writing\n", dname);
+      return 1;
+    }
+
+  if (copy_file_range (sfd, 0, dfd, 0, st.st_size, 0) != st.st_size)
+    {
+      dprintf (stderr, "cannot copy file %s to %s\n", sname, dname);
+      return 1;
+    }
+
+  close (sfd);
+  close (dfd);
+
+  chmod (dname, st.st_mode & 0777);
+
+  return 0;
+
+}
+
+static struct {
+  const char *name;
+  int (*func)(char **argv);
+} builtin_funcs[] = {
+  { "true", true_func },
+  { "echo", echo_func },
+  { "cp", copy_func },
+  { NULL, NULL }
+};
+
+static void
+run_command_array (char **argv)
+{
+  int i, j;
+  pid_t pid;
+  int status;
+  int (*builtin_func)(char **args);
+
+  if (argv[0] == NULL)
+    return;
+
+  builtin_func = NULL;
+
+  int new_stdin = 0;
+  int new_stdout = 1;
+  int new_stderr = 2;
+
+  dprintf (stderr, "dj: run_command_array\n");
+  for (i=0; argv[i]; i++)
+    dprintf (stderr, "%%DJ%% argv [%d] `%s'\n", i, argv[i]);
+
+  for (j=i=0; argv[i]; i++)
+    {
+      if (strcmp (argv[i], "<") == 0 && argv[i+1])
+	{
+	  new_stdin = open (argv[i+1], O_WRONLY|O_CREAT|O_TRUNC, 0777);
+	  ++ i;
+	  continue;
+	}
+      if (strcmp (argv[i], ">") == 0 && argv[i+1])
+	{
+	  new_stdout = open (argv[i+1], O_WRONLY|O_CREAT|O_TRUNC, 0777);
+	  ++ i;
+	  continue;
+	}
+      if (strcmp (argv[i], ">>") == 0 && argv[i+1])
+	{
+	  new_stdout = open (argv[i+1], O_WRONLY|O_CREAT|O_APPEND, 0777);
+	  ++ i;
+	  continue;
+	}
+      if (strcmp (argv[i], "2>") == 0 && argv[i+1])
+	{
+	  new_stderr = open (argv[i+1], O_WRONLY|O_CREAT|O_TRUNC, 0777);
+	  ++ i;
+	  continue;
+	}
+      argv[j++] = argv[i];
+    }
+  argv[j] = NULL;
+
+
+  for (i=0; builtin_funcs[i].name; i++)
+    if (strcmp (argv[0], builtin_funcs[i].name) == 0)
+       builtin_func = builtin_funcs[i].func;
+
+  dprintf (stderr, "builtin %p argv0 `%s'\n", builtin_func, argv[0]);
+
+  pid = fork();
+  if (pid < 0)
+    {
+      fprintf (stderr, "Can't fork");
+      perror("The error was");
+      exit(1);
+    }
+
+  if (pid == 0)
+    {
+      if (new_stdin != 0)
+	{
+	  dup2 (new_stdin, 0);
+	  close (new_stdin);
+	}
+      if (new_stdout != 1)
+	{
+	  dup2 (new_stdout, 1);
+	  close (new_stdout);
+	}
+      if (new_stderr != 2)
+	{
+	  dup2 (new_stderr, 2);
+	  close (new_stdout);
+	}
+
+      if (builtin_func != NULL)
+	exit (builtin_func(argv+1));
+
+      execvp (argv[0], argv);
+
+      fprintf (stderr, "Can't exec %s", argv[0]);
+      perror("The error was");
+      exit(1);
+    }
+
+  waitpid (pid, &status, 0);
+
+  dprintf (stderr, "exiting run_command_array\n");
+
+  if (WIFEXITED (status))
+    {
+      int rv = WEXITSTATUS (status);
+      if (rv)
+	exit (rv);
+    }
+  else
+    exit (1);
+}
+
+static void
+run_command_string (const char *cmdline, const char **iargs)
+{
+  char *args[100];
+  int ap = 0;
+  const char *start, *end;
+  int nargs;
+
+  for (nargs=0; iargs[nargs] != NULL; nargs++)
+    ;
+
+  dprintf (stderr, "%%DJ%%: run_command_string '%s'\n", cmdline);
+
+  while (ap < 99)
+    {
+      int in_quote = 0;
+
+      while (*cmdline && isspace (*cmdline))
+	cmdline ++;
+      if (*cmdline == 0)
+	break;
+
+      start = cmdline;
+      in_quote = (*cmdline == '\'' || *cmdline == '"') ? *cmdline : 0;
+
+      dprintf (stderr, "in_quote %d\n", in_quote);
+      while (*cmdline
+	     && (!isspace (*cmdline) || in_quote))
+	{
+	  if (*cmdline == in_quote
+	      && cmdline != start)
+	    in_quote = 0;
+	  dprintf (stderr, "[%c]%d ", *cmdline, in_quote);
+	  cmdline ++;
+	}
+      dprintf (stderr, "\n");
+
+      end = cmdline;
+      dprintf (stderr, "start<%s> end<%s>\n", start, end);
+      args[ap] = (char *) malloc (end - start + 1);
+      memcpy (args[ap], start, end - start);
+      args[ap][end - start] = 0;
+
+      dprintf (stderr, "args[%d] = <%s>\n", ap, args[ap]);
+      if (args[ap][0] == '\''
+	  && args[ap][strlen(args[ap])-1] == '\'')
+	{
+	  args[ap][strlen(args[ap])-1] = 0;
+	  args[ap] ++;
+	}
+
+      else if (args[ap][0] == '"'
+	  && args[ap][strlen(args[ap])-1] == '"')
+	{
+	  args[ap][strlen(args[ap])-1] = 0;
+	  args[ap] ++;
+	}
+
+      else if (args[ap][0] == '$'
+	       && isdigit (args[ap][1])
+	       && args[ap][2] == 0)
+	{
+	  int a = args[ap][1] - '1';
+	  if (0 <= a && a < nargs)
+	    args[ap] = strdup (iargs[a]);
+	}
+
+      ap ++;
+
+      if (*cmdline == 0)
+	break;
+    }
+
+  args[ap] = NULL;
+  run_command_array (args);
+}
+
+static void
+run_script (const char *filename, const char **args)
+{
+  char line[1000];
+  dprintf (stderr, "%%DJ%%: run_script '%s'\n", filename);
+  FILE *f = fopen (filename, "r");
+  if (f == NULL)
+    {
+      dprintf (stderr, "can't open %s for reading\n", filename);
+      perror("The error was");
+      exit (1);
+    }
+  while (fgets (line, sizeof(line), f) != NULL)
+    {
+      if (line[0] == '#')
+	{
+	  dprintf (stderr, "comment: %s\n", line);
+	  continue;
+	}
+      run_command_string (line, args);
+    }
+  fclose (f);
+}
+
+int
+main (int argc, const char **argv)
+{
+  int i;
+
+  for (i=0; i<argc; i++)
+    dprintf (stderr, "%%DJ%% sh [%d] `%s'\n", i, argv[i]);
+
+
+  if (strcmp (argv[1], "-c") == 0)
+    run_command_string (argv[2], argv+3);
+  else
+    run_script (argv[1], argv+2);
+
+  dprintf (stderr, "normal exit 0\n");
+  return 0;
+}
diff --git a/support/support.h b/support/support.h
index b61fe0735c..192b9e1424 100644
--- a/support/support.h
+++ b/support/support.h
@@ -76,6 +76,19 @@ char *xasprintf (const char *format, ...)
 char *xstrdup (const char *);
 char *xstrndup (const char *, size_t);
 
+/* Equivalent of "mkdir -p".  */
+void xmkdirp (const char *, int);
+
+/* These point to the TOP of the source/build tree, not your (or
+   support's) subdirectory.  */
+extern const char support_srcdir_root[];
+extern const char support_objdir_root[];
+
+/* Corresponds to the --prefix= passed to configure.  */
+extern const char support_install_prefix[];
+/* Corresponds to the install's lib/ or lib64/ directory.  */
+extern const char support_libdir_prefix[];
+
 __END_DECLS
 
 #endif /* SUPPORT_H */
diff --git a/support/support_paths.c b/support/support_paths.c
new file mode 100644
index 0000000000..a1c22315bd
--- /dev/null
+++ b/support/support_paths.c
@@ -0,0 +1,51 @@
+/* Various paths that might be needed.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/support.h>
+#include <support/check.h>
+
+/* The idea here is to make various makefile-level paths available to
+   support programs, as canonicalized absolute paths.  */
+
+/* These point to the TOP of the source/build tree, not your (or
+   support's) subdirectory.  */
+#ifdef SRCDIR_PATH
+const char support_srcdir_root[] = SRCDIR_PATH;
+#else
+# error please -DSRCDIR_PATH=something in the Makefile
+#endif
+
+#ifdef OBJDIR_PATH
+const char support_objdir_root[] = OBJDIR_PATH;
+#else
+# error please -DOBJDIR_PATH=something in the Makefile
+#endif
+
+#ifdef INSTDIR_PATH
+/* Corresponds to the --prefix= passed to configure.  */
+const char support_install_prefix[] = INSTDIR_PATH;
+#else
+# error please -DINSTDIR_PATH=something in the Makefile
+#endif
+
+#ifdef LIBDIR_PATH
+/* Corresponds to the install's lib/ or lib64/ directory.  */
+const char support_libdir_prefix[] = LIBDIR_PATH;
+#else
+# error please -DLIBDIR_PATH=something in the Makefile
+#endif
diff --git a/support/test-container.c b/support/test-container.c
new file mode 100644
index 0000000000..cb232a1baa
--- /dev/null
+++ b/support/test-container.c
@@ -0,0 +1,940 @@
+/* Run a test case in an isolated namespace.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define __USE_LARGEFILE64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sched.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <sys/sysmacros.h>
+#include <ctype.h>
+#include <utime.h>
+#include <errno.h>
+#include <error.h>
+
+#include <support/support.h>
+#include "check.h"
+#include "test-driver.h"
+
+int verbose = 0;
+
+/* Running a test in a container is tricky.  There are two main
+   categories of things to do:
+
+   1. "Once" actions, like setting up the container and doing an
+      install into it.
+
+   2. "Per-test" actions, like copying in support files and
+      configuring the container.
+
+
+   "Once" actions:
+
+   * mkdir $buildroot/testroot.pristine/
+   * install into it
+   * rsync to $buildroot/testroot.root/
+
+   "Per-test" actions:
+   * maybe rsync to $buildroot/testroot.root/
+   * copy support files and test binary
+   * chroot/unshare
+   * set up any mounts (like /proc)
+
+   Magic files:
+
+   For test $srcdir/foo/mytest.c we look for $srcdir/foo/mytest.root and, if found...
+
+   * mytest.root/ is rsync'd into container
+   * mytest.root/preclean.txt causes fresh rsync (with delete) before test if present
+   * mytest.root/files.txt has a list of files to copy - TBD
+     ($B/ or $S/ for build and source directories)
+       syntax:
+         # comment
+         mv FILE FILE
+	 cp FILE FILE
+	 rm FILE
+	 FILE must start with $B/, $S/, $I/, $L/, or /
+	  (expands to build dir, source dir, install dir, library dir
+	   (in container), or container's root)
+   * mytest.root/postclean.txt causes fresh rsync (with delete) after test if present
+
+   Note that $srcdir/foo/mytest.files may be used instead of a
+   files.txt in the sysroot, if there is no other reason for a sysroot.
+
+   Design goals:
+
+   * independent of other packages which may not be installed (like
+     rsync or Docker, or even "cp")
+
+   * Simple, easy to review code (i.e. prefer simple naive code over
+     complex efficient code)
+
+   TBD:
+
+   * The current implementation is not parallel-make-safe, as one test
+     could be modifying the chroot while another is running against
+     it.
+
+*/
+
+/*--------------------------------------------------*/
+/* Utility Functions */
+
+/* Temporarily concatenate multiple strings into one.  Allows up to 10
+   temporary results; use strdup () if you need them to be
+   permanent.  */
+
+static char *
+concat (const char *str, ...)
+{
+  /* Assume initialized to NULL/zero.  */
+  static char *bufs[10];
+  static size_t buflens[10];
+  static int bufn = 0;
+  int n;
+  size_t len;
+  va_list ap, ap2;
+  char *cp;
+  char *next;
+
+  va_start (ap, str);
+  va_copy (ap2, ap);
+
+  n = bufn;
+  bufn = (bufn + 1) % 10;
+  len = strlen (str);
+
+  while ((next = va_arg (ap, char *)) != NULL)
+    len = len + strlen (next);
+
+  va_end (ap);
+
+  if (bufs[n] == NULL)
+    {
+      bufs[n] = malloc (len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+  else if (buflens[n] < len + 1)
+    {
+      bufs[n] = realloc (bufs[n], len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+
+  strcpy (bufs[n], str);
+  cp = strchr (bufs[n], '\0');
+  while ((next = va_arg (ap2, char *)) != NULL)
+    {
+      strcpy (cp, next);
+      cp = strchr (cp, '\0');
+    }
+  *cp = 0;
+  va_end (ap2);
+
+  return bufs[n];
+}
+
+/* Try to mount SRC onto DEST.  */
+
+static void
+trymount (const char *src, const char *dest)
+{
+  if (mount (src, dest, "", MS_BIND, NULL) < 0)
+    FAIL_UNSUPPORTED ("can't mount %s onto %s\n", src, dest);
+}
+
+/* Special case of above for devices like /dev/zero where we have to
+   mount a device over a device, not a directory over a directory.  */
+
+static void
+devmount (const char *new_root_path, const char *which)
+{
+  int fd;
+  fd = open (concat (new_root_path, "/dev/", which, NULL),
+	     O_CREAT | O_TRUNC | O_RDWR, 0777);
+  close (fd);
+
+  trymount (concat ("/dev/", which, NULL),
+	    concat (new_root_path, "/dev/", which, NULL));
+}
+
+/* Returns true if the string "looks like" an environement variable
+   being set.  */
+
+static int
+is_env_setting (const char *a)
+{
+  int count_name = 0;
+
+  while (*a)
+    {
+      if (isalnum (*a) || *a == '_')
+	++count_name;
+      else if (*a == '=' && count_name > 0)
+	return 1;
+      else
+	return 0;
+      ++a;
+    }
+  return 0;
+}
+
+/* Break the_line into words and store in the_words.  Max nwords,
+   returns actual count.  */
+static int
+tokenize (char *the_line, char **the_words, int nwords)
+{
+  int rv = 0;
+
+  while (nwords > 0)
+    {
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && isspace (*the_line))
+	++the_line;
+
+      /* End of line?  */
+      if (*the_line == 0)
+	return rv;
+
+      /* THE_LINE points to a non-whitespace character, so we have a
+	 word.  */
+      *the_words = the_line;
+      ++the_words;
+      nwords--;
+      ++rv;
+
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && ! isspace (*the_line))
+	++the_line;
+
+      /* We now point at the trailing NUL *or* some whitespace.  */
+      if (*the_line == 0)
+	return rv;
+
+      /* It was whitespace, skip and keep tokenizing.  */
+      *the_line++ = 0;
+    }
+
+  /* We get here if we filled the words buffer.  */
+  return rv;
+}
+
+/*--------------------------------------------------*/
+/* mini-RSYNC implementation.  Optimize later.      */
+
+/* Set this to 1 if you need to debug the rsync function.  */
+#define RTRACE 0
+
+/* A few routines for an "rsync buffer" which stores the paths we're
+   working on.  We continuously grow and shrink the paths in each
+   buffer so there's lot of re-use.  */
+
+/* We rely on "initialized to zero" to set these up.  */
+typedef struct
+{
+  char *buf;
+  size_t len;
+  size_t size;
+} path_buf;
+
+static path_buf spath, dpath;
+
+static void
+r_setup (char *path, path_buf * pb)
+{
+  size_t len = strlen (path);
+  if (pb->buf == NULL || pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      if (pb->buf == NULL)
+	pb->buf = (char *) malloc (sz);
+      else
+	pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	FAIL_UNSUPPORTED ("Out of memory while rsyncing\n");
+
+      pb->size = sz;
+    }
+  strcpy (pb->buf, path);
+  pb->len = len;
+}
+
+static void
+r_append (const char *path, path_buf * pb)
+{
+  size_t len = strlen (path) + pb->len;
+  if (pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	FAIL_UNSUPPORTED ("Out of memory while rsyncing\n");
+
+      pb->size = sz;
+    }
+  strcpy (pb->buf + pb->len, path);
+  pb->len = len;
+}
+
+static int
+file_exists (char *path)
+{
+  struct stat st;
+  if (lstat (path, &st) == 0)
+    return 1;
+  return 0;
+}
+
+static void
+recursive_remove (char *path)
+{
+  pid_t child;
+  int status;
+  /* FIXME: re-implement without external dependencies at some point.
+     Fortunately, this runs outside the container.  */
+
+  child = fork ();
+
+  switch (child) {
+  case -1:
+    FAIL_UNSUPPORTED ("Unable to fork");
+  case 0:
+    /* Child.  */
+    execlp ("rm", "rm", "-rf", path, NULL);
+  default:
+    /* Parent.  */
+    waitpid (child, &status, 0);
+    break;
+  }
+}
+
+/* Used for both rsync and the files.txt "cp" command.  */
+
+static void
+copy_one_file (const char *sname, const char *dname)
+{
+  int sfd, dfd;
+  struct stat st;
+  struct utimbuf times;
+
+  sfd = open (sname, O_RDONLY);
+  if (sfd < 0)
+    FAIL_EXIT1 ("unable to open %s for reading\n", sname);
+
+  if (fstat (sfd, &st) < 0)
+    FAIL_EXIT1 ("unable to fstat %s\n", sname);
+
+  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+  if (dfd < 0)
+    FAIL_EXIT1 ("unable to open %s for writing\n", dname);
+
+  if (copy_file_range (sfd, 0, dfd, 0, st.st_size, 0) != st.st_size)
+    FAIL_EXIT1 ("cannot copy file %s to %s\n", sname, dname);
+
+  close (sfd);
+  close (dfd);
+
+  chmod (dname, st.st_mode & 0777);
+
+  times.actime = st.st_atime;
+  times.modtime = st.st_mtime;
+  utime (dname, &times);
+}
+
+/* We don't check *everything* about the two files to see if a copy is
+   needed, just the minimum to make sure we get the latest copy.  */
+static int
+need_sync (char *ap, char *bp, struct stat *a, struct stat *b)
+{
+  if ((a->st_mode & S_IFMT) != (b->st_mode & S_IFMT))
+    return 1;
+
+  if (S_ISLNK (a->st_mode))
+    {
+      int rv;
+      char *al, *bl;
+
+      if (a->st_size != b->st_size)
+	return 1;
+
+      al = (char *) malloc (a->st_size + 1);
+      bl = (char *) malloc (b->st_size + 1);
+      readlink (ap, al, a->st_size + 1);
+      readlink (bp, bl, b->st_size + 1);
+      al[a->st_size] = 0;
+      bl[b->st_size] = 0;
+      rv = strcmp (al, bl);
+      free (al);
+      free (bl);
+      if (rv == 0)
+	return 0; /* links are same */
+      return 1; /* links differ */
+    }
+
+#if RTRACE
+  if (a->st_size != b->st_size)
+    printf ("SIZE\n");
+  if ((a->st_mode & 0777) != (b->st_mode & 0777))
+    printf ("MODE\n");
+  if (a->st_mtime != b->st_mtime)
+    printf ("TIME\n");
+#endif
+
+  if (a->st_size == b->st_size
+      && ((a->st_mode & 0777) == (b->st_mode & 0777))
+      && a->st_mtime == b->st_mtime)
+    return 0;
+
+  return 1;
+}
+
+static void
+rsync_1 (path_buf * src, path_buf * dest, int and_delete)
+{
+  DIR *dir;
+  struct dirent *de;
+  struct stat s, d;
+
+  r_append ("/", src);
+  r_append ("/", dest);
+#if RTRACE
+  printf ("sync %s to %s %s\n", src->buf, dest->buf,
+	  and_delete ? "and delete" : "");
+#endif
+
+  size_t staillen = src->len;
+
+  size_t dtaillen = dest->len;
+
+  dir = opendir (src->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      if (lstat (src->buf, &s) != 0)
+	FAIL_EXIT1 ("%s obtained by readdir, but stat failed.\n", src->buf);
+
+      // It's OK if this one fails, since we know the file might be missing.
+      lstat (dest->buf, &d);
+
+      if (! need_sync (src->buf, dest->buf, &s, &d))
+	{
+	  if (S_ISDIR (s.st_mode))
+	    rsync_1 (src, dest, and_delete);
+	  continue;
+	}
+
+      if (d.st_mode != ~0)
+	switch (d.st_mode & S_IFMT)
+	  {
+	  case S_IFDIR:
+	    if (!S_ISDIR (s.st_mode))
+	      {
+#if RTRACE
+		printf ("-D %s\n", dest->buf);
+#endif
+		recursive_remove (dest->buf);
+	      }
+	    break;
+
+	  default:
+#if RTRACE
+	    printf ("-F %s\n", dest->buf);
+#endif
+	    unlink (dest->buf);
+	    break;
+	  }
+
+      switch (s.st_mode & S_IFMT)
+	{
+	case S_IFREG:
+#if RTRACE
+	  printf ("+F %s\n", dest->buf);
+#endif
+	  copy_one_file (src->buf, dest->buf);
+	  break;
+
+	case S_IFDIR:
+#if RTRACE
+	  printf ("+D %s\n", dest->buf);
+#endif
+	  mkdir (dest->buf, (s.st_mode & 0777) | 0700);
+	  rsync_1 (src, dest, and_delete);
+	  break;
+
+	case S_IFLNK:
+	  {
+	    char *lp = (char *) malloc (s.st_size + 1);
+#if RTRACE
+	    printf ("+L %s\n", dest->buf);
+#endif
+	    readlink (src->buf, lp, s.st_size + 1);
+	    lp[s.st_size] = 0;
+	    symlink (lp, dest->buf);
+	    free (lp);
+	    break;
+	  }
+
+	default:
+	  break;
+	}
+    }
+
+  closedir (dir);
+  src->len = staillen;
+  src->buf[staillen] = 0;
+  dest->len = dtaillen;
+  dest->buf[dtaillen] = 0;
+
+  if (!and_delete)
+    return;
+
+  /* The rest of this function removes any files/directories in DEST
+     that do not exist in SRC.  This is triggered as part of a
+     preclean or postsclean step.  */
+
+  dir = opendir (dest->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      
+      if (lstat (dest->buf, &d) != 0)
+	FAIL_EXIT1 ("%s obtained by readdir, but stat failed.\n", dest->buf);
+
+      if (s.st_mode == ~0)
+	{
+	  /* dest exists and src doesn't, clean it.  */
+	  switch (d.st_mode & S_IFMT)
+	    {
+	    case S_IFDIR:
+	      if (!S_ISDIR (s.st_mode))
+		{
+#if RTRACE
+		  printf ("-D %s\n", dest->buf);
+#endif
+		  recursive_remove (dest->buf);
+		}
+	      break;
+
+	    default:
+#if RTRACE
+	      printf ("-F %s\n", dest->buf);
+#endif
+	      unlink (dest->buf);
+	      break;
+	    }
+	}
+    }
+
+  closedir (dir);
+}
+
+static void
+rsync (char *src, char *dest, int and_delete)
+{
+  r_setup (src, &spath);
+  r_setup (dest, &dpath);
+
+  rsync_1 (&spath, &dpath, and_delete);
+}
+
+/*--------------------------------------------------*/
+/* Main */
+
+int
+main (int argc, char **argv)
+{
+  pid_t child;
+  char *pristine_root_path;
+  char *new_root_path;
+  char *new_cwd_path;
+  char *new_objdir_path;
+  char *new_srcdir_path;
+  char **new_child_proc;
+  char *command_root;
+  char *command_base;
+  char *so_base;
+  int do_postclean = 0;
+
+  uid_t original_uid;
+  gid_t original_gid;
+  int UMAP;
+  int GMAP;
+  char tmp[100];
+  struct stat st;
+  int lock_fd;
+
+  setbuf (stdout, NULL);
+
+  // The command line we're expecting looks like this:
+  // env <set some vars> ld.so <library path> test-binary
+
+  // We need to peel off any "env" or "ld.so" portion of the command
+  // line, and keep track of which env vars we should preserve and
+  // which we drop.
+
+  if (argc < 2)
+    {
+      fprintf (stderr, "Usage: containerize <program to run> <args...>\n");
+      exit (1);
+    }
+
+  if (strcmp (argv[1], "-v") == 0)
+    {
+      verbose = 1;
+      ++argv;
+      --argc;
+    }
+
+  if (strcmp (argv[1], "env") == 0)
+    {
+      ++argv;
+      --argc;
+      while (is_env_setting (argv[1]))
+	{
+	  // List variables we do NOT want to propogate.
+#if 0
+	  // until we discover why locale/iconv tests don't
+	  // work against an installed tree...
+	  if (memcmp (argv[1], "GCONV_PATH=", 11)
+	      && memcmp (argv[1], "LOCPATH=", 8))
+#endif
+	    {
+	      // Need to keep these.  Note that putenv stores a
+	      // pointer to our argv.
+	      putenv (argv[1]);
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  if (strncmp (argv[1], concat (support_objdir_root, "/elf/ld-linux-", NULL),
+	       strlen (support_objdir_root) + 14) == 0)
+    {
+      ++argv;
+      --argc;
+      while (argv[1][0] == '-')
+	{
+	  if (strcmp (argv[1], "--library-path") == 0)
+	    {
+	      ++argv;
+	      --argc;
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  pristine_root_path = strdup (concat (support_objdir_root, "/testroot.pristine", NULL));
+  new_root_path = strdup (concat (support_objdir_root, "/testroot.root", NULL));
+  new_cwd_path = get_current_dir_name ();
+  new_child_proc = argv + 1;
+
+  lock_fd = open (concat (pristine_root_path, "/lock.fd", NULL),
+		 O_CREAT | O_TRUNC | O_RDWR, 0666);
+  if (lock_fd < 0)
+    FAIL_UNSUPPORTED ("Cannot create testroot lock.\n");
+
+  while (flock (lock_fd, LOCK_EX) != 0)
+    {
+      if (errno != EINTR)
+	FAIL_UNSUPPORTED ("Cannot lock testroot.\n");
+    }
+
+  xmkdirp (new_root_path, 0755);
+
+  /* We look for extra setup info in a subdir in the same spot as the
+     test, with the same name but a ".root" extension.  This is that
+     directory.  We try to look in the source tree if the path we're
+     given refers to the build tree, but we rely on the path to be
+     absolute.  This is what the glibc makefiles do.  */
+  command_root = concat (argv[1], ".root", NULL);
+  if (strncmp (command_root, support_objdir_root, strlen (support_objdir_root)) == 0
+      && command_root[strlen (support_objdir_root)] == '/')
+    command_root = concat (support_srcdir_root, argv[1] + strlen (support_objdir_root), ".root", NULL);
+  command_root = strdup (command_root);
+
+  /* This cuts off the ".root" we appended above.  */
+  command_base = strdup (command_root);
+  command_base[strlen (command_base) - 5] = 0;
+
+  /* Shared object base directory.  */
+  so_base = strdup (argv[1]);
+  if (strrchr (so_base, '/') != NULL)
+    strrchr (so_base, '/')[1] = 0;
+
+  if (file_exists (concat (command_root, "/postclean.txt", NULL)))
+    do_postclean = 1;
+
+  rsync (pristine_root_path, new_root_path,
+	 1 || file_exists (concat (command_root, "/preclean.txt", NULL)));
+
+  if (stat (command_root, &st) >= 0
+      && S_ISDIR (st.st_mode))
+    rsync (command_root, new_root_path, 0);
+
+  new_objdir_path = strdup (concat (new_root_path, support_objdir_root, NULL));
+  new_srcdir_path = strdup (concat (new_root_path, support_srcdir_root, NULL));
+
+  /* new_cwd_path starts with '/' so no "/" needed between the two.  */
+  xmkdirp (concat (new_root_path, new_cwd_path, NULL), 0755);
+  xmkdirp (new_srcdir_path, 0755);
+  xmkdirp (new_objdir_path, 0755);
+
+  original_uid = getuid ();
+  original_gid = getgid ();
+
+  /* Handle the cp/mv/rm "script" here.  */
+  {
+    char *the_line = NULL;
+    size_t line_len = 0;
+    char *fname = concat (command_root, "/files.txt", NULL);
+    char *the_words[3];
+    FILE *f = fopen (fname, "r");
+
+    if (verbose && f)
+      fprintf (stderr, "reading %s\n", fname);
+
+    if (f == NULL)
+      {
+	/* Try foo.files instead of foo.root/files.txt, as a shortcut.  */
+	fname = concat (command_base, ".files", NULL);
+	f = fopen (fname, "r");
+	if (verbose && f)
+	  fprintf (stderr, "reading %s\n", fname);
+      }
+
+#if 0
+    /* I don't want to add this until we know we need it, but here it
+       is...  */
+    if (f == NULL)
+      {
+	/* Look for a Makefile-generated one also.  */
+	fname = concat (argv[1], ".files", NULL);
+	f = fopen (fname, "r");
+      }
+#endif
+
+    /* This is where we "interpret" the mini-script which is <test>.files.  */
+    if (f != NULL)
+      {
+	while (getline (&the_line, &line_len, f) > 0)
+	  {
+	    int nt = tokenize (the_line, the_words, 3);
+	    int i;
+
+	    for (i = 1; i < nt; ++i)
+	      {
+		if (memcmp (the_words[i], "$B/", 3) == 0)
+		  the_words[i] = concat (support_objdir_root, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$S/", 3) == 0)
+		  the_words[i] = concat (support_srcdir_root, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$I/", 3) == 0)
+		  the_words[i] = concat (new_root_path, support_install_prefix, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$L/", 3) == 0)
+		  the_words[i] = concat (new_root_path, support_libdir_prefix, the_words[i] + 2, NULL);
+		else if (the_words[i][0] == '/')
+		  the_words[i] = concat (new_root_path, the_words[i], NULL);
+	      }
+
+	    if (nt == 3 && the_words[2][strlen (the_words[2]) - 1] == '/')
+	      {
+		char *r = strrchr (the_words[1], '/');
+		if (r)
+		  the_words[2] = concat (the_words[2], r + 1, NULL);
+		else
+		  the_words[2] = concat (the_words[2], the_words[1], NULL);
+	      }
+
+	    if (nt == 2 && strcmp (the_words[0], "so") == 0)
+	      {
+		the_words[2] = concat (new_root_path, support_libdir_prefix, "/", the_words[1], NULL);
+		the_words[1] = concat (so_base, the_words[1], NULL);
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "cp") == 0)
+	      {
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "mv") == 0)
+	      {
+		rename (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "chmod") == 0)
+	      {
+		long int m;
+		m = strtol (the_words[1], NULL, 0);
+		chmod (the_words[2], m);
+	      }
+	    else if (nt == 2 && strcmp (the_words[0], "rm") == 0)
+	      {
+		unlink (the_words[1]);
+	      }
+	    else if (nt > 0 && the_words[0][0] != '#')
+	      {
+		printf ("\033[31minvalid [%s]\033[0m\n", the_words[0]);
+	      }
+	  }
+	fclose (f);
+      }
+  }
+
+  // The unshare here gives us our own spaces and capabilities.
+  if (unshare (CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS) < 0)
+    FAIL_UNSUPPORTED ("unable to unshare user/fs, ");
+
+  /* Some systems, by default, all mounts leak out of the namespace.  */
+  if (mount ("none", "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0)
+    FAIL_UNSUPPORTED ("could not create a private mount namespace\n");
+
+  trymount (support_srcdir_root, new_srcdir_path);
+  trymount (support_objdir_root, new_objdir_path);
+
+  xmkdirp (concat (new_root_path, "/dev", NULL), 0755);
+  devmount (new_root_path, "null");
+  devmount (new_root_path, "zero");
+  devmount (new_root_path, "urandom");
+
+  // We're done with the "old" root, switch to the new one.
+  if (chroot (new_root_path) < 0)
+    FAIL_UNSUPPORTED ("Can't chroot to %s - ", new_root_path);
+
+  if (chdir (new_cwd_path) < 0)
+    FAIL_UNSUPPORTED ("Can't cd to new %s - ", new_cwd_path);
+
+  /* To complete the containerization, we need to fork () at least
+     once.  We can't exec, nor can we somehow link the new child to
+     our parent.  So we run the child and propogate it's exit status
+     up.  */
+  child = fork ();
+  if (child < 0)
+    FAIL_UNSUPPORTED ("Unable to fork");
+  else if (child > 0)
+    {
+      /* Parent.  */
+      int status;
+      waitpid (child, &status, 0);
+
+      /* There's a bit of magic here, since the buildroot is mounted
+	 in our space, the paths are still valid, and since the mounts
+	 aren't recursive, it sees *only* the built root, not anything
+	 we would normally se if we rsync'd to "/" like mounted /dev
+	 files.  */
+      if (do_postclean)
+	  rsync (pristine_root_path, new_root_path, 1);
+
+      if (WIFEXITED (status))
+	exit (WEXITSTATUS (status));
+
+      if (WIFSIGNALED (status))
+	{
+	  printf ("%%SIGNALLED%%\n");
+	  exit (77);
+	}
+
+      printf ("%%EXITERROR%%\n");
+      exit (78);
+    }
+
+  /* The rest is the child process, which is now PID 1 and "in" the
+     new root.  */
+
+  mkdir ("/tmp", 0755);
+
+  // Now that we're pid 1 (effectively "root") we can mount /proc
+  mkdir ("/proc", 0777);
+  if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
+    FAIL_UNSUPPORTED ("Unable to mount /proc: ");
+
+  // We map our original UID to the same UID in the container so we
+  // can own our own files normally
+  UMAP = open ("/proc/self/uid_map", O_WRONLY);
+  if (UMAP < 0)
+    FAIL_UNSUPPORTED ("can't write to /proc/self/uid_map\n");
+
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_uid, (long long) original_uid);
+  write (UMAP, tmp, strlen (tmp));
+  close (UMAP);
+
+  // We must disable setgroups () before we can map our groups, else we
+  // get EPERM.
+  GMAP = open ("/proc/self/setgroups", O_WRONLY);
+  if (GMAP >= 0)
+    {
+      /* We support kernels old enough to not have this.  */
+      write (GMAP, "deny\n", 5);
+      close (GMAP);
+    }
+
+  // We map our original GID to the same GID in the container so we
+  // can own our own files normally
+  GMAP = open ("/proc/self/gid_map", O_WRONLY);
+  if (GMAP < 0)
+    FAIL_UNSUPPORTED ("can't write to /proc/self/gid_map\n");
+
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_gid, (long long) original_gid);
+  write (GMAP, tmp, strlen (tmp));
+  close (GMAP);
+
+  // Now run the child
+  execvp (new_child_proc[0], new_child_proc);
+
+  // Or don't run the child?
+  FAIL_EXIT1 ("Unable to exec %s\n", new_child_proc[0]);
+
+  // Because gcc won't know error () never returns...
+  exit (EXIT_UNSUPPORTED);
+}
diff --git a/support/true-container.c b/support/true-container.c
new file mode 100644
index 0000000000..157d9c999a
--- /dev/null
+++ b/support/true-container.c
@@ -0,0 +1,23 @@
+/* Minimal /bin/true for in-container use.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+int
+main(void)
+{
+  return 0;
+}
diff --git a/support/xmkdirp.c b/support/xmkdirp.c
new file mode 100644
index 0000000000..a15fddea48
--- /dev/null
+++ b/support/xmkdirp.c
@@ -0,0 +1,67 @@
+/* Error-checking replacement for "mkdir -p".
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/support.h>
+#include <support/check.h>
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+/* Equivalent of "mkdir -p".  Any failures cause FAIL_EXIT1 so no
+   return code is needed.  */
+
+void
+xmkdirp (const char *path, int mode)
+{
+  struct stat s;
+  const char *slash_p;
+  int rv;
+
+  if (path[0] == 0)
+    return;
+
+  if (stat (path, &s) == 0)
+    {
+      if (S_ISDIR (s.st_mode))
+	return;
+      errno = EEXIST;
+      FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);
+    }
+
+  slash_p = strrchr (path, '/');
+  if (slash_p != NULL)
+    {
+      while (slash_p > path && slash_p[-1] == '/')
+	--slash_p;
+      if (slash_p > path)
+	{
+	  char *parent = xstrndup (path, slash_p - path);
+	  xmkdirp (parent, mode);
+	  free (parent);
+	}
+    }
+
+  rv = mkdir (path, mode);
+  if (rv != 0)
+    FAIL_EXIT1 ("mkdir_p (\"%s\", 0%o): %m", path, mode);
+
+  return;
+}

Re: RFC V5 test-in-container

[Carlos O'Donell]

On 07/06/2018 01:20 AM, DJ Delorie wrote:
> 
> Joseph Myers <joseph@codesourcery.com> writes:
>>> I can write a /bin/sh substitute
>>
>> I think that's the right thing to do.
> 
> I tested by running all of posix/ and libio/ tests inside the container,
> decided to also add /bin/echo and /bin/true since they were (1) used,
> and (2) trivial.  Let the slippery slope begin :-)
> 
> Changes since V4:
> 
> 	add shell-container, echo-container, true-container helper programs
> 
> 	add $($*-ENV) to test-container rules

I don't think this is an "RFC" anymore :-)

I think we should consider this a full [PATCH].

You've done way more work than anyone ever does in an RFC!

Thank you for this.

-- 
Cheers,
Carlos.