[PATCH 0/3] Document ld.so --list-diagnostics, add syntax tests

[PATCH 0/3] Document ld.so --list-diagnostics, add syntax tests

[Florian Weimer]

Tested on x86_64-linux-gnu.

Florian Weimer (3):
  Linux: Avoid conflicting types in ld.so --list-diagnostics
  manual: Document ld.so --list-diagnostics output
  elf: Check that --list-diagnostics output has the expected syntax

 INSTALL                                       |   5 +
 elf/Makefile                                  |   9 +
 elf/tst-rtld-list-diagnostics.py              | 311 ++++++++++++++++++
 manual/dynlink.texi                           | 279 ++++++++++++++++
 manual/install.texi                           |   6 +
 .../unix/sysv/linux/dl-diagnostics-kernel.c   |  13 +-
 6 files changed, 618 insertions(+), 5 deletions(-)
 create mode 100644 elf/tst-rtld-list-diagnostics.py


base-commit: d97a12704bca5c2c326d856a69cd847c89784ae9
-- 
2.41.0

[PATCH 1/3] Linux: Avoid conflicting types in ld.so --list-diagnostics

[Florian Weimer]

The path auxv[*].a_val could either be an integer or a string,
depending on the a_type value.  Use a separate field, a_val_string, to
simplify mechanical parsing of the --list-diagnostics output.
---
 sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c b/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c
index e0cfa63da6..d522e2797e 100644
--- a/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c
+++ b/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c
@@ -30,16 +30,19 @@ print_auxv (void)
   for (ElfW(auxv_t) *av = GLRO(dl_auxv); av->a_type != AT_NULL; ++av)
     {
       _dl_printf ("auxv[0x%x].a_type=0x%lx\n"
-                  "auxv[0x%x].a_val=",
+                  "auxv[0x%x].a_val",
                   index, (unsigned long int) av->a_type, index);
       if (av->a_type == AT_EXECFN
           || av->a_type == AT_PLATFORM
           || av->a_type == AT_BASE_PLATFORM)
-        /* The address of the strings is not useful at all, so print
-           the strings themselves.  */
-        _dl_diagnostics_print_string ((const char *) av->a_un.a_val);
+        {
+          /* The address of the strings is not useful at all, so print
+             the strings themselves.  */
+          _dl_printf ("_string=");
+          _dl_diagnostics_print_string ((const char *) av->a_un.a_val);
+        }
       else
-        _dl_printf ("0x%lx", (unsigned long int) av->a_un.a_val);
+        _dl_printf ("=0x%lx", (unsigned long int) av->a_un.a_val);
       _dl_printf ("\n");
       ++index;
     }
-- 
2.41.0

Re: [PATCH 1/3] Linux: Avoid conflicting types in ld.so --list-diagnostics

[Adhemerval Zanella Netto]

On 04/08/23 15:16, Florian Weimer via Libc-alpha wrote:
> The path auxv[*].a_val could either be an integer or a string,
> depending on the a_type value.  Use a separate field, a_val_string, to
> simplify mechanical parsing of the --list-diagnostics output.

LGTM, thanks.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

> ---
>  sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c | 13 ++++++++-----
>  1 file changed, 8 insertions(+), 5 deletions(-)
> 
> diff --git a/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c b/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c
> index e0cfa63da6..d522e2797e 100644
> --- a/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c
> +++ b/sysdeps/unix/sysv/linux/dl-diagnostics-kernel.c
> @@ -30,16 +30,19 @@ print_auxv (void)
>    for (ElfW(auxv_t) *av = GLRO(dl_auxv); av->a_type != AT_NULL; ++av)
>      {
>        _dl_printf ("auxv[0x%x].a_type=0x%lx\n"
> -                  "auxv[0x%x].a_val=",
> +                  "auxv[0x%x].a_val",
>                    index, (unsigned long int) av->a_type, index);
>        if (av->a_type == AT_EXECFN
>            || av->a_type == AT_PLATFORM
>            || av->a_type == AT_BASE_PLATFORM)
> -        /* The address of the strings is not useful at all, so print
> -           the strings themselves.  */
> -        _dl_diagnostics_print_string ((const char *) av->a_un.a_val);
> +        {
> +          /* The address of the strings is not useful at all, so print
> +             the strings themselves.  */
> +          _dl_printf ("_string=");
> +          _dl_diagnostics_print_string ((const char *) av->a_un.a_val);
> +        }
>        else
> -        _dl_printf ("0x%lx", (unsigned long int) av->a_un.a_val);
> +        _dl_printf ("=0x%lx", (unsigned long int) av->a_un.a_val);
>        _dl_printf ("\n");
>        ++index;
>      }

[PATCH 2/3] manual: Document ld.so --list-diagnostics output

[Florian Weimer]

---
 manual/dynlink.texi | 279 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 279 insertions(+)

diff --git a/manual/dynlink.texi b/manual/dynlink.texi
index 45bf5a5b55..fc2cd2f0a4 100644
--- a/manual/dynlink.texi
+++ b/manual/dynlink.texi
@@ -13,9 +13,288 @@ as plugins) later at run time.
 Dynamic linkers are sometimes called @dfn{dynamic loaders}.
 
 @menu
+* Dynamic Linker Invocation::   Explicit invocation of the dynamic linker.
 * Dynamic Linker Introspection::    Interfaces for querying mapping information.
 @end menu
 
+@node Dynamic Linker Invocation
+
+@cindex program interpreter
+When a dynamically linked program starts, the operating system
+automatically loads the dynamic linker along with the program.
+@Theglibc{} also supports invoking the dynamic linker explicitly to
+launch a program.  This command uses the implied dynamic linker
+(also sometimes called the @dfn{program interpreter}):
+
+@smallexample
+sh -c 'echo "Hello, world!"'
+@end smallexample
+
+This command specifies the dynamic linker explicitly:
+
+@smallexample
+ld.so /bin/sh -c 'echo "Hello, world!"'
+@end smallexample
+
+Note that @command{ld.so} does not search the @env{PATH} environment
+variable, so the full file name of the executable needs to be specified.
+
+The @command{ld.so} program supports various options.  Options start
+@samp{--} and need to come before the program that is being launched.
+Some of the supported options are listed below.
+
+@table @code
+@item --list-diagnostics
+Print system diagnostic information in a machine-readable format.
+@xref{Dynamic Linker Diagnostics}.
+@end table
+
+@menu
+* Dynamic Linker Diagnostics::   Obtaining system diagnostic information.
+@end menu
+
+@node Dynamic Linker Diagnostics
+@section Dynamic Linker Diagnostics
+@cindex diagnostics (dynamic linker)
+
+The @samp{ld.so --list-diagnostics} produces machine-readable
+diagnostics output.  This output contains system data that affects
+behavior of @theglibc{}, and potentially application behavior as well.
+
+The exact set of diagnostic items can change between releases of
+@theglibc{}.  The output format itself is not expected to change
+radically.
+
+The following table shows some example lines that can be written by the
+diagnostics command.
+
+@table @code
+@item dl_pagesize=0x1000
+The system page size is 4096 bytes.
+
+@item env[0x14]="LANG=en_US.UTF-8"
+This item indicates that the 21st environment variable at process
+startup contains a setting for @code{LANG}.
+
+@item env_filtered[0x22]="DISPLAY"
+The 35th environment variable is @code{DISPLAY}.  Its value is not
+included in the output for privacy reasons because it is not recognized
+as harmless by the diagnostics code.
+
+@item path.prefix="/usr"
+This means that @theglibc{} was configured with @code{--prefix=/usr}.
+
+@item path.system_dirs[0x0]="/lib64/"
+@itemx path.system_dirs[0x1]="/usr/lib64/"
+The built-in dynamic linker search path contains two directories,
+@code{/lib64} and @code{/usr/lib64}.
+@end table
+
+@subsection Dynamic Linker Diagnostics Output Format
+
+As seen above, diagnostic lines assign values (integers or strings) to a
+sequences of labeled subscripts, separated by @samp{.}.  Some subscripts
+have integer indices associated with them.  The subscript indices are
+not necessarily contiguous or small, so an associative array should be
+used to store them.  Currently, all integers fit into the 64-bit
+unsigned integer range.  Every access path to a value has a fixed type
+(string or integer) independently of subscript index values.  Likewise,
+whether a subscript is indexed does not depend on previous indices (but
+may depend on previous subscript labels).
+
+A syntax description in ABNF (RFC 5234) follows.  Note that
+@code{%x30-39} denotes the range of decimal digits.  Diagnostic output
+lines are expected to match the @code{line} production.
+
+@c ABNF-START
+@smallexample
+HEXDIG = %x30-39 / %x61-6f ; lowercase a-f only
+ALPHA = %x41-5a / %x61-7a / %x7f ; letters and underscore
+ALPHA-NUMERIC = ALPHA / %x30-39 / "_"
+DQUOTE = %x22 ; "
+
+; Numbers are always hexadecimal and use a 0x prefix.
+hex-value-prefix = %x30 %x78
+hex-value = hex-value-prefix 1*HEXDIG
+
+; Strings use octal escape sequences and \\, \".
+string-char = %x20-21 / %x23-5c / %x5d-7e ; printable but not "\
+string-quoted-octal = %x30-33 2*2%x30-37
+string-quoted = "\" ("\" / DQUOTE / string-quoted-octal)
+string-value = DQUOTE *(string-char / string-quoted) DQUOTE
+
+value = hex-value / string-value
+
+label = ALPHA *ALPHA-NUMERIC
+index = "[" hex-value "]"
+subscript = label [index]
+
+line = subscript *("." subscript) "=" value
+@end smallexample
+
+Output lines can be parsed with the following Python function.  It
+assumes lines formatted according to the ABNF @code{line} production
+above.
+
+@c PYTHON-START
+@smallexample
+def parse_line(line):
+    """Parse a line of --list-diagnostics output.
+
+    This function returns a pair (SUBSCRIPTS, VALUE).  VALUE is either
+    a byte string or an integer.  SUBSCRIPT is a tuple of (LABEL,
+    INDEX) pairs, where LABEL is a field identifier (a string), and
+    INDEX is an integer or None, to indicate that this field is not
+    indexed.
+
+    """
+
+    # Extract the list of subscripts before the value.
+    idx = 0
+    subscripts = []
+    while line[idx] != '=':
+        start_idx = idx
+
+        # Extract the label.
+        while line[idx] not in '[.=':
+            idx += 1
+        label = line[start_idx:idx]
+
+        if line[idx] == '[':
+            # Subscript with a 0x index.
+            assert label
+            close_bracket = line.index(']', idx)
+            index = line[idx + 1:close_bracket]
+            assert index.startswith('0x')
+            index = int(index, 0)
+            subscripts.append((label, index))
+            idx = close_bracket + 1
+        else: # '.' or '='.
+            if label:
+                subscripts.append((label, None))
+            if line[idx] == '.':
+                idx += 1
+
+    # The value is either a string or a 0x number.
+    value = line[idx + 1:]
+    if value[0] == '"':
+        # Decode the escaped string into a byte string.
+        assert value[-1] == '"'
+        idx = 1
+        result = []
+        while True:
+            ch = value[idx]
+            if ch == '\\':
+                if value[idx + 1] in '"\\':
+                    result.append(ord(value[idx + 1]))
+                    idx += 2
+                else:
+                    result.append(int(value[idx + 1:idx + 4], 8))
+                    idx += 4
+            elif ch == '"':
+                assert idx == len(value) - 1
+                break
+            else:
+                result.append(ord(value[idx]))
+                idx += 1
+        value = bytes(result)
+    else:
+        # Convert the value into an integer.
+        assert value.startswith('0x')
+        value = int(value, 0)
+    return (tuple(subscripts), value)
+@end smallexample
+
+@subsection Dynamic Linker Diagnostics Values
+
+As mentioned above, the set of diagnostics may change between
+@theglibc{} releases.  Nevertheless, the following table documents a few
+common diagnostic items.
+
+@table @code
+@item dl_dst_lib=@var{string}
+The @code{$LIB} dynamic string token expands to @var{string}.
+
+@item dl_hwcap=@var{integer}
+@itemx dl_hwcap2=@var{integer}
+@cindex HWCAP (diagnostics)
+The HWCAP and HWCAP2 values, as returned for @code{getauxval}, and as
+used in other places depending on the architecture.
+
+@item dl_pagesize=@var{integer}
+@cindex page size (diagnostics)
+The system page size is @var{integer} bytes.
+
+@item dl_platform=@var{string}
+The @code{$PLATFORM} dynamic string token expands to @var{string}.
+
+@item dso.libc=@var{string}
+This is the soname of the shared @code{libc} object that is part of
+@theglibc{}.  On most architectures, this is @code{libc.so.6}.
+
+@item env[@var{index}]=@var{string}
+@itemx env_filtered[@var{index}]=@var{string}
+An environment variable from the process environment.  The integer
+@var{index} is the array index in the environment array.  Variables
+under @code{env} include the variable value after the @samp{=} (assuming
+that it was present), variables under @code{env_filtered} do not.
+
+@item path.prefix=@var{string}
+This indicates that @theglibc{} was configured using
+@samp{--prefix=@var{string}}.
+
+@item path.sysconfdir=@var{string}
+@Theglibc{} was configured (perhaps implicitly) with
+@samp{--sysconfdir=@var{string}} (typically @code{/etc}).
+
+@item path.system_dirs[@var{index}]=@var{string}
+These items list the elements of the built-in array that describes the
+default library search path.  The value @var{string} a directory file
+name with a trailing @samp{/}.
+
+@item path.rtld=@var{string}
+This string indicates the application binary interface (ABI) file name
+of the run-time dynamic linker.
+
+@item version.release="stable"
+@itemx version.release="development"
+The value @code{"stable"} indicates that this build of @theglibc{} is
+from a release branch.  Releases labeled as @code{"development"} are
+unreleased development versions.
+
+@item version.version="@var{major}.@var{minor}"
+@itemx version.version="@var{major}.@var{minor}.9000"
+@cindex version (diagnostics)
+@Theglibc{} version.  Development releases end in @samp{.9000}.
+
+@item auxv[@var{index}].a_type=@var{type}
+@itemx auxv[@var{index}].a_val=@var{integer}
+@itemx auxv[@var{index}].a_val_string=@var{string}
+@cindex auxiliary vector (diagnostics)
+An entry in the auxiliary vector (specific to Linux).  The values
+@var{type} (an integer) and @var{integer} correspond to the members of
+@code{struct auxv}.  If the value is a string, @code{a_val_string} is
+used instead of @code{a_val}, so that values have consistent types.
+
+The @code{AT_HWCAP} and @code{AT_HWCAP2} values in this output do not
+reflect adjustment by @theglibc{}.
+
+@item uname.sysname=@var{string}
+@itemx uname.nodename=@var{string}
+@itemx uname.release=@var{string}
+@itemx uname.version=@var{string}
+@itemx uname.machine=@var{string}
+@itemx uname.domain=@var{string}
+These Linux-specific items show the values of @code{struct utsname}, as
+reported by the @code{uname} function.  @xref{Platform Type}.
+
+@item x86.cpu_features.@dots{}
+@cindex CPUID (diagnostics)
+These items are specific to the i386 and x86-64 architectures.  They
+reflect supported CPU feature and information on cache geometry, mostly
+collected using the @code{CPUID} instruction.
+@end table
+
 @node Dynamic Linker Introspection
 @section Dynamic Linker Introspection
 
-- 
2.41.0

Re: [PATCH 2/3] manual: Document ld.so --list-diagnostics output

[Arsen Arsenović]

[-- Attachment #1: Type: text/plain, Size: 12173 bytes --]

Hi Florian,

Florian Weimer <fweimer@redhat.com> writes:

> ---
>  manual/dynlink.texi | 279 ++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 279 insertions(+)
>
> diff --git a/manual/dynlink.texi b/manual/dynlink.texi
> index 45bf5a5b55..fc2cd2f0a4 100644
> --- a/manual/dynlink.texi
> +++ b/manual/dynlink.texi
> @@ -13,9 +13,288 @@ as plugins) later at run time.
>  Dynamic linkers are sometimes called @dfn{dynamic loaders}.
>  
>  @menu
> +* Dynamic Linker Invocation::   Explicit invocation of the dynamic linker.
>  * Dynamic Linker Introspection::    Interfaces for querying mapping information.
>  @end menu
>  
> +@node Dynamic Linker Invocation
> +
> +@cindex program interpreter
> +When a dynamically linked program starts, the operating system
> +automatically loads the dynamic linker along with the program.
> +@Theglibc{} also supports invoking the dynamic linker explicitly to
> +launch a program.  This command uses the implied dynamic linker
> +(also sometimes called the @dfn{program interpreter}):
> +
> +@smallexample
> +sh -c 'echo "Hello, world!"'
> +@end smallexample
> +
> +This command specifies the dynamic linker explicitly:
> +
> +@smallexample
> +ld.so /bin/sh -c 'echo "Hello, world!"'
> +@end smallexample
> +
> +Note that @command{ld.so} does not search the @env{PATH} environment
> +variable, so the full file name of the executable needs to be specified.
> +
> +The @command{ld.so} program supports various options.  Options start
> +@samp{--} and need to come before the program that is being launched.
> +Some of the supported options are listed below.
> +
> +@table @code
> +@item --list-diagnostics
> +Print system diagnostic information in a machine-readable format.
> +@xref{Dynamic Linker Diagnostics}.
> +@end table
> +
> +@menu
> +* Dynamic Linker Diagnostics::   Obtaining system diagnostic information.
> +@end menu
> +
> +@node Dynamic Linker Diagnostics
> +@section Dynamic Linker Diagnostics
> +@cindex diagnostics (dynamic linker)
> +
> +The @samp{ld.so --list-diagnostics} produces machine-readable
> +diagnostics output.  This output contains system data that affects
> +behavior of @theglibc{}, and potentially application behavior as well.
> +
> +The exact set of diagnostic items can change between releases of
> +@theglibc{}.  The output format itself is not expected to change
> +radically.
> +
> +The following table shows some example lines that can be written by the
> +diagnostics command.
> +
> +@table @code
> +@item dl_pagesize=0x1000
> +The system page size is 4096 bytes.
> +
> +@item env[0x14]="LANG=en_US.UTF-8"
> +This item indicates that the 21st environment variable at process
> +startup contains a setting for @code{LANG}.
> +
> +@item env_filtered[0x22]="DISPLAY"
> +The 35th environment variable is @code{DISPLAY}.  Its value is not
> +included in the output for privacy reasons because it is not recognized
> +as harmless by the diagnostics code.
> +
> +@item path.prefix="/usr"
> +This means that @theglibc{} was configured with @code{--prefix=/usr}.
> +
> +@item path.system_dirs[0x0]="/lib64/"
> +@itemx path.system_dirs[0x1]="/usr/lib64/"
> +The built-in dynamic linker search path contains two directories,
> +@code{/lib64} and @code{/usr/lib64}.
> +@end table
> +
> +@subsection Dynamic Linker Diagnostics Output Format
> +
> +As seen above, diagnostic lines assign values (integers or strings) to a
> +sequences of labeled subscripts, separated by @samp{.}.  Some subscripts
> +have integer indices associated with them.  The subscript indices are
> +not necessarily contiguous or small, so an associative array should be
> +used to store them.  Currently, all integers fit into the 64-bit
> +unsigned integer range.  Every access path to a value has a fixed type
> +(string or integer) independently of subscript index values.  Likewise,
> +whether a subscript is indexed does not depend on previous indices (but
> +may depend on previous subscript labels).
> +
> +A syntax description in ABNF (RFC 5234) follows.  Note that
> +@code{%x30-39} denotes the range of decimal digits.  Diagnostic output
> +lines are expected to match the @code{line} production.
> +
> +@c ABNF-START
> +@smallexample
> +HEXDIG = %x30-39 / %x61-6f ; lowercase a-f only
> +ALPHA = %x41-5a / %x61-7a / %x7f ; letters and underscore
> +ALPHA-NUMERIC = ALPHA / %x30-39 / "_"
> +DQUOTE = %x22 ; "
> +
> +; Numbers are always hexadecimal and use a 0x prefix.
> +hex-value-prefix = %x30 %x78
> +hex-value = hex-value-prefix 1*HEXDIG
> +
> +; Strings use octal escape sequences and \\, \".
> +string-char = %x20-21 / %x23-5c / %x5d-7e ; printable but not "\
> +string-quoted-octal = %x30-33 2*2%x30-37
> +string-quoted = "\" ("\" / DQUOTE / string-quoted-octal)
> +string-value = DQUOTE *(string-char / string-quoted) DQUOTE
> +
> +value = hex-value / string-value
> +
> +label = ALPHA *ALPHA-NUMERIC
> +index = "[" hex-value "]"
> +subscript = label [index]
> +
> +line = subscript *("." subscript) "=" value
> +@end smallexample
> +
> +Output lines can be parsed with the following Python function.  It
> +assumes lines formatted according to the ABNF @code{line} production
> +above.
> +
> +@c PYTHON-START
> +@smallexample
> +def parse_line(line):
> +    """Parse a line of --list-diagnostics output.
> +
> +    This function returns a pair (SUBSCRIPTS, VALUE).  VALUE is either
> +    a byte string or an integer.  SUBSCRIPT is a tuple of (LABEL,
> +    INDEX) pairs, where LABEL is a field identifier (a string), and
> +    INDEX is an integer or None, to indicate that this field is not
> +    indexed.
> +
> +    """
> +
> +    # Extract the list of subscripts before the value.
> +    idx = 0
> +    subscripts = []
> +    while line[idx] != '=':
> +        start_idx = idx
> +
> +        # Extract the label.
> +        while line[idx] not in '[.=':
> +            idx += 1
> +        label = line[start_idx:idx]
> +
> +        if line[idx] == '[':
> +            # Subscript with a 0x index.
> +            assert label
> +            close_bracket = line.index(']', idx)
> +            index = line[idx + 1:close_bracket]
> +            assert index.startswith('0x')
> +            index = int(index, 0)
> +            subscripts.append((label, index))
> +            idx = close_bracket + 1
> +        else: # '.' or '='.
> +            if label:
> +                subscripts.append((label, None))
> +            if line[idx] == '.':
> +                idx += 1
> +
> +    # The value is either a string or a 0x number.
> +    value = line[idx + 1:]
> +    if value[0] == '"':
> +        # Decode the escaped string into a byte string.
> +        assert value[-1] == '"'
> +        idx = 1
> +        result = []
> +        while True:
> +            ch = value[idx]
> +            if ch == '\\':
> +                if value[idx + 1] in '"\\':
> +                    result.append(ord(value[idx + 1]))
> +                    idx += 2
> +                else:
> +                    result.append(int(value[idx + 1:idx + 4], 8))
> +                    idx += 4
> +            elif ch == '"':
> +                assert idx == len(value) - 1
> +                break
> +            else:
> +                result.append(ord(value[idx]))
> +                idx += 1
> +        value = bytes(result)
> +    else:
> +        # Convert the value into an integer.
> +        assert value.startswith('0x')
> +        value = int(value, 0)
> +    return (tuple(subscripts), value)
> +@end smallexample
> +
> +@subsection Dynamic Linker Diagnostics Values
> +
> +As mentioned above, the set of diagnostics may change between
> +@theglibc{} releases.  Nevertheless, the following table documents a few
> +common diagnostic items.
> +
> +@table @code
> +@item dl_dst_lib=@var{string}
> +The @code{$LIB} dynamic string token expands to @var{string}.
> +
> +@item dl_hwcap=@var{integer}
> +@itemx dl_hwcap2=@var{integer}
> +@cindex HWCAP (diagnostics)
> +The HWCAP and HWCAP2 values, as returned for @code{getauxval}, and as
> +used in other places depending on the architecture.

Please put indices before @items they refer to.  I've recently gone over
all GCC manuals to correct for this exact error, as we've made some
upstream changes to Texinfo rely on index-then-item ordering to provide
a nice pilcrow anchor for copyable links.

See
https://inbox.sourceware.org/gcc-patches/20230223102714.3606058-3-arsen@aarsen.me/
for some context, as well as the GCC docs for the resulting pilcrows
(note that there's no Texinfo release which will demonstrate this yet,
so we use a snapshot for GCC).

> +@item dl_pagesize=@var{integer}
> +@cindex page size (diagnostics)
> +The system page size is @var{integer} bytes.
> +
> +@item dl_platform=@var{string}
> +The @code{$PLATFORM} dynamic string token expands to @var{string}.
> +
> +@item dso.libc=@var{string}
> +This is the soname of the shared @code{libc} object that is part of
> +@theglibc{}.  On most architectures, this is @code{libc.so.6}.
> +
> +@item env[@var{index}]=@var{string}
> +@itemx env_filtered[@var{index}]=@var{string}
> +An environment variable from the process environment.  The integer
> +@var{index} is the array index in the environment array.  Variables
> +under @code{env} include the variable value after the @samp{=} (assuming
> +that it was present), variables under @code{env_filtered} do not.
> +
> +@item path.prefix=@var{string}
> +This indicates that @theglibc{} was configured using
> +@samp{--prefix=@var{string}}.
> +
> +@item path.sysconfdir=@var{string}
> +@Theglibc{} was configured (perhaps implicitly) with
> +@samp{--sysconfdir=@var{string}} (typically @code{/etc}).
> +
> +@item path.system_dirs[@var{index}]=@var{string}
> +These items list the elements of the built-in array that describes the
> +default library search path.  The value @var{string} a directory file
> +name with a trailing @samp{/}.
> +
> +@item path.rtld=@var{string}
> +This string indicates the application binary interface (ABI) file name
> +of the run-time dynamic linker.
> +
> +@item version.release="stable"
> +@itemx version.release="development"
> +The value @code{"stable"} indicates that this build of @theglibc{} is
> +from a release branch.  Releases labeled as @code{"development"} are
> +unreleased development versions.
> +
> +@item version.version="@var{major}.@var{minor}"
> +@itemx version.version="@var{major}.@var{minor}.9000"
> +@cindex version (diagnostics)
> +@Theglibc{} version.  Development releases end in @samp{.9000}.
> +
> +@item auxv[@var{index}].a_type=@var{type}
> +@itemx auxv[@var{index}].a_val=@var{integer}
> +@itemx auxv[@var{index}].a_val_string=@var{string}
> +@cindex auxiliary vector (diagnostics)
> +An entry in the auxiliary vector (specific to Linux).  The values
> +@var{type} (an integer) and @var{integer} correspond to the members of
> +@code{struct auxv}.  If the value is a string, @code{a_val_string} is
> +used instead of @code{a_val}, so that values have consistent types.
> +
> +The @code{AT_HWCAP} and @code{AT_HWCAP2} values in this output do not
> +reflect adjustment by @theglibc{}.
> +
> +@item uname.sysname=@var{string}
> +@itemx uname.nodename=@var{string}
> +@itemx uname.release=@var{string}
> +@itemx uname.version=@var{string}
> +@itemx uname.machine=@var{string}
> +@itemx uname.domain=@var{string}
> +These Linux-specific items show the values of @code{struct utsname}, as
> +reported by the @code{uname} function.  @xref{Platform Type}.
> +
> +@item x86.cpu_features.@dots{}
> +@cindex CPUID (diagnostics)
> +These items are specific to the i386 and x86-64 architectures.  They
> +reflect supported CPU feature and information on cache geometry, mostly
> +collected using the @code{CPUID} instruction.
> +@end table
> +
>  @node Dynamic Linker Introspection
>  @section Dynamic Linker Introspection

Thank you!  Have a lovely night.
-- 
Arsen Arsenović

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 381 bytes --]

Re: [PATCH 2/3] manual: Document ld.so --list-diagnostics output

[Florian Weimer]

* Arsen Arsenović:

>> +@item dl_hwcap=@var{integer}
>> +@itemx dl_hwcap2=@var{integer}
>> +@cindex HWCAP (diagnostics)
>> +The HWCAP and HWCAP2 values, as returned for @code{getauxval}, and as
>> +used in other places depending on the architecture.
>
> Please put indices before @items they refer to.  I've recently gone over
> all GCC manuals to correct for this exact error, as we've made some
> upstream changes to Texinfo rely on index-then-item ordering to provide
> a nice pilcrow anchor for copyable links.
>
> See
> https://inbox.sourceware.org/gcc-patches/20230223102714.3606058-3-arsen@aarsen.me/
> for some context, as well as the GCC docs for the resulting pilcrows
> (note that there's no Texinfo release which will demonstrate this yet,
> so we use a snapshot for GCC).

Thank you, I've changed this locally.

Florian

Re: [PATCH 2/3] manual: Document ld.so --list-diagnostics output

[Adhemerval Zanella Netto]

Looks good, some minor comments below.

On 04/08/23 15:16, Florian Weimer via Libc-alpha wrote:
> ---
>  manual/dynlink.texi | 279 ++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 279 insertions(+)
> 
> diff --git a/manual/dynlink.texi b/manual/dynlink.texi
> index 45bf5a5b55..fc2cd2f0a4 100644
> --- a/manual/dynlink.texi
> +++ b/manual/dynlink.texi
> @@ -13,9 +13,288 @@ as plugins) later at run time.
>  Dynamic linkers are sometimes called @dfn{dynamic loaders}.
>  
>  @menu
> +* Dynamic Linker Invocation::   Explicit invocation of the dynamic linker.
>  * Dynamic Linker Introspection::    Interfaces for querying mapping information.
>  @end menu
>  
> +@node Dynamic Linker Invocation
> +
> +@cindex program interpreter
> +When a dynamically linked program starts, the operating system
> +automatically loads the dynamic linker along with the program.
> +@Theglibc{} also supports invoking the dynamic linker explicitly to
> +launch a program.  This command uses the implied dynamic linker
> +(also sometimes called the @dfn{program interpreter}):
> +
> +@smallexample
> +sh -c 'echo "Hello, world!"'
> +@end smallexample
> +
> +This command specifies the dynamic linker explicitly:
> +
> +@smallexample
> +ld.so /bin/sh -c 'echo "Hello, world!"'
> +@end smallexample
> +
> +Note that @command{ld.so} does not search the @env{PATH} environment
> +variable, so the full file name of the executable needs to be specified.
> +
> +The @command{ld.so} program supports various options.  Options start
> +@samp{--} and need to come before the program that is being launched.
> +Some of the supported options are listed below.
> +
> +@table @code
> +@item --list-diagnostics
> +Print system diagnostic information in a machine-readable format.
> +@xref{Dynamic Linker Diagnostics}.
> +@end table
> +
> +@menu
> +* Dynamic Linker Diagnostics::   Obtaining system diagnostic information.
> +@end menu
> +
> +@node Dynamic Linker Diagnostics
> +@section Dynamic Linker Diagnostics
> +@cindex diagnostics (dynamic linker)
> +
> +The @samp{ld.so --list-diagnostics} produces machine-readable
> +diagnostics output.  This output contains system data that affects
> +behavior of @theglibc{}, and potentially application behavior as well.

Maybe 'the behavior'.

> +
> +The exact set of diagnostic items can change between releases of
> +@theglibc{}.  The output format itself is not expected to change
> +radically.
> +
> +The following table shows some example lines that can be written by the
> +diagnostics command.
> +
> +@table @code
> +@item dl_pagesize=0x1000
> +The system page size is 4096 bytes.
> +
> +@item env[0x14]="LANG=en_US.UTF-8"
> +This item indicates that the 21st environment variable at process
> +startup contains a setting for @code{LANG}.
> +
> +@item env_filtered[0x22]="DISPLAY"
> +The 35th environment variable is @code{DISPLAY}.  Its value is not
> +included in the output for privacy reasons because it is not recognized
> +as harmless by the diagnostics code.
> +
> +@item path.prefix="/usr"
> +This means that @theglibc{} was configured with @code{--prefix=/usr}.
> +
> +@item path.system_dirs[0x0]="/lib64/"
> +@itemx path.system_dirs[0x1]="/usr/lib64/"
> +The built-in dynamic linker search path contains two directories,
> +@code{/lib64} and @code{/usr/lib64}.
> +@end table
> +
> +@subsection Dynamic Linker Diagnostics Output Format
> +
> +As seen above, diagnostic lines assign values (integers or strings) to a
> +sequences of labeled subscripts, separated by @samp{.}.  Some subscripts

s/sequences/sequence

> +have integer indices associated with them.  The subscript indices are
> +not necessarily contiguous or small, so an associative array should be
> +used to store them.  Currently, all integers fit into the 64-bit
> +unsigned integer range.  Every access path to a value has a fixed type
> +(string or integer) independently of subscript index values.  Likewise,

s/independently/independent

> +whether a subscript is indexed does not depend on previous indices (but
> +may depend on previous subscript labels).
> +
> +A syntax description in ABNF (RFC 5234) follows.  Note that
> +@code{%x30-39} denotes the range of decimal digits.  Diagnostic output
> +lines are expected to match the @code{line} production.
> +
> +@c ABNF-START
> +@smallexample
> +HEXDIG = %x30-39 / %x61-6f ; lowercase a-f only
> +ALPHA = %x41-5a / %x61-7a / %x7f ; letters and underscore
> +ALPHA-NUMERIC = ALPHA / %x30-39 / "_"
> +DQUOTE = %x22 ; "
> +
> +; Numbers are always hexadecimal and use a 0x prefix.
> +hex-value-prefix = %x30 %x78
> +hex-value = hex-value-prefix 1*HEXDIG
> +
> +; Strings use octal escape sequences and \\, \".
> +string-char = %x20-21 / %x23-5c / %x5d-7e ; printable but not "\
> +string-quoted-octal = %x30-33 2*2%x30-37
> +string-quoted = "\" ("\" / DQUOTE / string-quoted-octal)
> +string-value = DQUOTE *(string-char / string-quoted) DQUOTE
> +
> +value = hex-value / string-value
> +
> +label = ALPHA *ALPHA-NUMERIC
> +index = "[" hex-value "]"
> +subscript = label [index]
> +
> +line = subscript *("." subscript) "=" value
> +@end smallexample
> +
> +Output lines can be parsed with the following Python function.  It
> +assumes lines formatted according to the ABNF @code{line} production
> +above.

I don't think it would be useful to add the python script example below,
it is not an full example (reader will need to fill the blacks to actually
run it) and he ABNF description should be suffice.

> +
> +@c PYTHON-START
> +@smallexample
> +def parse_line(line):
> +    """Parse a line of --list-diagnostics output.
> +
> +    This function returns a pair (SUBSCRIPTS, VALUE).  VALUE is either
> +    a byte string or an integer.  SUBSCRIPT is a tuple of (LABEL,
> +    INDEX) pairs, where LABEL is a field identifier (a string), and
> +    INDEX is an integer or None, to indicate that this field is not
> +    indexed.
> +
> +    """
> +
> +    # Extract the list of subscripts before the value.
> +    idx = 0
> +    subscripts = []
> +    while line[idx] != '=':
> +        start_idx = idx
> +
> +        # Extract the label.
> +        while line[idx] not in '[.=':
> +            idx += 1
> +        label = line[start_idx:idx]
> +
> +        if line[idx] == '[':
> +            # Subscript with a 0x index.
> +            assert label
> +            close_bracket = line.index(']', idx)
> +            index = line[idx + 1:close_bracket]
> +            assert index.startswith('0x')
> +            index = int(index, 0)
> +            subscripts.append((label, index))
> +            idx = close_bracket + 1
> +        else: # '.' or '='.
> +            if label:
> +                subscripts.append((label, None))
> +            if line[idx] == '.':
> +                idx += 1
> +
> +    # The value is either a string or a 0x number.
> +    value = line[idx + 1:]
> +    if value[0] == '"':
> +        # Decode the escaped string into a byte string.
> +        assert value[-1] == '"'
> +        idx = 1
> +        result = []
> +        while True:
> +            ch = value[idx]
> +            if ch == '\\':
> +                if value[idx + 1] in '"\\':
> +                    result.append(ord(value[idx + 1]))
> +                    idx += 2
> +                else:
> +                    result.append(int(value[idx + 1:idx + 4], 8))
> +                    idx += 4
> +            elif ch == '"':
> +                assert idx == len(value) - 1
> +                break
> +            else:
> +                result.append(ord(value[idx]))
> +                idx += 1
> +        value = bytes(result)
> +    else:
> +        # Convert the value into an integer.
> +        assert value.startswith('0x')
> +        value = int(value, 0)
> +    return (tuple(subscripts), value)
> +@end smallexample
> +
> +@subsection Dynamic Linker Diagnostics Values
> +
> +As mentioned above, the set of diagnostics may change between
> +@theglibc{} releases.  Nevertheless, the following table documents a few
> +common diagnostic items.
> +
> +@table @code
> +@item dl_dst_lib=@var{string}
> +The @code{$LIB} dynamic string token expands to @var{string}.

s/to/to a

> +
> +@item dl_hwcap=@var{integer}
> +@itemx dl_hwcap2=@var{integer}
> +@cindex HWCAP (diagnostics)
> +The HWCAP and HWCAP2 values, as returned for @code{getauxval}, and as
> +used in other places depending on the architecture.
> +
> +@item dl_pagesize=@var{integer}
> +@cindex page size (diagnostics)
> +The system page size is @var{integer} bytes.

Maybe add it is in hexadecimal.

> +
> +@item dl_platform=@var{string}
> +The @code{$PLATFORM} dynamic string token expands to @var{string}.

s/to/to a

> +
> +@item dso.libc=@var{string}
> +This is the soname of the shared @code{libc} object that is part of
> +@theglibc{}.  On most architectures, this is @code{libc.so.6}.
> +
> +@item env[@var{index}]=@var{string}
> +@itemx env_filtered[@var{index}]=@var{string}
> +An environment variable from the process environment.  The integer
> +@var{index} is the array index in the environment array.  Variables
> +under @code{env} include the variable value after the @samp{=} (assuming
> +that it was present), variables under @code{env_filtered} do not.
> +
> +@item path.prefix=@var{string}
> +This indicates that @theglibc{} was configured using
> +@samp{--prefix=@var{string}}.
> +
> +@item path.sysconfdir=@var{string}
> +@Theglibc{} was configured (perhaps implicitly) with
> +@samp{--sysconfdir=@var{string}} (typically @code{/etc}).
> +
> +@item path.system_dirs[@var{index}]=@var{string}
> +These items list the elements of the built-in array that describes the
> +default library search path.  The value @var{string} a directory file

s/a /is a

> +name with a trailing @samp{/}.
> +
> +@item path.rtld=@var{string}
> +This string indicates the application binary interface (ABI) file name
> +of the run-time dynamic linker.
> +
> +@item version.release="stable"
> +@itemx version.release="development"
> +The value @code{"stable"} indicates that this build of @theglibc{} is
> +from a release branch.  Releases labeled as @code{"development"} are
> +unreleased development versions.
> +
> +@item version.version="@var{major}.@var{minor}"
> +@itemx version.version="@var{major}.@var{minor}.9000"
> +@cindex version (diagnostics)
> +@Theglibc{} version.  Development releases end in @samp{.9000}.
> +
> +@item auxv[@var{index}].a_type=@var{type}
> +@itemx auxv[@var{index}].a_val=@var{integer}
> +@itemx auxv[@var{index}].a_val_string=@var{string}
> +@cindex auxiliary vector (diagnostics)
> +An entry in the auxiliary vector (specific to Linux).  The values
> +@var{type} (an integer) and @var{integer} correspond to the members of
> +@code{struct auxv}.  If the value is a string, @code{a_val_string} is
> +used instead of @code{a_val}, so that values have consistent types.
> +
> +The @code{AT_HWCAP} and @code{AT_HWCAP2} values in this output do not
> +reflect adjustment by @theglibc{}.
> +
> +@item uname.sysname=@var{string}
> +@itemx uname.nodename=@var{string}
> +@itemx uname.release=@var{string}
> +@itemx uname.version=@var{string}
> +@itemx uname.machine=@var{string}
> +@itemx uname.domain=@var{string}
> +These Linux-specific items show the values of @code{struct utsname}, as
> +reported by the @code{uname} function.  @xref{Platform Type}.
> +
> +@item x86.cpu_features.@dots{}
> +@cindex CPUID (diagnostics)
> +These items are specific to the i386 and x86-64 architectures.  They
> +reflect supported CPU feature and information on cache geometry, mostly

s/feature/features

> +collected using the @code{CPUID} instruction.
> +@end table
> +
>  @node Dynamic Linker Introspection
>  @section Dynamic Linker Introspection
>  

Re: [PATCH 2/3] manual: Document ld.so --list-diagnostics output

[Florian Weimer]

* Adhemerval Zanella Netto:

>> +Output lines can be parsed with the following Python function.  It
>> +assumes lines formatted according to the ABNF @code{line} production
>> +above.
>
> I don't think it would be useful to add the python script example below,
> it is not an full example (reader will need to fill the blacks to actually
> run it) and he ABNF description should be suffice.

I'm going to remove it.

>> +As mentioned above, the set of diagnostics may change between
>> +@theglibc{} releases.  Nevertheless, the following table documents a few
>> +common diagnostic items.
>> +
>> +@table @code
>> +@item dl_dst_lib=@var{string}
>> +The @code{$LIB} dynamic string token expands to @var{string}.
>
> s/to/to a

I think @var{string} as a variable should not have an article.

>> +
>> +@item dl_hwcap=@var{integer}
>> +@itemx dl_hwcap2=@var{integer}
>> +@cindex HWCAP (diagnostics)
>> +The HWCAP and HWCAP2 values, as returned for @code{getauxval}, and as
>> +used in other places depending on the architecture.
>> +
>> +@item dl_pagesize=@var{integer}
>> +@cindex page size (diagnostics)
>> +The system page size is @var{integer} bytes.
>
> Maybe add it is in hexadecimal.

I'm going to add a sentence to the introductory paragraph:

All numbers are in hexadecimal, with a @samp{0x} prefix.

I've applied the rest of your suggestions.

Thanks,
Florian

[PATCH 3/3] elf: Check that --list-diagnostics output has the expected syntax

[Florian Weimer]

Parts of elf/tst-rtld-list-diagnostics.py have been copied from
scripts/tst-ld-trace.py.

The abnf module is entirely optional and used to verify the
ABNF grammar as included in the manual.
---
 INSTALL                          |   5 +
 elf/Makefile                     |   9 +
 elf/tst-rtld-list-diagnostics.py | 311 +++++++++++++++++++++++++++++++
 manual/install.texi              |   6 +
 4 files changed, 331 insertions(+)
 create mode 100644 elf/tst-rtld-list-diagnostics.py

diff --git a/INSTALL b/INSTALL
index 268acadd75..3f662bf427 100644
--- a/INSTALL
+++ b/INSTALL
@@ -585,6 +585,11 @@ build the GNU C Library:
      in your system.  As of release time PExpect 4.8.0 is the newest
      verified to work to test the pretty printers.
 
+   • The Python ‘abnf’ module.
+
+     This module is used to verify some ABNF grammars in the manual.
+     Version 2.2.0 has been confirmed to work as expected.
+
    • GDB 7.8 or later with support for Python 2.7/3.4 or later
 
      GDB itself needs to be configured with Python support in order to
diff --git a/elf/Makefile b/elf/Makefile
index c00e2ccfc5..9176cbf1e3 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -1123,6 +1123,7 @@ tests-special += \
   $(objpfx)argv0test.out \
   $(objpfx)tst-pathopt.out \
   $(objpfx)tst-rtld-help.out \
+  $(objpfx)tst-rtld-list-diagnostics.out \
   $(objpfx)tst-rtld-load-self.out \
   $(objpfx)tst-rtld-preload.out \
   $(objpfx)tst-sprof-basic.out \
@@ -2799,6 +2800,14 @@ $(objpfx)tst-ro-dynamic-mod.so: $(objpfx)tst-ro-dynamic-mod.os \
 		-Wl,--script=tst-ro-dynamic-mod.map \
 		$(objpfx)tst-ro-dynamic-mod.os
 
+$(objpfx)tst-rtld-list-diagnostics.out: tst-rtld-list-diagnostics.py \
+  $(..)manual/dynlink.texi $(objpfx)$(rtld-installed-name)
+	$(PYTHON) tst-rtld-list-diagnostics.py \
+	  --manual=$(..)manual/dynlink.texi \
+	  "$(test-wrapper-env) $(objpfx)$(rtld-installed-name) --list-diagnostics" \
+	  > $@; \
+	$(evaluate-test)
+
 $(objpfx)tst-rtld-run-static.out: $(objpfx)/ldconfig
 
 $(objpfx)tst-dl_find_object.out: \
diff --git a/elf/tst-rtld-list-diagnostics.py b/elf/tst-rtld-list-diagnostics.py
new file mode 100644
index 0000000000..f4ff06fd86
--- /dev/null
+++ b/elf/tst-rtld-list-diagnostics.py
@@ -0,0 +1,311 @@
+#!/usr/bin/python3
+# Test that the ld.so --list-diagnostics output has the expected syntax.
+# Copyright (C) 2022-2023 Free Software Foundation, Inc.
+# Copyright The GNU Toolchain Authors.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <https://www.gnu.org/licenses/>.
+
+import argparse
+import collections
+import subprocess
+import sys
+
+try:
+    subprocess.run
+except:
+    class _CompletedProcess:
+        def __init__(self, args, returncode, stdout=None, stderr=None):
+            self.args = args
+            self.returncode = returncode
+            self.stdout = stdout
+            self.stderr = stderr
+
+    def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
+        assert(timeout is None)
+        with subprocess.Popen(*popenargs, **kwargs) as process:
+            try:
+                stdout, stderr = process.communicate(input)
+            except:
+                process.kill()
+                process.wait()
+                raise
+            returncode = process.poll()
+            if check and returncode:
+                raise subprocess.CalledProcessError(returncode, popenargs)
+        return _CompletedProcess(popenargs, returncode, stdout, stderr)
+
+    subprocess.run = _run
+
+# Number of errors encountered.  Zero means no errors (test passes).
+errors = 0
+
+# PYTHON-START
+def parse_line(line):
+    """Parse a line of --list-diagnostics output.
+
+    This function returns a pair (SUBSCRIPTS, VALUE).  VALUE is either
+    a byte string or an integer.  SUBSCRIPT is a tuple of (LABEL,
+    INDEX) pairs, where LABEL is a field identifier (a string), and
+    INDEX is an integer or None, to indicate that this field is not
+    indexed.
+
+    """
+
+    # Extract the list of subscripts before the value.
+    idx = 0
+    subscripts = []
+    while line[idx] != '=':
+        start_idx = idx
+
+        # Extract the label.
+        while line[idx] not in '[.=':
+            idx += 1
+        label = line[start_idx:idx]
+
+        if line[idx] == '[':
+            # Subscript with a 0x index.
+            assert label
+            close_bracket = line.index(']', idx)
+            index = line[idx + 1:close_bracket]
+            assert index.startswith('0x')
+            index = int(index, 0)
+            subscripts.append((label, index))
+            idx = close_bracket + 1
+        else: # '.' or '='.
+            if label:
+                subscripts.append((label, None))
+            if line[idx] == '.':
+                idx += 1
+
+    # The value is either a string or a 0x number.
+    value = line[idx + 1:]
+    if value[0] == '"':
+        # Decode the escaped string into a byte string.
+        assert value[-1] == '"'
+        idx = 1
+        result = []
+        while True:
+            ch = value[idx]
+            if ch == '\\':
+                if value[idx + 1] in '"\\':
+                    result.append(ord(value[idx + 1]))
+                    idx += 2
+                else:
+                    result.append(int(value[idx + 1:idx + 4], 8))
+                    idx += 4
+            elif ch == '"':
+                assert idx == len(value) - 1
+                break
+            else:
+                result.append(ord(value[idx]))
+                idx += 1
+        value = bytes(result)
+    else:
+        # Convert the value into an integer.
+        assert value.startswith('0x')
+        value = int(value, 0)
+    return (tuple(subscripts), value)
+# PYTHON-END
+
+assert parse_line('a.b[0x1]=0x2') == ((('a', None), ('b', 1)), 2)
+assert parse_line(r'b[0x3]="four\040\"\\"') == ((('b', 3),), b'four \"\\')
+
+# ABNF for a line of --list-diagnostics output.
+diagnostics_abnf = r"""
+HEXDIG = %x30-39 / %x61-6f ; lowercase a-f only
+ALPHA = %x41-5a / %x61-7a / %x7f ; letters and underscore
+ALPHA-NUMERIC = ALPHA / %x30-39 / "_"
+DQUOTE = %x22 ; "
+
+; Numbers are always hexadecimal and use a 0x prefix.
+hex-value-prefix = %x30 %x78
+hex-value = hex-value-prefix 1*HEXDIG
+
+; Strings use octal escape sequences and \\, \".
+string-char = %x20-21 / %x23-5c / %x5d-7e ; printable but not "\
+string-quoted-octal = %x30-33 2*2%x30-37
+string-quoted = "\" ("\" / DQUOTE / string-quoted-octal)
+string-value = DQUOTE *(string-char / string-quoted) DQUOTE
+
+value = hex-value / string-value
+
+label = ALPHA *ALPHA-NUMERIC
+index = "[" hex-value "]"
+subscript = label [index]
+
+line = subscript *("." subscript) "=" value
+"""
+
+def check_consistency_with_manual(manual_path):
+    """Verify that the code fragments in the manual match this script.
+
+    The code fragments are duplicated to clarify the dual license.
+    """
+
+    global errors
+
+    def extract_lines(path, start_line, end_line, skip_lines=()):
+        result = []
+        with open(path) as inp:
+            capturing = False
+            for line in inp:
+                if line.strip() == start_line:
+                    capturing = True
+                elif not capturing or line.strip() in skip_lines:
+                    continue
+                elif line.strip() == end_line:
+                    capturing = False
+                else:
+                    result.append(line)
+        if not result:
+            raise ValueError('{!r} not found in {!r}'.format(start_line, path))
+        if capturing:
+            raise ValueError('{!r} not found in {!r}'.format(end_line, path))
+        return result
+
+    def check(name, manual, script):
+        global errors
+
+        if manual == script:
+            return
+        print('error: {} fragment in manual is different'.format(name))
+        import difflib
+        sys.stdout.writelines(difflib.unified_diff(
+            manual, script, fromfile='manual', tofile='script'))
+        errors += 1
+
+    manual_python = extract_lines(manual_path,
+                                  '@c PYTHON-START', '@end smallexample',
+                                  skip_lines=('@smallexample',))
+    script_python = extract_lines(__file__, '# PYTHON-START', '# PYTHON-END')
+    check('Python code', manual_python, script_python)
+
+    manual_abnf = extract_lines(manual_path,
+                                '@c ABNF-START', '@end smallexample',
+                                skip_lines=('@smallexample',))
+    check('ABNF', diagnostics_abnf.splitlines(keepends=True)[1:], manual_abnf)
+
+# If the abnf module can be imported, run an additional check that the
+# 'line' production from the ABNF grammar matches --list-diagnostics
+# output lines.
+try:
+    import abnf
+except ImportError:
+    abnf = None
+    print('info: skipping ABNF validation because the abnf module is missing')
+
+if abnf is not None:
+    class Grammar(abnf.Rule):
+        pass
+
+    Grammar.load_grammar(diagnostics_abnf)
+
+    def parse_abnf(line):
+        global errors
+
+        # Just verify that the line parses.
+        try:
+            Grammar('line').parse_all(line)
+        except abnf.ParseError:
+            print('error: ABNF parse error:', repr(line))
+            errors += 1
+else:
+    def parse_abnf(line):
+        pass
+
+
+def parse_diagnostics(cmd):
+    global errors
+    diag_out = subprocess.run(cmd, stdout=subprocess.PIPE, check=True,
+                              universal_newlines=True).stdout
+    if diag_out[-1] != '\n':
+        print('error: ld.so output does not end in newline')
+        errors += 1
+
+    PathType = collections.namedtuple('PathType',
+                                      'has_index value_type original_line')
+    # Mapping tuples of labels to PathType values.
+    path_types = {}
+
+    seen_subscripts = {}
+
+    for line in diag_out.splitlines():
+        parse_abnf(line)
+        subscripts, value = parse_line(line)
+
+        # Check for duplicates.
+        if subscripts in seen_subscripts:
+            print('error: duplicate value assignment:', repr(line))
+            print('  previous line:,', repr(seen_subscripts[line]))
+            errors += 1
+        else:
+            seen_subscripts[subscripts] = line
+
+        # Compare types against the previously seen labels.
+        labels = tuple([label for label, index in subscripts])
+        has_index = tuple([index is not None for label, index in subscripts])
+        value_type = type(value)
+        if labels in path_types:
+            previous_type = path_types[labels]
+            if has_index != previous_type.has_index:
+                print('error: line has mismatch of indexing:', repr(line))
+                print('  index types:', has_index)
+                print('  previous:   ', previous_type.has_index)
+                print('  previous line:', repr(previous_type.original_line))
+                errors += 1
+            if value_type != previous_type.value_type:
+                print('error: line has mismatch of value type:', repr(line))
+                print('  value type:', value_type.__name__)
+                print('  previous:  ', previous_type.value_type.__name__)
+                print('  previous line:', repr(previous_type.original_line))
+                errors += 1
+        else:
+            path_types[labels] = PathType(has_index, value_type, line)
+
+        # Check that this line does not add indexing to a previous value.
+        for idx in range(1, len(subscripts) - 1):
+            if subscripts[:idx] in path_types:
+                print('error: line assigns to atomic value:', repr(line))
+                print('  previous line:', repr(previous_type.original_line))
+                errors += 1
+
+    if errors:
+        sys.exit(1)
+
+def get_parser():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--manual',
+                        help='path to .texi file for consistency checks')
+    parser.add_argument('command',
+                        help='comand to run')
+    return parser
+
+
+def main(argv):
+    parser = get_parser()
+    opts = parser.parse_args(argv)
+
+    if opts.manual:
+        check_consistency_with_manual(opts.manual)
+
+    # Remove the initial 'env' command.
+    parse_diagnostics(opts.command.split()[1:])
+
+    if errors:
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/manual/install.texi b/manual/install.texi
index e8f36d5726..2107eb7268 100644
--- a/manual/install.texi
+++ b/manual/install.texi
@@ -632,6 +632,12 @@ GDB, and should be compatible with the Python version in your system.
 As of release time PExpect 4.8.0 is the newest verified to work to test
 the pretty printers.
 
+@item
+The Python @code{abnf} module.
+
+This module is used to verify some ABNF grammars in the manual.
+Version 2.2.0 has been confirmed to work as expected.
+
 @item
 GDB 7.8 or later with support for Python 2.7/3.4 or later
 
-- 
2.41.0

Re: [PATCH 3/3] elf: Check that --list-diagnostics output has the expected syntax

[Adhemerval Zanella Netto]

On 04/08/23 15:16, Florian Weimer via Libc-alpha wrote:
> Parts of elf/tst-rtld-list-diagnostics.py have been copied from
> scripts/tst-ld-trace.py.
> 
> The abnf module is entirely optional and used to verify the
> ABNF grammar as included in the manual.
> ---
>  INSTALL                          |   5 +
>  elf/Makefile                     |   9 +
>  elf/tst-rtld-list-diagnostics.py | 311 +++++++++++++++++++++++++++++++
>  manual/install.texi              |   6 +
>  4 files changed, 331 insertions(+)
>  create mode 100644 elf/tst-rtld-list-diagnostics.py
> 
> diff --git a/INSTALL b/INSTALL
> index 268acadd75..3f662bf427 100644
> --- a/INSTALL
> +++ b/INSTALL
> @@ -585,6 +585,11 @@ build the GNU C Library:
>       in your system.  As of release time PExpect 4.8.0 is the newest
>       verified to work to test the pretty printers.
>  
> +   • The Python ‘abnf’ module.
> +
> +     This module is used to verify some ABNF grammars in the manual.
> +     Version 2.2.0 has been confirmed to work as expected.
> +

This part mixes requires and recommended tools, I think it should be clear
that the abnf module is complete optional and it not being present only
reduces the test coverage.

The rest look ok.

>     • GDB 7.8 or later with support for Python 2.7/3.4 or later
>  
>       GDB itself needs to be configured with Python support in order to
> diff --git a/elf/Makefile b/elf/Makefile
> index c00e2ccfc5..9176cbf1e3 100644
> --- a/elf/Makefile
> +++ b/elf/Makefile
> @@ -1123,6 +1123,7 @@ tests-special += \
>    $(objpfx)argv0test.out \
>    $(objpfx)tst-pathopt.out \
>    $(objpfx)tst-rtld-help.out \
> +  $(objpfx)tst-rtld-list-diagnostics.out \
>    $(objpfx)tst-rtld-load-self.out \
>    $(objpfx)tst-rtld-preload.out \
>    $(objpfx)tst-sprof-basic.out \
> @@ -2799,6 +2800,14 @@ $(objpfx)tst-ro-dynamic-mod.so: $(objpfx)tst-ro-dynamic-mod.os \
>  		-Wl,--script=tst-ro-dynamic-mod.map \
>  		$(objpfx)tst-ro-dynamic-mod.os
>  
> +$(objpfx)tst-rtld-list-diagnostics.out: tst-rtld-list-diagnostics.py \
> +  $(..)manual/dynlink.texi $(objpfx)$(rtld-installed-name)
> +	$(PYTHON) tst-rtld-list-diagnostics.py \
> +	  --manual=$(..)manual/dynlink.texi \
> +	  "$(test-wrapper-env) $(objpfx)$(rtld-installed-name) --list-diagnostics" \
> +	  > $@; \
> +	$(evaluate-test)
> +
>  $(objpfx)tst-rtld-run-static.out: $(objpfx)/ldconfig
>  
>  $(objpfx)tst-dl_find_object.out: \
> diff --git a/elf/tst-rtld-list-diagnostics.py b/elf/tst-rtld-list-diagnostics.py
> new file mode 100644
> index 0000000000..f4ff06fd86
> --- /dev/null
> +++ b/elf/tst-rtld-list-diagnostics.py
> @@ -0,0 +1,311 @@
> +#!/usr/bin/python3
> +# Test that the ld.so --list-diagnostics output has the expected syntax.
> +# Copyright (C) 2022-2023 Free Software Foundation, Inc.
> +# Copyright The GNU Toolchain Authors.
> +# This file is part of the GNU C Library.
> +#
> +# The GNU C Library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# The GNU C Library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with the GNU C Library; if not, see
> +# <https://www.gnu.org/licenses/>.
> +
> +import argparse
> +import collections
> +import subprocess
> +import sys
> +
> +try:
> +    subprocess.run
> +except:
> +    class _CompletedProcess:
> +        def __init__(self, args, returncode, stdout=None, stderr=None):
> +            self.args = args
> +            self.returncode = returncode
> +            self.stdout = stdout
> +            self.stderr = stderr
> +
> +    def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
> +        assert(timeout is None)
> +        with subprocess.Popen(*popenargs, **kwargs) as process:
> +            try:
> +                stdout, stderr = process.communicate(input)
> +            except:
> +                process.kill()
> +                process.wait()
> +                raise
> +            returncode = process.poll()
> +            if check and returncode:
> +                raise subprocess.CalledProcessError(returncode, popenargs)
> +        return _CompletedProcess(popenargs, returncode, stdout, stderr)
> +
> +    subprocess.run = _run
> +
> +# Number of errors encountered.  Zero means no errors (test passes).
> +errors = 0
> +
> +# PYTHON-START
> +def parse_line(line):
> +    """Parse a line of --list-diagnostics output.
> +
> +    This function returns a pair (SUBSCRIPTS, VALUE).  VALUE is either
> +    a byte string or an integer.  SUBSCRIPT is a tuple of (LABEL,
> +    INDEX) pairs, where LABEL is a field identifier (a string), and
> +    INDEX is an integer or None, to indicate that this field is not
> +    indexed.
> +
> +    """
> +
> +    # Extract the list of subscripts before the value.
> +    idx = 0
> +    subscripts = []
> +    while line[idx] != '=':
> +        start_idx = idx
> +
> +        # Extract the label.
> +        while line[idx] not in '[.=':
> +            idx += 1
> +        label = line[start_idx:idx]
> +
> +        if line[idx] == '[':
> +            # Subscript with a 0x index.
> +            assert label
> +            close_bracket = line.index(']', idx)
> +            index = line[idx + 1:close_bracket]
> +            assert index.startswith('0x')
> +            index = int(index, 0)
> +            subscripts.append((label, index))
> +            idx = close_bracket + 1
> +        else: # '.' or '='.
> +            if label:
> +                subscripts.append((label, None))
> +            if line[idx] == '.':
> +                idx += 1
> +
> +    # The value is either a string or a 0x number.
> +    value = line[idx + 1:]
> +    if value[0] == '"':
> +        # Decode the escaped string into a byte string.
> +        assert value[-1] == '"'
> +        idx = 1
> +        result = []
> +        while True:
> +            ch = value[idx]
> +            if ch == '\\':
> +                if value[idx + 1] in '"\\':
> +                    result.append(ord(value[idx + 1]))
> +                    idx += 2
> +                else:
> +                    result.append(int(value[idx + 1:idx + 4], 8))
> +                    idx += 4
> +            elif ch == '"':
> +                assert idx == len(value) - 1
> +                break
> +            else:
> +                result.append(ord(value[idx]))
> +                idx += 1
> +        value = bytes(result)
> +    else:
> +        # Convert the value into an integer.
> +        assert value.startswith('0x')
> +        value = int(value, 0)
> +    return (tuple(subscripts), value)
> +# PYTHON-END
> +
> +assert parse_line('a.b[0x1]=0x2') == ((('a', None), ('b', 1)), 2)
> +assert parse_line(r'b[0x3]="four\040\"\\"') == ((('b', 3),), b'four \"\\')
> +
> +# ABNF for a line of --list-diagnostics output.
> +diagnostics_abnf = r"""
> +HEXDIG = %x30-39 / %x61-6f ; lowercase a-f only
> +ALPHA = %x41-5a / %x61-7a / %x7f ; letters and underscore
> +ALPHA-NUMERIC = ALPHA / %x30-39 / "_"
> +DQUOTE = %x22 ; "
> +
> +; Numbers are always hexadecimal and use a 0x prefix.
> +hex-value-prefix = %x30 %x78
> +hex-value = hex-value-prefix 1*HEXDIG
> +
> +; Strings use octal escape sequences and \\, \".
> +string-char = %x20-21 / %x23-5c / %x5d-7e ; printable but not "\
> +string-quoted-octal = %x30-33 2*2%x30-37
> +string-quoted = "\" ("\" / DQUOTE / string-quoted-octal)
> +string-value = DQUOTE *(string-char / string-quoted) DQUOTE
> +
> +value = hex-value / string-value
> +
> +label = ALPHA *ALPHA-NUMERIC
> +index = "[" hex-value "]"
> +subscript = label [index]
> +
> +line = subscript *("." subscript) "=" value
> +"""
> +
> +def check_consistency_with_manual(manual_path):
> +    """Verify that the code fragments in the manual match this script.
> +
> +    The code fragments are duplicated to clarify the dual license.
> +    """
> +
> +    global errors
> +
> +    def extract_lines(path, start_line, end_line, skip_lines=()):
> +        result = []
> +        with open(path) as inp:
> +            capturing = False
> +            for line in inp:
> +                if line.strip() == start_line:
> +                    capturing = True
> +                elif not capturing or line.strip() in skip_lines:
> +                    continue
> +                elif line.strip() == end_line:
> +                    capturing = False
> +                else:
> +                    result.append(line)
> +        if not result:
> +            raise ValueError('{!r} not found in {!r}'.format(start_line, path))
> +        if capturing:
> +            raise ValueError('{!r} not found in {!r}'.format(end_line, path))
> +        return result
> +
> +    def check(name, manual, script):
> +        global errors
> +
> +        if manual == script:
> +            return
> +        print('error: {} fragment in manual is different'.format(name))
> +        import difflib
> +        sys.stdout.writelines(difflib.unified_diff(
> +            manual, script, fromfile='manual', tofile='script'))
> +        errors += 1
> +
> +    manual_python = extract_lines(manual_path,
> +                                  '@c PYTHON-START', '@end smallexample',
> +                                  skip_lines=('@smallexample',))
> +    script_python = extract_lines(__file__, '# PYTHON-START', '# PYTHON-END')
> +    check('Python code', manual_python, script_python)
> +
> +    manual_abnf = extract_lines(manual_path,
> +                                '@c ABNF-START', '@end smallexample',
> +                                skip_lines=('@smallexample',))
> +    check('ABNF', diagnostics_abnf.splitlines(keepends=True)[1:], manual_abnf)
> +
> +# If the abnf module can be imported, run an additional check that the
> +# 'line' production from the ABNF grammar matches --list-diagnostics
> +# output lines.
> +try:
> +    import abnf
> +except ImportError:
> +    abnf = None
> +    print('info: skipping ABNF validation because the abnf module is missing')
> +
> +if abnf is not None:
> +    class Grammar(abnf.Rule):
> +        pass
> +
> +    Grammar.load_grammar(diagnostics_abnf)
> +
> +    def parse_abnf(line):
> +        global errors
> +
> +        # Just verify that the line parses.
> +        try:
> +            Grammar('line').parse_all(line)
> +        except abnf.ParseError:
> +            print('error: ABNF parse error:', repr(line))
> +            errors += 1
> +else:
> +    def parse_abnf(line):
> +        pass
> +
> +
> +def parse_diagnostics(cmd):
> +    global errors
> +    diag_out = subprocess.run(cmd, stdout=subprocess.PIPE, check=True,
> +                              universal_newlines=True).stdout
> +    if diag_out[-1] != '\n':
> +        print('error: ld.so output does not end in newline')
> +        errors += 1
> +
> +    PathType = collections.namedtuple('PathType',
> +                                      'has_index value_type original_line')
> +    # Mapping tuples of labels to PathType values.
> +    path_types = {}
> +
> +    seen_subscripts = {}
> +
> +    for line in diag_out.splitlines():
> +        parse_abnf(line)
> +        subscripts, value = parse_line(line)
> +
> +        # Check for duplicates.
> +        if subscripts in seen_subscripts:
> +            print('error: duplicate value assignment:', repr(line))
> +            print('  previous line:,', repr(seen_subscripts[line]))
> +            errors += 1
> +        else:
> +            seen_subscripts[subscripts] = line
> +
> +        # Compare types against the previously seen labels.
> +        labels = tuple([label for label, index in subscripts])
> +        has_index = tuple([index is not None for label, index in subscripts])
> +        value_type = type(value)
> +        if labels in path_types:
> +            previous_type = path_types[labels]
> +            if has_index != previous_type.has_index:
> +                print('error: line has mismatch of indexing:', repr(line))
> +                print('  index types:', has_index)
> +                print('  previous:   ', previous_type.has_index)
> +                print('  previous line:', repr(previous_type.original_line))
> +                errors += 1
> +            if value_type != previous_type.value_type:
> +                print('error: line has mismatch of value type:', repr(line))
> +                print('  value type:', value_type.__name__)
> +                print('  previous:  ', previous_type.value_type.__name__)
> +                print('  previous line:', repr(previous_type.original_line))
> +                errors += 1
> +        else:
> +            path_types[labels] = PathType(has_index, value_type, line)
> +
> +        # Check that this line does not add indexing to a previous value.
> +        for idx in range(1, len(subscripts) - 1):
> +            if subscripts[:idx] in path_types:
> +                print('error: line assigns to atomic value:', repr(line))
> +                print('  previous line:', repr(previous_type.original_line))
> +                errors += 1
> +
> +    if errors:
> +        sys.exit(1)
> +
> +def get_parser():
> +    parser = argparse.ArgumentParser(description=__doc__)
> +    parser.add_argument('--manual',
> +                        help='path to .texi file for consistency checks')
> +    parser.add_argument('command',
> +                        help='comand to run')
> +    return parser
> +
> +
> +def main(argv):
> +    parser = get_parser()
> +    opts = parser.parse_args(argv)
> +
> +    if opts.manual:
> +        check_consistency_with_manual(opts.manual)
> +
> +    # Remove the initial 'env' command.
> +    parse_diagnostics(opts.command.split()[1:])
> +
> +    if errors:
> +        sys.exit(1)
> +
> +if __name__ == '__main__':
> +    main(sys.argv[1:])
> diff --git a/manual/install.texi b/manual/install.texi
> index e8f36d5726..2107eb7268 100644
> --- a/manual/install.texi
> +++ b/manual/install.texi
> @@ -632,6 +632,12 @@ GDB, and should be compatible with the Python version in your system.
>  As of release time PExpect 4.8.0 is the newest verified to work to test
>  the pretty printers.
>  
> +@item
> +The Python @code{abnf} module.
> +
> +This module is used to verify some ABNF grammars in the manual.
> +Version 2.2.0 has been confirmed to work as expected.
> +
>  @item
>  GDB 7.8 or later with support for Python 2.7/3.4 or later
>