RFA: libiberty: Fix infinite recursion in rust demangler (PRs 98886 and 99935)

RFA: libiberty: Fix infinite recursion in rust demangler (PRs 98886 and 99935)

[Nick Clifton]

Hi Guys,

  I would like to propose the patch below to fix a couple of sources
  of infinite recursion in libiberty's rust demangling code.  This patch
  is based upon the one submitted for PR 99935, but extended to cope
  with the case presented in PR 98886 and also fixed so that the "uint"
  type is not used.

  Tested with a patched version of the binutils sources on an
  x86-pc-linux-gnu target.

Cheers
  Nick

2022-01-26  Nick Clifton  <nickc@redhat.com>

	* rust-demangle.c (struct rust_demangler): Add a recursion
	counter.
	(demangle_path): Increment/decrement the recursion counter upon
	entry and exit.  Fail if the counter exceeds a fixed limit.
	(demangle_type): Likewise.
	(rust_demangle_callback): Initialise the recursion counter,
	disabling if requested by the option flags.

diff --git a/libiberty/rust-demangle.c b/libiberty/rust-demangle.c
index 18c760491bd..3b24d63892a 100644
--- a/libiberty/rust-demangle.c
+++ b/libiberty/rust-demangle.c
@@ -74,6 +74,12 @@ struct rust_demangler
   /* Rust mangling version, with legacy mangling being -1. */
   int version;
 
+  /* Recursion depth.  */
+  unsigned int recursion;
+  /* Maximum number of times demangle_path may be called recursively.  */
+#define RUST_MAX_RECURSION_COUNT  1024
+#define RUST_NO_RECURSION_LIMIT   ((unsigned int) -1)
+
   uint64_t bound_lifetime_depth;
 };
 
@@ -671,6 +677,15 @@ demangle_path (struct rust_demangler *rdm, int in_value)
   if (rdm->errored)
     return;
 
+  if (rdm->recursion != RUST_NO_RECURSION_LIMIT)
+    {
+      ++ rdm->recursion;
+      if (rdm->recursion > RUST_MAX_RECURSION_COUNT)
+	/* FIXME: There ought to be a way to report
+	   that the recursion limit has been reached.  */
+	goto fail_return;
+    }
+
   switch (tag = next (rdm))
     {
     case 'C':
@@ -688,10 +703,7 @@ demangle_path (struct rust_demangler *rdm, int in_value)
     case 'N':
       ns = next (rdm);
       if (!ISLOWER (ns) && !ISUPPER (ns))
-        {
-          rdm->errored = 1;
-          return;
-        }
+	goto fail_return;
 
       demangle_path (rdm, in_value);
 
@@ -776,9 +788,15 @@ demangle_path (struct rust_demangler *rdm, int in_value)
         }
       break;
     default:
-      rdm->errored = 1;
-      return;
+      goto fail_return;
     }
+  goto pass_return;
+
+ fail_return:
+  rdm->errored = 1;
+ pass_return:
+  if (rdm->recursion != RUST_NO_RECURSION_LIMIT)
+    -- rdm->recursion;
 }
 
 static void
@@ -870,6 +888,19 @@ demangle_type (struct rust_demangler *rdm)
       return;
     }
 
+   if (rdm->recursion != RUST_NO_RECURSION_LIMIT)
+    {
+      ++ rdm->recursion;
+      if (rdm->recursion > RUST_MAX_RECURSION_COUNT)
+	/* FIXME: There ought to be a way to report
+	   that the recursion limit has been reached.  */
+	{
+	  rdm->errored = 1;
+	  -- rdm->recursion;
+	  return;
+	}
+    }
+
   switch (tag)
     {
     case 'R':
@@ -1030,6 +1061,9 @@ demangle_type (struct rust_demangler *rdm)
       rdm->next--;
       demangle_path (rdm, 0);
     }
+
+  if (rdm->recursion != RUST_NO_RECURSION_LIMIT)
+    -- rdm->recursion;
 }
 
 /* A trait in a trait object may have some "existential projections"
@@ -1320,6 +1354,7 @@ rust_demangle_callback (const char *mangled, int options,
   rdm.skipping_printing = 0;
   rdm.verbose = (options & DMGL_VERBOSE) != 0;
   rdm.version = 0;
+  rdm.recursion = (options & DMGL_NO_RECURSE_LIMIT) ? RUST_NO_RECURSION_LIMIT : 0;
   rdm.bound_lifetime_depth = 0;
 
   /* Rust symbols always start with _R (v0) or _ZN (legacy). */

Re: RFA: libiberty: Fix infinite recursion in rust demangler (PRs 98886 and 99935)

[Jeff Law]

On 1/26/2022 8:29 AM, Nick Clifton via Gcc-patches wrote:
> Hi Guys,
>
>    I would like to propose the patch below to fix a couple of sources
>    of infinite recursion in libiberty's rust demangling code.  This patch
>    is based upon the one submitted for PR 99935, but extended to cope
>    with the case presented in PR 98886 and also fixed so that the "uint"
>    type is not used.
>
>    Tested with a patched version of the binutils sources on an
>    x86-pc-linux-gnu target.
>
> Cheers
>    Nick
>
> 2022-01-26  Nick Clifton  <nickc@redhat.com>
>
> 	* rust-demangle.c (struct rust_demangler): Add a recursion
> 	counter.
> 	(demangle_path): Increment/decrement the recursion counter upon
> 	entry and exit.  Fail if the counter exceeds a fixed limit.
> 	(demangle_type): Likewise.
> 	(rust_demangle_callback): Initialise the recursion counter,
> 	disabling if requested by the option flags.
OK.  I'm both amazed and disappointed at how all the demanglers seem to 
be prone to infinite recursion.

jeff