[gcc r13-4453] c++: comptypes ICE with BOUND_TEMPLATE_TEMPLATE_PARMs [PR107539]

[Patrick Palka <ppalka@gcc.gnu.org>]

https://gcc.gnu.org/g:a4e577b044d69977f93b2cb7769dc991eadf2cf0

commit r13-4453-ga4e577b044d69977f93b2cb7769dc991eadf2cf0
Author: Patrick Palka <ppalka@redhat.com>
Date:   Thu Dec 1 20:11:53 2022 -0500

    c++: comptypes ICE with BOUND_TEMPLATE_TEMPLATE_PARMs [PR107539]
    
    Here we end up giving the two BOUND_TEMPLATE_TEMPLATE_PARMs
    C<decltype(f::t)> and C<decltype(g::t)> the same TYPE_CANONICAL because
    the hash table that interns TYPE_CANONICAL for template type parameters
    doesn't set the comparing_specializations flag which controls how
    PARM_DECLs from different contexts compare equal.
    
    Later, from spec_hasher::equal for the corresponding two specializations
    A<C<decltype(f::t)>> and A<C<decltype(g::t)>>, we compare the two bound
    ttps with comparing_specializations set hence they now (structurally)
    compare different despite having the same TYPE_CANONICAL, and so we get
    the error:
    
      internal compiler error: same canonical type node for different types
        'C<decltype (t)>' and 'C<decltype (t)>'
    
    This suggests that we should be setting comparing_specializations from
    ctp_hasher::equal to match spec_hasher::equal.  But doing so introduces
    a separate ICE in cpp2a/concepts-placeholder3.C:
    
      internal compiler error: canonical types differ for identical types
        'auto [requires ::same_as<<placeholder>, decltype(f::x)>]' and
        'auto [requires ::same_as<<placeholder>, decltype(g::x)>]'
    
    because norm_hasher::equal doesn't set comparing_specializations either.
    
    I'm not sure when exactly we need to set comparing_specializations given
    what it controls (TYPENAME_TYPE equality/hashing and PARM_DECL equality)
    but it seems to be the conservative choice to set the flag wherever we
    have a global hash table that relies on type equality.  To that end this
    patch sets comparing_specializations in ctp_hasher and norm_hasher, as
    well as in atom_hasher and sat_hasher for good measure.  This turns out
    to be a compile time win of about 2% in some concepts tests, probably
    because of the improved TYPENAME_TYPE hashing enabled by the flag.
    
            PR c++/107539
    
    gcc/cp/ChangeLog:
    
            * constraint.cc (norm_hasher::hash, norm_hasher::equal): Set
            comparing_specializations.
            (sat_hasher::hash, sat_hasher::equal): Likewise.
            * cp-tree.h (atom_hasher::hash, atom_hasher::equal): Likewise.
            * pt.cc (ctp_hasher::hash, ctp_hasher::equal): Likewise.
    
    gcc/testsuite/ChangeLog:
    
            * g++.dg/template/canon-type-19.C: New test.

Diff:
---
 gcc/cp/constraint.cc                          | 18 +++++++++++++++---
 gcc/cp/cp-tree.h                              | 10 ++++++++--
 gcc/cp/pt.cc                                  |  7 ++++++-
 gcc/testsuite/g++.dg/template/canon-type-19.C | 18 ++++++++++++++++++
 4 files changed, 47 insertions(+), 6 deletions(-)

diff --git a/gcc/cp/constraint.cc b/gcc/cp/constraint.cc
index ab0f66b3d7e..37eae03afdb 100644
--- a/gcc/cp/constraint.cc
+++ b/gcc/cp/constraint.cc
@@ -715,14 +715,20 @@ struct norm_hasher : ggc_ptr_hash<norm_entry>
 {
   static hashval_t hash (norm_entry *e)
   {
-    hashval_t hash = iterative_hash_template_arg (e->tmpl, 0);
-    return iterative_hash_template_arg (e->args, hash);
+    ++comparing_specializations;
+    hashval_t val = iterative_hash_template_arg (e->tmpl, 0);
+    val = iterative_hash_template_arg (e->args, val);
+    --comparing_specializations;
+    return val;
   }
 
   static bool equal (norm_entry *e1, norm_entry *e2)
   {
-    return e1->tmpl == e2->tmpl
+    ++comparing_specializations;
+    bool eq = e1->tmpl == e2->tmpl
       && template_args_equal (e1->args, e2->args);
+    --comparing_specializations;
+    return eq;
   }
 };
 
@@ -2530,6 +2536,9 @@ struct sat_hasher : ggc_ptr_hash<sat_entry>
 {
   static hashval_t hash (sat_entry *e)
   {
+    auto cso = make_temp_override (comparing_specializations);
+    ++comparing_specializations;
+
     if (ATOMIC_CONSTR_MAP_INSTANTIATED_P (e->atom))
       {
 	/* Atoms with instantiated mappings are built during satisfaction.
@@ -2564,6 +2573,9 @@ struct sat_hasher : ggc_ptr_hash<sat_entry>
 
   static bool equal (sat_entry *e1, sat_entry *e2)
   {
+    auto cso = make_temp_override (comparing_specializations);
+    ++comparing_specializations;
+
     if (ATOMIC_CONSTR_MAP_INSTANTIATED_P (e1->atom)
 	!= ATOMIC_CONSTR_MAP_INSTANTIATED_P (e2->atom))
       return false;
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index 548b533266a..addd26ea077 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -8418,12 +8418,18 @@ struct atom_hasher : default_hash_traits<tree>
 {
   static hashval_t hash (tree t)
   {
-    return hash_atomic_constraint (t);
+    ++comparing_specializations;
+    hashval_t val = hash_atomic_constraint (t);
+    --comparing_specializations;
+    return val;
   }
 
   static bool equal (tree t1, tree t2)
   {
-    return atomic_constraints_identical_p (t1, t2);
+    ++comparing_specializations;
+    bool eq = atomic_constraints_identical_p (t1, t2);
+    --comparing_specializations;
+    return eq;
   }
 };
 
diff --git a/gcc/cp/pt.cc b/gcc/cp/pt.cc
index 08de273a900..31691618d1b 100644
--- a/gcc/cp/pt.cc
+++ b/gcc/cp/pt.cc
@@ -4492,18 +4492,23 @@ struct ctp_hasher : ggc_ptr_hash<tree_node>
 {
   static hashval_t hash (tree t)
   {
+    ++comparing_specializations;
     tree_code code = TREE_CODE (t);
     hashval_t val = iterative_hash_object (code, 0);
     val = iterative_hash_object (TEMPLATE_TYPE_LEVEL (t), val);
     val = iterative_hash_object (TEMPLATE_TYPE_IDX (t), val);
     if (TREE_CODE (t) == BOUND_TEMPLATE_TEMPLATE_PARM)
       val = iterative_hash_template_arg (TYPE_TI_ARGS (t), val);
+    --comparing_specializations;
     return val;
   }
 
   static bool equal (tree t, tree u)
   {
-    return comptypes (t, u, COMPARE_STRUCTURAL);
+    ++comparing_specializations;
+    bool eq = comptypes (t, u, COMPARE_STRUCTURAL);
+    --comparing_specializations;
+    return eq;
   }
 };
 
diff --git a/gcc/testsuite/g++.dg/template/canon-type-19.C b/gcc/testsuite/g++.dg/template/canon-type-19.C
new file mode 100644
index 00000000000..27ef3f1de3c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/canon-type-19.C
@@ -0,0 +1,18 @@
+// PR c++/107539
+// { dg-do compile { target c++11 } }
+
+template<class T>
+struct A { };
+
+template<template<class> class C>
+struct B {
+  template<class T>
+  void f(T t) {
+    A<C<decltype(t)>> a1;
+  }
+
+  template<class T>
+  void g(T t) {
+    A<C<decltype(t)>> a2;
+  }
+};