diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index d8b76d83d6850c3ce5318f3acd7cdc2a8cbc140b..228cb964278e7f57ef3c43093dbba16fe3e6748e 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -214,6 +214,14 @@ rtl-ssa-warn = $(STRICT_WARN)
 GCC_WARN_CFLAGS = $(LOOSE_WARN) $(C_LOOSE_WARN) $($(@D)-warn) $(if $(filter-out $(STRICT_WARN),$($(@D)-warn)),,$(C_STRICT_WARN)) $(NOCOMMON_FLAG) $($@-warn)
 GCC_WARN_CXXFLAGS = $(LOOSE_WARN) $($(@D)-warn) $(NOCOMMON_FLAG) $($@-warn)
 
+# The number of splits to be made for the match.pd files.
+NUM_MATCH_SPLITS = @DEFAULT_MATCHPD_PARTITIONS@
+MATCH_SPLITS_SEQ = $(shell seq 1 $(NUM_MATCH_SPLITS))
+GIMPLE_MATCH_PD_SEQ_SRC = $(patsubst %, gimple-match-%.cc, $(MATCH_SPLITS_SEQ))
+GIMPLE_MATCH_PD_SEQ_O = $(patsubst %, gimple-match-%.o, $(MATCH_SPLITS_SEQ))
+GENERIC_MATCH_PD_SEQ_SRC = $(patsubst %, generic-match-%.cc, $(MATCH_SPLITS_SEQ))
+GENERIC_MATCH_PD_SEQ_O = $(patsubst %, generic-match-%.o, $(MATCH_SPLITS_SEQ))
+
 # These files are to have specific diagnostics suppressed, or are not to
 # be subject to -Werror:
 # flex output may yield harmless "no previous prototype" warnings
@@ -222,8 +230,8 @@ gengtype-lex.o-warn = -Wno-error
 libgcov-util.o-warn = -Wno-error
 libgcov-driver-tool.o-warn = -Wno-error
 libgcov-merge-tool.o-warn = -Wno-error
-gimple-match.o-warn = -Wno-unused
-generic-match.o-warn = -Wno-unused
+gimple-match-head.o-warn = -Wno-unused
+gimple-match-exports.o-warn = -Wno-unused
 dfp.o-warn = -Wno-strict-aliasing
 
 # All warnings have to be shut off in stage1 if the compiler used then
@@ -1309,8 +1317,9 @@ ANALYZER_OBJS = \
 # will build them sooner, because they are large and otherwise tend to be
 # the last objects to finish building.
 OBJS = \
-	gimple-match.o \
-	generic-match.o \
+	$(GIMPLE_MATCH_PD_SEQ_O) \
+	gimple-match-exports.o \
+	$(GENERIC_MATCH_PD_SEQ_O) \
 	insn-attrtab.o \
 	insn-automata.o \
 	insn-dfatab.o \
@@ -1803,7 +1812,8 @@ MOSTLYCLEANFILES = insn-flags.h insn-config.h insn-codes.h \
  insn-output.cc insn-recog.cc insn-emit.cc insn-extract.cc insn-peep.cc \
  insn-attr.h insn-attr-common.h insn-attrtab.cc insn-dfatab.cc \
  insn-latencytab.cc insn-opinit.cc insn-opinit.h insn-preds.cc insn-constants.h \
- tm-preds.h tm-constrs.h checksum-options gimple-match.cc generic-match.cc \
+ tm-preds.h tm-constrs.h checksum-options $(GIMPLE_MATCH_PD_SEQ_SRC) \
+ $(GENERIC_MATCH_PD_SEQ_SRC) gimple-match-auto.h generic-match-auto.h \
  tree-check.h min-insn-modes.cc insn-modes.cc insn-modes.h insn-modes-inline.h \
  genrtl.h gt-*.h gtype-*.h gtype-desc.cc gtyp-input.list \
  case-cfn-macros.h cfn-operators.pd \
@@ -2418,7 +2428,8 @@ $(common_out_object_file): $(common_out_file)
 .PRECIOUS: insn-config.h insn-flags.h insn-codes.h insn-constants.h \
   insn-emit.cc insn-recog.cc insn-extract.cc insn-output.cc insn-peep.cc \
   insn-attr.h insn-attr-common.h insn-attrtab.cc insn-dfatab.cc \
-  insn-latencytab.cc insn-preds.cc gimple-match.cc generic-match.cc \
+  insn-latencytab.cc insn-preds.cc $(GIMPLE_MATCH_PD_SEQ_SRC) \
+  $(GENERIC_MATCH_PD_SEQ_SRC) gimple-match-auto.h generic-match-auto.h \
   insn-target-def.h
 
 # Dependencies for the md file.  The first time through, we just assume
@@ -2661,19 +2672,34 @@ s-tm-texi: build/genhooks$(build_exeext) $(srcdir)/doc/tm.texi.in
 	  false; \
 	fi
 
-gimple-match.cc: s-match gimple-match-head.cc ; @true
-generic-match.cc: s-match generic-match-head.cc ; @true
-
-s-match: build/genmatch$(build_exeext) $(srcdir)/match.pd cfn-operators.pd
-	$(RUN_GEN) build/genmatch$(build_exeext) --gimple $(srcdir)/match.pd \
-	    > tmp-gimple-match.cc
-	$(RUN_GEN) build/genmatch$(build_exeext) --generic $(srcdir)/match.pd \
-	    > tmp-generic-match.cc
-	$(SHELL) $(srcdir)/../move-if-change tmp-gimple-match.cc \
-	    					gimple-match.cc
-	$(SHELL) $(srcdir)/../move-if-change tmp-generic-match.cc \
-	    					generic-match.cc
-	$(STAMP) s-match
+$(GIMPLE_MATCH_PD_SEQ_SRC): s-gimple-match gimple-match-head.cc \
+		    gimple-match-exports.cc; @true
+gimple-match-auto.h: s-gimple-match gimple-match-head.cc \
+		    gimple-match-exports.cc; @true
+$(GENERIC_MATCH_PD_SEQ_SRC): s-generic-match generic-match-head.cc; @true
+generic-match-auto.h: s-generic-match generic-match-head.cc; @true
+
+s-gimple-match: build/genmatch$(build_exeext) \
+	    $(srcdir)/match.pd cfn-operators.pd
+	$(RUN_GEN) build/genmatch$(build_exeext) --gimple \
+	    --splits=$(NUM_MATCH_SPLITS) $(srcdir)/match.pd `pwd`/tmp-
+	$(foreach id, $(MATCH_SPLITS_SEQ), \
+	  $(SHELL) $(srcdir)/../move-if-change tmp-gimple-match-$(id).cc \
+	    gimple-match-$(id).cc;)
+	$(SHELL) $(srcdir)/../move-if-change tmp-gimple-match-auto.h \
+						gimple-match-auto.h
+	$(STAMP) s-gimple-match
+
+s-generic-match: build/genmatch$(build_exeext) \
+	    $(srcdir)/match.pd cfn-operators.pd
+	$(RUN_GEN) build/genmatch$(build_exeext) --generic \
+	    --splits=$(NUM_MATCH_SPLITS) $(srcdir)/match.pd `pwd`/tmp-
+	$(foreach id, $(MATCH_SPLITS_SEQ), \
+	  $(SHELL) $(srcdir)/../move-if-change tmp-generic-match-$(id).cc \
+	    generic-match-$(id).cc;)
+	$(SHELL) $(srcdir)/../move-if-change tmp-generic-match-auto.h \
+						generic-match-auto.h
+	$(STAMP) s-generic-match
 
 GTFILES = $(CPPLIB_H) $(srcdir)/input.h $(srcdir)/coretypes.h \
   $(host_xm_file_list) \
@@ -2808,7 +2834,8 @@ generated_files = config.h tm.h $(TM_P_H) $(TM_D_H) $(TM_H) multilib.h \
        $(ALL_GTFILES_H) gtype-desc.cc gtype-desc.h version.h \
        options.h target-hooks-def.h insn-opinit.h \
        common/common-target-hooks-def.h pass-instances.def \
-       gimple-match.cc generic-match.cc \
+       $(GIMPLE_MATCH_PD_SEQ_SRC) $(GENERIC_MATCH_PD_SEQ_SRC) \
+       gimple-match-auto.h generic-match-auto.h \
        c-family/c-target-hooks-def.h d/d-target-hooks-def.h \
        case-cfn-macros.h \
        cfn-operators.pd omp-device-properties.h
diff --git a/gcc/configure b/gcc/configure
index c7b26d1927de62d7b3a49ea9ac0a998979659cf2..843fd5449dd3c82fd5f7fc869ce951e89446ff6b 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -838,6 +838,7 @@ enable_gcov
 enable_shared
 enable_fixed_point
 enable_decimal_float
+DEFAULT_MATCHPD_PARTITIONS
 with_float
 with_cpu
 enable_multiarch
@@ -965,6 +966,7 @@ enable_valgrind_annotations
 enable_multilib
 enable_multiarch
 with_stack_clash_protection_guard_size
+with_matchpd_partitions
 enable___cxa_atexit
 enable_decimal_float
 enable_fixed_point
@@ -1826,6 +1828,9 @@ Optional Packages:
   --with-stack-clash-protection-guard-size=size
                           Set the default stack clash protection guard size
                           for specific targets as a power of two in bytes.
+  --with-matchpd-partitions=num
+                          Set the number of partitions to make for gimple and
+                          generic when splitting match.pd. [default=10]
   --with-dwarf2           force the default debug format to be DWARF 2 (or
                           later)
   --with-specs=SPECS      add SPECS to driver command-line processing
@@ -7889,6 +7894,26 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+# Specify the number of splits of match.pd to generate.
+
+# Check whether --with-matchpd-partitions was given.
+if test "${with_matchpd_partitions+set}" = set; then :
+  withval=$with_matchpd_partitions; DEFAULT_MATCHPD_PARTITIONS="$with_matchpd_partitions"
+else
+  DEFAULT_MATCHPD_PARTITIONS=10
+fi
+
+if (test $DEFAULT_MATCHPD_PARTITIONS -lt 1); then
+  as_fn_error $? "Invalid value $DEFAULT_MATCHPD_PARTITIONS for --with-matchpd-partitions. Cannot be negative." "$LINENO" 5
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define DEFAULT_MATCHPD_PARTITIONS $DEFAULT_MATCHPD_PARTITIONS
+_ACEOF
+
+
+
 # Enable __cxa_atexit for C++.
 # Check whether --enable-__cxa_atexit was given.
 if test "${enable___cxa_atexit+set}" = set; then :
@@ -19825,7 +19850,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 19828 "configure"
+#line 19853 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -19931,7 +19956,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 19934 "configure"
+#line 19959 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
diff --git a/gcc/configure.ac b/gcc/configure.ac
index 09082e8ccae395b902a3ef79eb4f27a1186a331e..de5e3aef2c81e1465198f903bfcd1219363c78d8 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -921,6 +921,21 @@ fi
 AC_DEFINE_UNQUOTED(DEFAULT_STK_CLASH_GUARD_SIZE, $DEFAULT_STK_CLASH_GUARD_SIZE,
 	[Define to larger than zero set the default stack clash protector size.])
 
+# Specify the number of splits of match.pd to generate.
+AC_ARG_WITH(matchpd-partitions,
+[AS_HELP_STRING([--with-matchpd-partitions=num],
+[Set the number of partitions to make for gimple and generic when splitting match.pd. [default=10]])],
+[DEFAULT_MATCHPD_PARTITIONS="$with_matchpd_partitions"], [DEFAULT_MATCHPD_PARTITIONS=10])
+if (test $DEFAULT_MATCHPD_PARTITIONS -lt 1); then
+  AC_MSG_ERROR(m4_normalize([
+		Invalid value $DEFAULT_MATCHPD_PARTITIONS for --with-matchpd-partitions. \
+		Cannot be negative.]))
+fi
+
+AC_DEFINE_UNQUOTED(DEFAULT_MATCHPD_PARTITIONS, $DEFAULT_MATCHPD_PARTITIONS,
+	[Define to larger than one set the number of match.pd partitions to make.])
+AC_SUBST(DEFAULT_MATCHPD_PARTITIONS)
+
 # Enable __cxa_atexit for C++.
 AC_ARG_ENABLE(__cxa_atexit,
 [AS_HELP_STRING([--enable-__cxa_atexit], [enable __cxa_atexit for C++])],
diff --git a/gcc/generic-match-head.cc b/gcc/generic-match-head.cc
index 16b8f9f3b61d3d5651a5a41a8c0552f50b55cc7c..397933cf5b2a7b774e9147d7c4876915d96e226e 100644
--- a/gcc/generic-match-head.cc
+++ b/gcc/generic-match-head.cc
@@ -115,4 +115,4 @@ void dump_debug (bool simplify, int loc, const char *file, int lineno)
   else
     fprintf (dump_file, "Matching expression %s:%d, %s:%d\n", "match.pd", loc,
 	     file, lineno);
-}
\ No newline at end of file
+}
diff --git a/gcc/genmatch.cc b/gcc/genmatch.cc
index bd7c6ff4a3fb89d456b02242707fd823b737f20d..5fd7b25b97b23aab50611ccd5fd8ee9c5cbb6887 100644
--- a/gcc/genmatch.cc
+++ b/gcc/genmatch.cc
@@ -183,6 +183,30 @@ fprintf_indent (FILE *f, unsigned int indent, const char *format, ...)
   va_end (ap);
 }
 
+/* Like fprintf, but print to two files, one header one C implementation.  */
+FILE *header_file = NULL;
+
+static void
+#if GCC_VERSION >= 4001
+__attribute__((format (printf, 4, 5)))
+#endif
+emit_func (FILE *f, bool open, bool close, const char *format, ...)
+{
+  gcc_assert (header_file);
+  va_list ap1, ap2;
+  if (open)
+    fprintf (header_file, "extern ");
+  va_start (ap1, format);
+  va_start (ap2, format);
+  vfprintf (f, format, ap1);
+  vfprintf (header_file, format, ap2);
+  va_end (ap1);
+  va_end (ap2);
+  if (close)
+    fprintf (header_file, ";\n");
+  fputc ('\n', f);
+}
+
 static void
 output_line_directive (FILE *f, location_t location,
 		       bool dumpfile = false, bool fnargs = false,
@@ -220,6 +244,34 @@ output_line_directive (FILE *f, location_t location,
     fprintf (f, "/* #line %d \"%s\" */\n", loc.line, loc.file);
 }
 
+/* Find the file to write into next.  We try to evenly distribute the contents
+   over the different files.  */
+
+#define SIZED_BASED_CHUNKS 1
+
+int current_file = 0;
+FILE *get_out_file (FILE **parts, int n)
+{
+#ifdef SIZED_BASED_CHUNKS
+   FILE *f = NULL;
+   long min = 0;
+   /* We've started writing all the files at pos 0, so ftell is equivalent
+      to the size and should be much faster.  */
+   for (int i = 0; i < n; i++)
+     {
+	long res = ftell (parts[i]);
+	if (!f || res < min)
+	  {
+	    min = res;
+	    f = parts[i];
+	  }
+     }
+  return f;
+#else
+  return parts[current_file++ % n];
+#endif
+}
+
 
 /* Pull in tree codes and builtin function codes from their
    definition files.  */
@@ -1735,7 +1787,7 @@ public:
   dt_node *root;
 
   void insert (class simplify *, unsigned);
-  void gen (FILE *f, bool gimple);
+  void gen (FILE **f, int n, bool gimple);
   void print (FILE *f = stderr);
 
   decision_tree () { root = new dt_node (dt_node::DT_NODE, NULL); }
@@ -3830,7 +3882,7 @@ sinfo_hashmap_traits::equal_keys (const key_type &v,
    tree.  */
 
 void
-decision_tree::gen (FILE *f, bool gimple)
+decision_tree::gen (FILE **files, int n_parts, bool gimple)
 {
   sinfo_map_t si;
 
@@ -3859,11 +3911,14 @@ decision_tree::gen (FILE *f, bool gimple)
 	  output_line_directive (stderr, s->s->s->result->location);
 	}
 
+      /* Cycle the file buffers.  */
+      FILE *f = get_out_file (files, n_parts);
+
       /* Generate a split out function with the leaf transform code.  */
       s->fname = xasprintf ("%s_simplify_%u", gimple ? "gimple" : "generic",
 			    fcnt++);
       if (gimple)
-	fprintf (f, "\nstatic bool\n"
+	emit_func (f, true, false, "\nbool\n"
 		 "%s (gimple_match_op *res_op, gimple_seq *seq,\n"
 		 "                 tree (*valueize)(tree) ATTRIBUTE_UNUSED,\n"
 		 "                 const tree ARG_UNUSED (type), tree *ARG_UNUSED "
@@ -3871,27 +3926,28 @@ decision_tree::gen (FILE *f, bool gimple)
 		 s->fname);
       else
 	{
-	  fprintf (f, "\nstatic tree\n"
+	  emit_func (f, true, false, "\ntree\n"
 		   "%s (location_t ARG_UNUSED (loc), const tree ARG_UNUSED (type),\n",
 		   (*iter).second->fname);
 	  for (unsigned i = 0;
 	       i < as_a <expr *>(s->s->s->match)->ops.length (); ++i)
-	    fprintf (f, " tree ARG_UNUSED (_p%d),", i);
-	  fprintf (f, " tree *captures\n");
+	    emit_func (f, false, false, " tree ARG_UNUSED (_p%d),", i);
+	  emit_func (f, false, false, " tree *captures\n");
 	}
       for (unsigned i = 0; i < s->s->s->for_subst_vec.length (); ++i)
 	{
 	  if (! s->s->s->for_subst_vec[i].first->used)
 	    continue;
 	  if (is_a <operator_id *> (s->s->s->for_subst_vec[i].second))
-	    fprintf (f, ", const enum tree_code ARG_UNUSED (%s)",
+	    emit_func (f, false, false, ", const enum tree_code ARG_UNUSED (%s)",
 		     s->s->s->for_subst_vec[i].first->id);
 	  else if (is_a <fn_id *> (s->s->s->for_subst_vec[i].second))
-	    fprintf (f, ", const combined_fn ARG_UNUSED (%s)",
+	    emit_func (f, false, false, ", const combined_fn ARG_UNUSED (%s)",
 		     s->s->s->for_subst_vec[i].first->id);
 	}
 
-      fprintf (f, ")\n{\n");
+      emit_func (f, false, true, ")");
+      fprintf (f, "{\n");
       s->s->gen_1 (f, 2, gimple, s->s->s->result);
       if (gimple)
 	fprintf (f, "  return false;\n");
@@ -3919,8 +3975,12 @@ decision_tree::gen (FILE *f, bool gimple)
 		  && e->operation->kind != id_base::CODE))
 	    continue;
 
+
+	  /* Cycle the file buffers.  */
+	  FILE *f = get_out_file (files, n_parts);
+
 	  if (gimple)
-	    fprintf (f, "\nstatic bool\n"
+	    emit_func (f, true, false,"\nbool\n"
 		     "gimple_simplify_%s (gimple_match_op *res_op,"
 		     " gimple_seq *seq,\n"
 		     "                 tree (*valueize)(tree) "
@@ -3929,13 +3989,13 @@ decision_tree::gen (FILE *f, bool gimple)
 		     "ARG_UNUSED (type)\n",
 		     e->operation->id);
 	  else
-	    fprintf (f, "\nstatic tree\n"
+	    emit_func (f, true, false, "\ntree\n"
 		     "generic_simplify_%s (location_t ARG_UNUSED (loc), enum "
 		     "tree_code ARG_UNUSED (code), const tree ARG_UNUSED (type)",
 		     e->operation->id);
 	  for (unsigned i = 0; i < n; ++i)
-	    fprintf (f, ", tree _p%d", i);
-	  fprintf (f, ")\n");
+	    emit_func (f, false, false,", tree _p%d", i);
+	  emit_func (f, false, true, ")");
 	  fprintf (f, "{\n");
 	  dop->gen_kids (f, 2, gimple, 0);
 	  if (gimple)
@@ -3950,18 +4010,22 @@ decision_tree::gen (FILE *f, bool gimple)
 	 with compiler warnings, by generating a simple stub.  */
       if (! has_kids_p)
 	{
+
+	  /* Cycle the file buffers.  */
+	  FILE *f = get_out_file (files, n_parts);
+
 	  if (gimple)
-	    fprintf (f, "\nstatic bool\n"
+	    emit_func (f, true, false, "\nbool\n"
 			"gimple_simplify (gimple_match_op*, gimple_seq*,\n"
 			"                 tree (*)(tree), code_helper,\n"
 			"                 const tree");
 	  else
-	    fprintf (f, "\ntree\n"
+	    emit_func (f, true, false,"\ntree\n"
 			"generic_simplify (location_t, enum tree_code,\n"
 			"                  const tree");
 	  for (unsigned i = 0; i < n; ++i)
-	    fprintf (f, ", tree");
-	  fprintf (f, ")\n");
+	    emit_func (f, false, false, ", tree");
+	  emit_func (f, false, true, ")");
 	  fprintf (f, "{\n");
 	  if (gimple)
 	    fprintf (f, "  return false;\n");
@@ -3971,20 +4035,24 @@ decision_tree::gen (FILE *f, bool gimple)
 	  continue;
 	}
 
+
+      /* Cycle the file buffers.  */
+      FILE *f = get_out_file (files, n_parts);
+
       /* Then generate the main entry with the outermost switch and
          tail-calls to the split-out functions.  */
       if (gimple)
-	fprintf (f, "\nstatic bool\n"
+	emit_func (f, true, false, "\nbool\n"
 		 "gimple_simplify (gimple_match_op *res_op, gimple_seq *seq,\n"
 		 "                 tree (*valueize)(tree) ATTRIBUTE_UNUSED,\n"
 		 "                 code_helper code, const tree type");
       else
-	fprintf (f, "\ntree\n"
+	emit_func (f, true, false, "\ntree\n"
 		 "generic_simplify (location_t loc, enum tree_code code, "
 		 "const tree type ATTRIBUTE_UNUSED");
       for (unsigned i = 0; i < n; ++i)
-	fprintf (f, ", tree _p%d", i);
-      fprintf (f, ")\n");
+	emit_func (f, false, false, ", tree _p%d", i);
+      emit_func (f, false, true, ")");
       fprintf (f, "{\n");
 
       if (gimple)
@@ -4039,11 +4107,11 @@ decision_tree::gen (FILE *f, bool gimple)
 void
 write_predicate (FILE *f, predicate_id *p, decision_tree &dt, bool gimple)
 {
-  fprintf (f, "\nbool\n"
-	   "%s%s (tree t%s%s)\n"
-	   "{\n", gimple ? "gimple_" : "tree_", p->id,
-	   p->nargs > 0 ? ", tree *res_ops" : "",
-	   gimple ? ", tree (*valueize)(tree) ATTRIBUTE_UNUSED" : "");
+  emit_func (f, true, true, "\nbool\n%s%s (tree t%s%s)",
+		gimple ? "gimple_" : "tree_", p->id,
+		p->nargs > 0 ? ", tree *res_ops" : "",
+		gimple ? ", tree (*valueize)(tree) ATTRIBUTE_UNUSED" : "");
+  fprintf (f, "{\n");
   /* Conveniently make 'type' available.  */
   fprintf_indent (f, 2, "const tree type = TREE_TYPE (t);\n");
 
@@ -4062,6 +4130,9 @@ write_header (FILE *f, const char *head)
 {
   fprintf (f, "/* Generated automatically by the program `genmatch' from\n");
   fprintf (f, "   a IL pattern matching and simplification description.  */\n");
+  fprintf (f, "#pragma GCC diagnostic push\n");
+  fprintf (f, "#pragma GCC diagnostic ignored \"-Wunused-variable\"\n");
+  fprintf (f, "#pragma GCC diagnostic ignored \"-Wunused-function\"\n");
 
   /* Include the header instead of writing it awkwardly quoted here.  */
   fprintf (f, "\n#include \"%s\"\n", head);
@@ -5221,13 +5292,20 @@ main (int argc, char **argv)
     return 1;
 
   bool gimple = true;
-  char *input = argv[argc-1];
-  for (int i = 1; i < argc - 1; ++i)
+  /* Set a default value for the tool to 5, but GCC itself uses
+     whatever default is determined by the configure variable
+     DEFAULT_MATCHPD_PARTITIONS.  */
+  int n_parts = 5;
+  char *input = argv[argc-2];
+  char *outdir = argv[argc-1];
+  for (int i = 1; i < argc - 2; ++i)
     {
       if (strcmp (argv[i], "--gimple") == 0)
 	gimple = true;
       else if (strcmp (argv[i], "--generic") == 0)
 	gimple = false;
+      else if (strncmp (argv[i], "--splits=", 9) == 0)
+	sscanf (argv[i], "--splits=%d", &n_parts);
       else if (strcmp (argv[i], "-v") == 0)
 	verbose = 1;
       else if (strcmp (argv[i], "-vv") == 0)
@@ -5235,7 +5313,7 @@ main (int argc, char **argv)
       else
 	{
 	  fprintf (stderr, "Usage: genmatch "
-		   "[--gimple] [--generic] [-v[v]] input\n");
+		   "[--gimple] [--generic] [--splits=<n>] [-v[v]] input outdir\n");
 	  return 1;
 	}
     }
@@ -5286,10 +5364,33 @@ main (int argc, char **argv)
   /* Parse ahead!  */
   parser p (r, gimple);
 
-  if (gimple)
-    write_header (stdout, "gimple-match-head.cc");
-  else
-    write_header (stdout, "generic-match-head.cc");
+  /* Create file buffers.  */
+  char *s_file = XNEWVEC (char, strlen(outdir) + 40);
+  FILE **parts = XNEWVEC (FILE*, n_parts);
+  for (int i = 0; i < n_parts; i++)
+    {
+      sprintf(s_file,
+	      gimple ? "%sgimple-match-%d.cc" : "%sgeneric-match-%d.cc",
+	      outdir, i+1); /* Start numbering at 1 to make Makefile simple.  */
+      parts[i] = fopen (s_file, "w");
+
+      if (gimple)
+	write_header (parts[i], "gimple-match-auto.h");
+      else
+	write_header (parts[i], "generic-match-auto.h");
+    }
+
+    sprintf(s_file,
+	    gimple ? "%sgimple-match-auto.h" : "%sgeneric-match-auto.h",
+	    outdir);
+    header_file = fopen (s_file, "w");
+    fprintf (header_file,
+	     "#ifndef GCC_GIMPLE_MATCH_AUTO_H\n#define GCC_GIMPLE_MATCH_AUTO_H\n");
+    if (gimple)
+       fprintf (header_file, "#include \"gimple-match-head.cc\"\n");
+    else
+       fprintf (header_file, "#include \"generic-match-head.cc\"\n");
+  XDELETEVEC (s_file);
 
   /* Go over all predicates defined with patterns and perform
      lowering and code generation.  */
@@ -5309,7 +5410,10 @@ main (int argc, char **argv)
       if (verbose == 2)
 	dt.print (stderr);
 
-      write_predicate (stdout, pred, dt, gimple);
+      /* Cycle the file buffers.  */
+      FILE *f = get_out_file (parts, n_parts);
+
+      write_predicate (f, pred, dt, gimple);
     }
 
   /* Lower the main simplifiers and generate code for them.  */
@@ -5326,7 +5430,18 @@ main (int argc, char **argv)
   if (verbose == 2)
     dt.print (stderr);
 
-  dt.gen (stdout, gimple);
+  dt.gen (parts, n_parts, gimple);
+
+  for (int i = 0; i < n_parts; i++)
+    {
+      fprintf (parts[i], "#pragma GCC diagnostic pop\n");
+      fclose (parts[i]);
+    }
+  XDELETEVEC (parts);
+
+  fprintf (header_file,
+	   "#endif /* GCC_GIMPLE_MATCH_AUTO_H.  */\n");
+  fclose (header_file);
 
   /* Finalize.  */
   cpp_finish (r, NULL);
diff --git a/gcc/gimple-match-exports.cc b/gcc/gimple-match-exports.cc
new file mode 100644
index 0000000000000000000000000000000000000000..3b9b6e7fc18555ebcb6e1f9213faa7a65212df8c
--- /dev/null
+++ b/gcc/gimple-match-exports.cc
@@ -0,0 +1,1223 @@
+/* Helpers for the autogenerated gimple-match.cc file.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "backend.h"
+#include "target.h"
+#include "rtl.h"
+#include "tree.h"
+#include "gimple.h"
+#include "ssa.h"
+#include "cgraph.h"
+#include "vec-perm-indices.h"
+#include "fold-const.h"
+#include "fold-const-call.h"
+#include "stor-layout.h"
+#include "gimple-iterator.h"
+#include "gimple-fold.h"
+#include "calls.h"
+#include "tree-dfa.h"
+#include "builtins.h"
+#include "gimple-match.h"
+#include "tree-pass.h"
+#include "internal-fn.h"
+#include "case-cfn-macros.h"
+#include "gimplify.h"
+#include "optabs-tree.h"
+#include "tree-eh.h"
+#include "dbgcnt.h"
+#include "tm.h"
+#include "gimple-range.h"
+#include "langhooks.h"
+
+tree (*mprts_hook) (gimple_match_op *);
+
+/* Forward declarations of the private auto-generated matchers.
+   They expect valueized operands in canonical order and do not
+   perform simplification of all-constant operands.  */
+
+static bool gimple_resimplify1 (gimple_seq *, gimple_match_op *, tree (*)(tree));
+static bool gimple_resimplify2 (gimple_seq *, gimple_match_op *, tree (*)(tree));
+static bool gimple_resimplify3 (gimple_seq *, gimple_match_op *, tree (*)(tree));
+static bool gimple_resimplify4 (gimple_seq *, gimple_match_op *, tree (*)(tree));
+static bool gimple_resimplify5 (gimple_seq *, gimple_match_op *, tree (*)(tree));
+
+/* Match and simplify the toplevel valueized operation THIS.
+   Replaces THIS with a simplified and/or canonicalized result and
+   returns whether any change was made.  */
+
+bool
+gimple_match_op::resimplify (gimple_seq *seq, tree (*valueize)(tree))
+{
+  switch (num_ops)
+    {
+    case 1:
+      return gimple_resimplify1 (seq, this, valueize);
+    case 2:
+      return gimple_resimplify2 (seq, this, valueize);
+    case 3:
+      return gimple_resimplify3 (seq, this, valueize);
+    case 4:
+      return gimple_resimplify4 (seq, this, valueize);
+    case 5:
+      return gimple_resimplify5 (seq, this, valueize);
+    default:
+      gcc_unreachable ();
+    }
+}
+
+/* Return whether T is a constant that we'll dispatch to fold to
+   evaluate fully constant expressions.  */
+
+static inline bool
+constant_for_folding (tree t)
+{
+  return (CONSTANT_CLASS_P (t)
+	  /* The following is only interesting to string builtins.  */
+	  || (TREE_CODE (t) == ADDR_EXPR
+	      && TREE_CODE (TREE_OPERAND (t, 0)) == STRING_CST));
+}
+
+/* Try to convert conditional operation ORIG_OP into an IFN_COND_*
+   operation.  Return true on success, storing the new operation in NEW_OP.  */
+
+static bool
+convert_conditional_op (gimple_match_op *orig_op,
+			gimple_match_op *new_op)
+{
+  internal_fn ifn;
+  if (orig_op->code.is_tree_code ())
+    ifn = get_conditional_internal_fn ((tree_code) orig_op->code);
+  else
+    {
+      auto cfn = combined_fn (orig_op->code);
+      if (!internal_fn_p (cfn))
+	return false;
+      ifn = get_conditional_internal_fn (as_internal_fn (cfn));
+    }
+  if (ifn == IFN_LAST)
+    return false;
+  unsigned int num_ops = orig_op->num_ops;
+  new_op->set_op (as_combined_fn (ifn), orig_op->type, num_ops + 2);
+  new_op->ops[0] = orig_op->cond.cond;
+  for (unsigned int i = 0; i < num_ops; ++i)
+    new_op->ops[i + 1] = orig_op->ops[i];
+  tree else_value = orig_op->cond.else_value;
+  if (!else_value)
+    else_value = targetm.preferred_else_value (ifn, orig_op->type,
+					       num_ops, orig_op->ops);
+  new_op->ops[num_ops + 1] = else_value;
+  return true;
+}
+/* Helper for gimple_simplify valueizing OP using VALUEIZE and setting
+   VALUEIZED to true if valueization changed OP.  */
+
+inline tree
+do_valueize (tree op, tree (*valueize)(tree), bool &valueized)
+{
+  if (valueize && TREE_CODE (op) == SSA_NAME)
+    {
+      tree tem = valueize (op);
+      if (tem && tem != op)
+	{
+	  op = tem;
+	  valueized = true;
+	}
+    }
+  return op;
+}
+
+/* If in GIMPLE the operation described by RES_OP should be single-rhs,
+   build a GENERIC tree for that expression and update RES_OP accordingly.  */
+
+void
+maybe_build_generic_op (gimple_match_op *res_op)
+{
+  tree_code code = (tree_code) res_op->code;
+  tree val;
+  switch (code)
+    {
+    case REALPART_EXPR:
+    case IMAGPART_EXPR:
+    case VIEW_CONVERT_EXPR:
+      val = build1 (code, res_op->type, res_op->ops[0]);
+      res_op->set_value (val);
+      break;
+    case BIT_FIELD_REF:
+      val = build3 (code, res_op->type, res_op->ops[0], res_op->ops[1],
+		    res_op->ops[2]);
+      REF_REVERSE_STORAGE_ORDER (val) = res_op->reverse;
+      res_op->set_value (val);
+      break;
+    default:;
+    }
+}
+
+/* Try to build RES_OP, which is known to be a call to FN.  Return null
+   if the target doesn't support the function.  */
+
+static gcall *
+build_call_internal (internal_fn fn, gimple_match_op *res_op)
+{
+  if (direct_internal_fn_p (fn))
+    {
+      tree_pair types = direct_internal_fn_types (fn, res_op->type,
+						  res_op->ops);
+      if (!direct_internal_fn_supported_p (fn, types, OPTIMIZE_FOR_BOTH))
+	return NULL;
+    }
+  return gimple_build_call_internal (fn, res_op->num_ops,
+				     res_op->op_or_null (0),
+				     res_op->op_or_null (1),
+				     res_op->op_or_null (2),
+				     res_op->op_or_null (3),
+				     res_op->op_or_null (4));
+}
+
+/* RES_OP is the result of a simplification.  If it is conditional,
+   try to replace it with the equivalent UNCOND form, such as an
+   IFN_COND_* call or a VEC_COND_EXPR.  Also try to resimplify the
+   result of the replacement if appropriate, adding any new statements to
+   SEQ and using VALUEIZE as the valueization function.  Return true if
+   this resimplification occurred and resulted in at least one change.  */
+
+static bool
+maybe_resimplify_conditional_op (gimple_seq *seq, gimple_match_op *res_op,
+				 tree (*valueize) (tree))
+{
+  if (!res_op->cond.cond)
+    return false;
+
+  if (!res_op->cond.else_value
+      && res_op->code.is_tree_code ())
+    {
+      /* The "else" value doesn't matter.  If the "then" value is a
+	 gimple value, just use it unconditionally.  This isn't a
+	 simplification in itself, since there was no operation to
+	 build in the first place.  */
+      if (gimple_simplified_result_is_gimple_val (res_op))
+	{
+	  res_op->cond.cond = NULL_TREE;
+	  return false;
+	}
+
+      /* Likewise if the operation would not trap.  */
+      bool honor_trapv = (INTEGRAL_TYPE_P (res_op->type)
+			  && TYPE_OVERFLOW_TRAPS (res_op->type));
+      tree_code op_code = (tree_code) res_op->code;
+      bool op_could_trap;
+
+      /* COND_EXPR will trap if, and only if, the condition
+	 traps and hence we have to check this.  For all other operations, we
+	 don't need to consider the operands.  */
+      if (op_code == COND_EXPR)
+	op_could_trap = generic_expr_could_trap_p (res_op->ops[0]);
+      else
+	op_could_trap = operation_could_trap_p ((tree_code) res_op->code,
+						FLOAT_TYPE_P (res_op->type),
+						honor_trapv,
+						res_op->op_or_null (1));
+
+      if (!op_could_trap)
+	{
+	  res_op->cond.cond = NULL_TREE;
+	  return false;
+	}
+    }
+
+  /* If the "then" value is a gimple value and the "else" value matters,
+     create a VEC_COND_EXPR between them, then see if it can be further
+     simplified.  */
+  gimple_match_op new_op;
+  if (res_op->cond.else_value
+      && VECTOR_TYPE_P (res_op->type)
+      && gimple_simplified_result_is_gimple_val (res_op))
+    {
+      new_op.set_op (VEC_COND_EXPR, res_op->type,
+		     res_op->cond.cond, res_op->ops[0],
+		     res_op->cond.else_value);
+      *res_op = new_op;
+      return gimple_resimplify3 (seq, res_op, valueize);
+    }
+
+  /* Otherwise try rewriting the operation as an IFN_COND_* call.
+     Again, this isn't a simplification in itself, since it's what
+     RES_OP already described.  */
+  if (convert_conditional_op (res_op, &new_op))
+    *res_op = new_op;
+
+  return false;
+}
+
+/* If RES_OP is a call to a conditional internal function, try simplifying
+   the associated unconditional operation and using the result to build
+   a new conditional operation.  For example, if RES_OP is:
+
+     IFN_COND_ADD (COND, A, B, ELSE)
+
+   try simplifying (plus A B) and using the result to build a replacement
+   for the whole IFN_COND_ADD.
+
+   Return true if this approach led to a simplification, otherwise leave
+   RES_OP unchanged (and so suitable for other simplifications).  When
+   returning true, add any new statements to SEQ and use VALUEIZE as the
+   valueization function.
+
+   RES_OP is known to be a call to IFN.  */
+
+static bool
+try_conditional_simplification (internal_fn ifn, gimple_match_op *res_op,
+				gimple_seq *seq, tree (*valueize) (tree))
+{
+  code_helper op;
+  tree_code code = conditional_internal_fn_code (ifn);
+  if (code != ERROR_MARK)
+    op = code;
+  else
+    {
+      ifn = get_unconditional_internal_fn (ifn);
+      if (ifn == IFN_LAST)
+	return false;
+      op = as_combined_fn (ifn);
+    }
+
+  unsigned int num_ops = res_op->num_ops;
+  gimple_match_op cond_op (gimple_match_cond (res_op->ops[0],
+					      res_op->ops[num_ops - 1]),
+			   op, res_op->type, num_ops - 2);
+
+  memcpy (cond_op.ops, res_op->ops + 1, (num_ops - 1) * sizeof *cond_op.ops);
+  switch (num_ops - 2)
+    {
+    case 1:
+      if (!gimple_resimplify1 (seq, &cond_op, valueize))
+	return false;
+      break;
+    case 2:
+      if (!gimple_resimplify2 (seq, &cond_op, valueize))
+	return false;
+      break;
+    case 3:
+      if (!gimple_resimplify3 (seq, &cond_op, valueize))
+	return false;
+      break;
+    default:
+      gcc_unreachable ();
+    }
+  *res_op = cond_op;
+  maybe_resimplify_conditional_op (seq, res_op, valueize);
+  return true;
+}
+
+/* Helper for the autogenerated code, valueize OP.  */
+
+tree
+do_valueize (tree (*valueize)(tree), tree op)
+{
+  if (valueize && TREE_CODE (op) == SSA_NAME)
+    {
+      tree tem = valueize (op);
+      if (tem)
+	return tem;
+    }
+  return op;
+}
+
+/* Push the exploded expression described by RES_OP as a statement to
+   SEQ if necessary and return a gimple value denoting the value of the
+   expression.  If RES is not NULL then the result will be always RES
+   and even gimple values are pushed to SEQ.  */
+
+tree
+maybe_push_res_to_seq (gimple_match_op *res_op, gimple_seq *seq, tree res)
+{
+  tree *ops = res_op->ops;
+  unsigned num_ops = res_op->num_ops;
+
+  /* The caller should have converted conditional operations into an UNCOND
+     form and resimplified as appropriate.  The conditional form only
+     survives this far if that conversion failed.  */
+  if (res_op->cond.cond)
+    return NULL_TREE;
+
+  if (res_op->code.is_tree_code ())
+    {
+      if (!res
+	  && gimple_simplified_result_is_gimple_val (res_op))
+	return ops[0];
+      if (mprts_hook)
+	{
+	  tree tem = mprts_hook (res_op);
+	  if (tem)
+	    return tem;
+	}
+    }
+
+  if (!seq)
+    return NULL_TREE;
+
+  /* Play safe and do not allow abnormals to be mentioned in
+     newly created statements.  */
+  for (unsigned int i = 0; i < num_ops; ++i)
+    if (TREE_CODE (ops[i]) == SSA_NAME
+	&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ops[i]))
+      return NULL_TREE;
+
+  if (num_ops > 0 && COMPARISON_CLASS_P (ops[0]))
+    for (unsigned int i = 0; i < 2; ++i)
+      if (TREE_CODE (TREE_OPERAND (ops[0], i)) == SSA_NAME
+	  && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (ops[0], i)))
+	return NULL_TREE;
+
+  if (res_op->code.is_tree_code ())
+    {
+      auto code = tree_code (res_op->code);
+      if (!res)
+	{
+	  if (gimple_in_ssa_p (cfun))
+	    res = make_ssa_name (res_op->type);
+	  else
+	    res = create_tmp_reg (res_op->type);
+	}
+      maybe_build_generic_op (res_op);
+      gimple *new_stmt = gimple_build_assign (res, code,
+					      res_op->op_or_null (0),
+					      res_op->op_or_null (1),
+					      res_op->op_or_null (2));
+      gimple_seq_add_stmt_without_update (seq, new_stmt);
+      return res;
+    }
+  else
+    {
+      gcc_assert (num_ops != 0);
+      auto fn = combined_fn (res_op->code);
+      gcall *new_stmt = NULL;
+      if (internal_fn_p (fn))
+	{
+	  /* Generate the given function if we can.  */
+	  internal_fn ifn = as_internal_fn (fn);
+	  new_stmt = build_call_internal (ifn, res_op);
+	  if (!new_stmt)
+	    return NULL_TREE;
+	}
+      else
+	{
+	  /* Find the function we want to call.  */
+	  tree decl = builtin_decl_implicit (as_builtin_fn (fn));
+	  if (!decl)
+	    return NULL;
+
+	  /* We can't and should not emit calls to non-const functions.  */
+	  if (!(flags_from_decl_or_type (decl) & ECF_CONST))
+	    return NULL;
+
+	  new_stmt = gimple_build_call (decl, num_ops,
+					res_op->op_or_null (0),
+					res_op->op_or_null (1),
+					res_op->op_or_null (2),
+					res_op->op_or_null (3),
+					res_op->op_or_null (4));
+	}
+      if (!res)
+	{
+	  if (gimple_in_ssa_p (cfun))
+	    res = make_ssa_name (res_op->type);
+	  else
+	    res = create_tmp_reg (res_op->type);
+	}
+      gimple_call_set_lhs (new_stmt, res);
+      gimple_seq_add_stmt_without_update (seq, new_stmt);
+      return res;
+    }
+}
+
+
+/* Public API overloads follow for operation being tree_code or
+   built_in_function and for one to three operands or arguments.
+   They return NULL_TREE if nothing could be simplified or
+   the resulting simplified value with parts pushed to SEQ.
+   If SEQ is NULL then if the simplification needs to create
+   new stmts it will fail.  If VALUEIZE is non-NULL then all
+   SSA names will be valueized using that hook prior to
+   applying simplifications.  */
+
+/* Unary ops.  */
+
+tree
+gimple_simplify (enum tree_code code, tree type,
+		 tree op0,
+		 gimple_seq *seq, tree (*valueize)(tree))
+{
+  if (constant_for_folding (op0))
+    {
+      tree res = const_unop (code, type, op0);
+      if (res != NULL_TREE
+	  && CONSTANT_CLASS_P (res))
+	return res;
+    }
+
+  gimple_match_op res_op;
+  if (!gimple_simplify (&res_op, seq, valueize, code, type, op0))
+    return NULL_TREE;
+  return maybe_push_res_to_seq (&res_op, seq);
+}
+
+/* Binary ops.  */
+
+tree
+gimple_simplify (enum tree_code code, tree type,
+		 tree op0, tree op1,
+		 gimple_seq *seq, tree (*valueize)(tree))
+{
+  if (constant_for_folding (op0) && constant_for_folding (op1))
+    {
+      tree res = const_binop (code, type, op0, op1);
+      if (res != NULL_TREE
+	  && CONSTANT_CLASS_P (res))
+	return res;
+    }
+
+  /* Canonicalize operand order both for matching and fallback stmt
+     generation.  */
+  if ((commutative_tree_code (code)
+       || TREE_CODE_CLASS (code) == tcc_comparison)
+      && tree_swap_operands_p (op0, op1))
+    {
+      std::swap (op0, op1);
+      if (TREE_CODE_CLASS (code) == tcc_comparison)
+	code = swap_tree_comparison (code);
+    }
+
+  gimple_match_op res_op;
+  if (!gimple_simplify (&res_op, seq, valueize, code, type, op0, op1))
+    return NULL_TREE;
+  return maybe_push_res_to_seq (&res_op, seq);
+}
+
+/* Ternary ops.  */
+
+tree
+gimple_simplify (enum tree_code code, tree type,
+		 tree op0, tree op1, tree op2,
+		 gimple_seq *seq, tree (*valueize)(tree))
+{
+  if (constant_for_folding (op0) && constant_for_folding (op1)
+      && constant_for_folding (op2))
+    {
+      tree res = fold_ternary/*_to_constant */ (code, type, op0, op1, op2);
+      if (res != NULL_TREE
+	  && CONSTANT_CLASS_P (res))
+	return res;
+    }
+
+  /* Canonicalize operand order both for matching and fallback stmt
+     generation.  */
+  if (commutative_ternary_tree_code (code)
+      && tree_swap_operands_p (op0, op1))
+    std::swap (op0, op1);
+
+  gimple_match_op res_op;
+  if (!gimple_simplify (&res_op, seq, valueize, code, type, op0, op1, op2))
+    return NULL_TREE;
+  return maybe_push_res_to_seq (&res_op, seq);
+}
+
+/* Builtin or internal function with one argument.  */
+
+tree
+gimple_simplify (combined_fn fn, tree type,
+		 tree arg0,
+		 gimple_seq *seq, tree (*valueize)(tree))
+{
+  if (constant_for_folding (arg0))
+    {
+      tree res = fold_const_call (fn, type, arg0);
+      if (res && CONSTANT_CLASS_P (res))
+	return res;
+    }
+
+  gimple_match_op res_op;
+  if (!gimple_simplify (&res_op, seq, valueize, fn, type, arg0))
+    return NULL_TREE;
+  return maybe_push_res_to_seq (&res_op, seq);
+}
+
+/* Builtin or internal function with two arguments.  */
+
+tree
+gimple_simplify (combined_fn fn, tree type,
+		 tree arg0, tree arg1,
+		 gimple_seq *seq, tree (*valueize)(tree))
+{
+  if (constant_for_folding (arg0)
+      && constant_for_folding (arg1))
+    {
+      tree res = fold_const_call (fn, type, arg0, arg1);
+      if (res && CONSTANT_CLASS_P (res))
+	return res;
+    }
+
+  gimple_match_op res_op;
+  if (!gimple_simplify (&res_op, seq, valueize, fn, type, arg0, arg1))
+    return NULL_TREE;
+  return maybe_push_res_to_seq (&res_op, seq);
+}
+
+/* Builtin or internal function with three arguments.  */
+
+tree
+gimple_simplify (combined_fn fn, tree type,
+		 tree arg0, tree arg1, tree arg2,
+		 gimple_seq *seq, tree (*valueize)(tree))
+{
+  if (constant_for_folding (arg0)
+      && constant_for_folding (arg1)
+      && constant_for_folding (arg2))
+    {
+      tree res = fold_const_call (fn, type, arg0, arg1, arg2);
+      if (res && CONSTANT_CLASS_P (res))
+	return res;
+    }
+
+  gimple_match_op res_op;
+  if (!gimple_simplify (&res_op, seq, valueize, fn, type, arg0, arg1, arg2))
+    return NULL_TREE;
+  return maybe_push_res_to_seq (&res_op, seq);
+}
+
+/* Common subroutine of gimple_extract_op and gimple_simplify.  Try to
+   describe STMT in RES_OP, returning true on success.  Before recording
+   an operand, call:
+
+   - VALUEIZE_CONDITION for a COND_EXPR condition
+   - VALUEIZE_OP for every other top-level operand
+
+   Both routines take a tree argument and returns a tree.  */
+
+template<typename ValueizeOp, typename ValueizeCondition>
+inline bool
+gimple_extract (gimple *stmt, gimple_match_op *res_op,
+		ValueizeOp valueize_op,
+		ValueizeCondition valueize_condition)
+{
+  switch (gimple_code (stmt))
+    {
+    case GIMPLE_ASSIGN:
+      {
+	enum tree_code code = gimple_assign_rhs_code (stmt);
+	tree type = TREE_TYPE (gimple_assign_lhs (stmt));
+	switch (gimple_assign_rhs_class (stmt))
+	  {
+	  case GIMPLE_SINGLE_RHS:
+	    if (code == REALPART_EXPR
+		|| code == IMAGPART_EXPR
+		|| code == VIEW_CONVERT_EXPR)
+	      {
+		tree op0 = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
+		res_op->set_op (code, type, valueize_op (op0));
+		return true;
+	      }
+	    else if (code == BIT_FIELD_REF)
+	      {
+		tree rhs1 = gimple_assign_rhs1 (stmt);
+		tree op0 = valueize_op (TREE_OPERAND (rhs1, 0));
+		res_op->set_op (code, type, op0,
+				TREE_OPERAND (rhs1, 1),
+				TREE_OPERAND (rhs1, 2),
+				REF_REVERSE_STORAGE_ORDER (rhs1));
+		return true;
+	      }
+	    else if (code == SSA_NAME)
+	      {
+		tree op0 = gimple_assign_rhs1 (stmt);
+		res_op->set_op (TREE_CODE (op0), type, valueize_op (op0));
+		return true;
+	      }
+	    break;
+	  case GIMPLE_UNARY_RHS:
+	    {
+	      tree rhs1 = gimple_assign_rhs1 (stmt);
+	      res_op->set_op (code, type, valueize_op (rhs1));
+	      return true;
+	    }
+	  case GIMPLE_BINARY_RHS:
+	    {
+	      tree rhs1 = valueize_op (gimple_assign_rhs1 (stmt));
+	      tree rhs2 = valueize_op (gimple_assign_rhs2 (stmt));
+	      res_op->set_op (code, type, rhs1, rhs2);
+	      return true;
+	    }
+	  case GIMPLE_TERNARY_RHS:
+	    {
+	      tree rhs1 = gimple_assign_rhs1 (stmt);
+	      if (code == COND_EXPR && COMPARISON_CLASS_P (rhs1))
+		rhs1 = valueize_condition (rhs1);
+	      else
+		rhs1 = valueize_op (rhs1);
+	      tree rhs2 = valueize_op (gimple_assign_rhs2 (stmt));
+	      tree rhs3 = valueize_op (gimple_assign_rhs3 (stmt));
+	      res_op->set_op (code, type, rhs1, rhs2, rhs3);
+	      return true;
+	    }
+	  default:
+	    gcc_unreachable ();
+	  }
+	break;
+      }
+
+    case GIMPLE_CALL:
+      /* ???  This way we can't simplify calls with side-effects.  */
+      if (gimple_call_lhs (stmt) != NULL_TREE
+	  && gimple_call_num_args (stmt) >= 1
+	  && gimple_call_num_args (stmt) <= 5)
+	{
+	  combined_fn cfn;
+	  if (gimple_call_internal_p (stmt))
+	    cfn = as_combined_fn (gimple_call_internal_fn (stmt));
+	  else
+	    {
+	      tree fn = gimple_call_fn (stmt);
+	      if (!fn)
+		return false;
+
+	      fn = valueize_op (fn);
+	      if (TREE_CODE (fn) != ADDR_EXPR
+		  || TREE_CODE (TREE_OPERAND (fn, 0)) != FUNCTION_DECL)
+		return false;
+
+	      tree decl = TREE_OPERAND (fn, 0);
+	      if (DECL_BUILT_IN_CLASS (decl) != BUILT_IN_NORMAL
+		  || !gimple_builtin_call_types_compatible_p (stmt, decl))
+		return false;
+
+	      cfn = as_combined_fn (DECL_FUNCTION_CODE (decl));
+	    }
+
+	  unsigned int num_args = gimple_call_num_args (stmt);
+	  res_op->set_op (cfn, TREE_TYPE (gimple_call_lhs (stmt)), num_args);
+	  for (unsigned i = 0; i < num_args; ++i)
+	    res_op->ops[i] = valueize_op (gimple_call_arg (stmt, i));
+	  return true;
+	}
+      break;
+
+    case GIMPLE_COND:
+      {
+	tree lhs = valueize_op (gimple_cond_lhs (stmt));
+	tree rhs = valueize_op (gimple_cond_rhs (stmt));
+	res_op->set_op (gimple_cond_code (stmt), boolean_type_node, lhs, rhs);
+	return true;
+      }
+
+    default:
+      break;
+    }
+
+  return false;
+}
+
+/* Try to describe STMT in RES_OP, returning true on success.
+   For GIMPLE_CONDs, describe the condition that is being tested.
+   For GIMPLE_ASSIGNs, describe the rhs of the assignment.
+   For GIMPLE_CALLs, describe the call.  */
+
+bool
+gimple_extract_op (gimple *stmt, gimple_match_op *res_op)
+{
+  auto nop = [](tree op) { return op; };
+  return gimple_extract (stmt, res_op, nop, nop);
+}
+
+/* The main STMT based simplification entry.  It is used by the fold_stmt
+   and the fold_stmt_to_constant APIs.  */
+
+bool
+gimple_simplify (gimple *stmt, gimple_match_op *res_op, gimple_seq *seq,
+		 tree (*valueize)(tree), tree (*top_valueize)(tree))
+{
+  bool valueized = false;
+  auto valueize_op = [&](tree op)
+    {
+      return do_valueize (op, top_valueize, valueized);
+    };
+  auto valueize_condition = [&](tree op) -> tree
+    {
+      bool cond_valueized = false;
+      tree lhs = do_valueize (TREE_OPERAND (op, 0), top_valueize,
+			      cond_valueized);
+      tree rhs = do_valueize (TREE_OPERAND (op, 1), top_valueize,
+			      cond_valueized);
+      gimple_match_op res_op2 (res_op->cond, TREE_CODE (op),
+			       TREE_TYPE (op), lhs, rhs);
+      if ((gimple_resimplify2 (seq, &res_op2, valueize)
+	   || cond_valueized)
+	  && res_op2.code.is_tree_code ())
+	{
+	  auto code = tree_code (res_op2.code);
+	  if (TREE_CODE_CLASS (code) == tcc_comparison)
+	    {
+	      valueized = true;
+	      return build2 (code, TREE_TYPE (op),
+			     res_op2.ops[0], res_op2.ops[1]);
+	    }
+	  else if (code == SSA_NAME
+		   || code == INTEGER_CST
+		   || code == VECTOR_CST)
+	    {
+	      valueized = true;
+	      return res_op2.ops[0];
+	    }
+	}
+      return valueize_op (op);
+    };
+
+  if (!gimple_extract (stmt, res_op, valueize_op, valueize_condition))
+    return false;
+
+  if (res_op->code.is_internal_fn ())
+    {
+      internal_fn ifn = internal_fn (res_op->code);
+      if (try_conditional_simplification (ifn, res_op, seq, valueize))
+	return true;
+    }
+
+  if (!res_op->reverse
+      && res_op->num_ops
+      && res_op->resimplify (seq, valueize))
+    return true;
+
+  return valueized;
+}
+
+/* Helper that matches and simplifies the toplevel result from
+   a gimple_simplify run (where we don't want to build
+   a stmt in case it's used in in-place folding).  Replaces
+   RES_OP with a simplified and/or canonicalized result and
+   returns whether any change was made.  */
+
+static bool
+gimple_resimplify1 (gimple_seq *seq, gimple_match_op *res_op,
+		    tree (*valueize)(tree))
+{
+  if (constant_for_folding (res_op->ops[0]))
+    {
+      tree tem = NULL_TREE;
+      if (res_op->code.is_tree_code ())
+	{
+	  auto code = tree_code (res_op->code);
+	  if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code))
+	      && TREE_CODE_LENGTH (code) == 1)
+	    tem = const_unop (code, res_op->type, res_op->ops[0]);
+	}
+      else
+	tem = fold_const_call (combined_fn (res_op->code), res_op->type,
+			       res_op->ops[0]);
+      if (tem != NULL_TREE
+	  && CONSTANT_CLASS_P (tem))
+	{
+	  if (TREE_OVERFLOW_P (tem))
+	    tem = drop_tree_overflow (tem);
+	  res_op->set_value (tem);
+	  maybe_resimplify_conditional_op (seq, res_op, valueize);
+	  return true;
+	}
+    }
+
+  /* Limit recursion, there are cases like PR80887 and others, for
+     example when value-numbering presents us with unfolded expressions
+     that we are really not prepared to handle without eventual
+     oscillation like ((_50 + 0) + 8) where _50 gets mapped to _50
+     itself as available expression.  */
+  static unsigned depth;
+  if (depth > 10)
+    {
+      if (dump_file && (dump_flags & TDF_FOLDING))
+	fprintf (dump_file, "Aborting expression simplification due to "
+		 "deep recursion\n");
+      return false;
+    }
+
+  ++depth;
+  gimple_match_op res_op2 (*res_op);
+  if (gimple_simplify (&res_op2, seq, valueize,
+		       res_op->code, res_op->type, res_op->ops[0]))
+    {
+      --depth;
+      *res_op = res_op2;
+      return true;
+    }
+  --depth;
+
+  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
+    return true;
+
+  return false;
+}
+
+/* Helper that matches and simplifies the toplevel result from
+   a gimple_simplify run (where we don't want to build
+   a stmt in case it's used in in-place folding).  Replaces
+   RES_OP with a simplified and/or canonicalized result and
+   returns whether any change was made.  */
+
+static bool
+gimple_resimplify2 (gimple_seq *seq, gimple_match_op *res_op,
+		    tree (*valueize)(tree))
+{
+  if (constant_for_folding (res_op->ops[0])
+      && constant_for_folding (res_op->ops[1]))
+    {
+      tree tem = NULL_TREE;
+      if (res_op->code.is_tree_code ())
+	{
+	  auto code = tree_code (res_op->code);
+	  if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code))
+	      && TREE_CODE_LENGTH (code) == 2)
+	    tem = const_binop (code, res_op->type,
+			       res_op->ops[0], res_op->ops[1]);
+	}
+      else
+	tem = fold_const_call (combined_fn (res_op->code), res_op->type,
+			       res_op->ops[0], res_op->ops[1]);
+      if (tem != NULL_TREE
+	  && CONSTANT_CLASS_P (tem))
+	{
+	  if (TREE_OVERFLOW_P (tem))
+	    tem = drop_tree_overflow (tem);
+	  res_op->set_value (tem);
+	  maybe_resimplify_conditional_op (seq, res_op, valueize);
+	  return true;
+	}
+    }
+
+  /* Canonicalize operand order.  */
+  bool canonicalized = false;
+  bool is_comparison
+    = (res_op->code.is_tree_code ()
+       && TREE_CODE_CLASS (tree_code (res_op->code)) == tcc_comparison);
+  if ((is_comparison || commutative_binary_op_p (res_op->code, res_op->type))
+      && tree_swap_operands_p (res_op->ops[0], res_op->ops[1]))
+    {
+      std::swap (res_op->ops[0], res_op->ops[1]);
+      if (is_comparison)
+	res_op->code = swap_tree_comparison (tree_code (res_op->code));
+      canonicalized = true;
+    }
+
+  /* Limit recursion, see gimple_resimplify1.  */
+  static unsigned depth;
+  if (depth > 10)
+    {
+      if (dump_file && (dump_flags & TDF_FOLDING))
+	fprintf (dump_file, "Aborting expression simplification due to "
+		 "deep recursion\n");
+      return false;
+    }
+
+  ++depth;
+  gimple_match_op res_op2 (*res_op);
+  if (gimple_simplify (&res_op2, seq, valueize,
+		       res_op->code, res_op->type,
+		       res_op->ops[0], res_op->ops[1]))
+    {
+      --depth;
+      *res_op = res_op2;
+      return true;
+    }
+  --depth;
+
+  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
+    return true;
+
+  return canonicalized;
+}
+
+/* Helper that matches and simplifies the toplevel result from
+   a gimple_simplify run (where we don't want to build
+   a stmt in case it's used in in-place folding).  Replaces
+   RES_OP with a simplified and/or canonicalized result and
+   returns whether any change was made.  */
+
+static bool
+gimple_resimplify3 (gimple_seq *seq, gimple_match_op *res_op,
+		    tree (*valueize)(tree))
+{
+  if (constant_for_folding (res_op->ops[0])
+      && constant_for_folding (res_op->ops[1])
+      && constant_for_folding (res_op->ops[2]))
+    {
+      tree tem = NULL_TREE;
+      if (res_op->code.is_tree_code ())
+	{
+	  auto code = tree_code (res_op->code);
+	  if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code))
+	      && TREE_CODE_LENGTH (code) == 3)
+	    tem = fold_ternary/*_to_constant*/ (code, res_op->type,
+						res_op->ops[0], res_op->ops[1],
+						res_op->ops[2]);
+	}
+      else
+	tem = fold_const_call (combined_fn (res_op->code), res_op->type,
+			       res_op->ops[0], res_op->ops[1], res_op->ops[2]);
+      if (tem != NULL_TREE
+	  && CONSTANT_CLASS_P (tem))
+	{
+	  if (TREE_OVERFLOW_P (tem))
+	    tem = drop_tree_overflow (tem);
+	  res_op->set_value (tem);
+	  maybe_resimplify_conditional_op (seq, res_op, valueize);
+	  return true;
+	}
+    }
+
+  /* Canonicalize operand order.  */
+  bool canonicalized = false;
+  int argno = first_commutative_argument (res_op->code, res_op->type);
+  if (argno >= 0
+      && tree_swap_operands_p (res_op->ops[argno], res_op->ops[argno + 1]))
+    {
+      std::swap (res_op->ops[argno], res_op->ops[argno + 1]);
+      canonicalized = true;
+    }
+
+  /* Limit recursion, see gimple_resimplify1.  */
+  static unsigned depth;
+  if (depth > 10)
+    {
+      if (dump_file && (dump_flags & TDF_FOLDING))
+	fprintf (dump_file, "Aborting expression simplification due to "
+		 "deep recursion\n");
+      return false;
+    }
+
+  ++depth;
+  gimple_match_op res_op2 (*res_op);
+  if (gimple_simplify (&res_op2, seq, valueize,
+		       res_op->code, res_op->type,
+		       res_op->ops[0], res_op->ops[1], res_op->ops[2]))
+    {
+      --depth;
+      *res_op = res_op2;
+      return true;
+    }
+  --depth;
+
+  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
+    return true;
+
+  return canonicalized;
+}
+
+/* Helper that matches and simplifies the toplevel result from
+   a gimple_simplify run (where we don't want to build
+   a stmt in case it's used in in-place folding).  Replaces
+   RES_OP with a simplified and/or canonicalized result and
+   returns whether any change was made.  */
+
+static bool
+gimple_resimplify4 (gimple_seq *seq, gimple_match_op *res_op,
+		    tree (*valueize)(tree))
+{
+  /* No constant folding is defined for four-operand functions.  */
+
+  /* Canonicalize operand order.  */
+  bool canonicalized = false;
+  int argno = first_commutative_argument (res_op->code, res_op->type);
+  if (argno >= 0
+      && tree_swap_operands_p (res_op->ops[argno], res_op->ops[argno + 1]))
+    {
+      std::swap (res_op->ops[argno], res_op->ops[argno + 1]);
+      canonicalized = true;
+    }
+
+  /* Limit recursion, see gimple_resimplify1.  */
+  static unsigned depth;
+  if (depth > 10)
+    {
+      if (dump_file && (dump_flags & TDF_FOLDING))
+	fprintf (dump_file, "Aborting expression simplification due to "
+		 "deep recursion\n");
+      return false;
+    }
+
+  ++depth;
+  gimple_match_op res_op2 (*res_op);
+  if (gimple_simplify (&res_op2, seq, valueize,
+		       res_op->code, res_op->type,
+		       res_op->ops[0], res_op->ops[1], res_op->ops[2],
+		       res_op->ops[3]))
+    {
+      --depth;
+      *res_op = res_op2;
+      return true;
+    }
+  --depth;
+
+  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
+    return true;
+
+  return canonicalized;
+}
+
+/* Helper that matches and simplifies the toplevel result from
+   a gimple_simplify run (where we don't want to build
+   a stmt in case it's used in in-place folding).  Replaces
+   RES_OP with a simplified and/or canonicalized result and
+   returns whether any change was made.  */
+
+static bool
+gimple_resimplify5 (gimple_seq *seq, gimple_match_op *res_op,
+		    tree (*valueize)(tree))
+{
+  /* No constant folding is defined for five-operand functions.  */
+
+  /* Canonicalize operand order.  */
+  bool canonicalized = false;
+  int argno = first_commutative_argument (res_op->code, res_op->type);
+  if (argno >= 0
+      && tree_swap_operands_p (res_op->ops[argno], res_op->ops[argno + 1]))
+    {
+      std::swap (res_op->ops[argno], res_op->ops[argno + 1]);
+      canonicalized = true;
+    }
+
+  gimple_match_op res_op2 (*res_op);
+  if (gimple_simplify (&res_op2, seq, valueize,
+		       res_op->code, res_op->type,
+		       res_op->ops[0], res_op->ops[1], res_op->ops[2],
+		       res_op->ops[3], res_op->ops[4]))
+    {
+      *res_op = res_op2;
+      return true;
+    }
+
+  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
+    return true;
+
+  return canonicalized;
+}
+
+/* Return a canonical form for CODE when operating on TYPE.  The idea
+   is to remove redundant ways of representing the same operation so
+   that code_helpers can be hashed and compared for equality.
+
+   The only current canonicalization is to replace built-in functions
+   with internal functions, in cases where internal-fn.def defines
+   such an internal function.
+
+   Note that the new code_helper cannot necessarily be used in place of
+   the original code_helper.  For example, the new code_helper might be
+   an internal function that the target does not support.  */
+
+code_helper
+canonicalize_code (code_helper code, tree type)
+{
+  if (code.is_fn_code ())
+    return associated_internal_fn (combined_fn (code), type);
+  return code;
+}
+
+/* Return true if CODE is a binary operation and if CODE is commutative when
+   operating on type TYPE.  */
+
+bool
+commutative_binary_op_p (code_helper code, tree type)
+{
+  if (code.is_tree_code ())
+    return commutative_tree_code (tree_code (code));
+  auto cfn = combined_fn (code);
+  return commutative_binary_fn_p (associated_internal_fn (cfn, type));
+}
+
+/* Return true if CODE represents a ternary operation and if the first two
+   operands are commutative when CODE is operating on TYPE.  */
+
+bool
+commutative_ternary_op_p (code_helper code, tree type)
+{
+  if (code.is_tree_code ())
+    return commutative_ternary_tree_code (tree_code (code));
+  auto cfn = combined_fn (code);
+  return commutative_ternary_fn_p (associated_internal_fn (cfn, type));
+}
+
+/* If CODE is commutative in two consecutive operands, return the
+   index of the first, otherwise return -1.  */
+
+int
+first_commutative_argument (code_helper code, tree type)
+{
+  if (code.is_tree_code ())
+    {
+      auto tcode = tree_code (code);
+      if (commutative_tree_code (tcode)
+	  || commutative_ternary_tree_code (tcode))
+	return 0;
+      return -1;
+    }
+  auto cfn = combined_fn (code);
+  return first_commutative_argument (associated_internal_fn (cfn, type));
+}
+
+/* Return true if CODE is a binary operation that is associative when
+   operating on type TYPE.  */
+
+bool
+associative_binary_op_p (code_helper code, tree type)
+{
+  if (code.is_tree_code ())
+    return associative_tree_code (tree_code (code));
+  auto cfn = combined_fn (code);
+  return associative_binary_fn_p (associated_internal_fn (cfn, type));
+}
+
+/* Return true if the target directly supports operation CODE on type TYPE.
+   QUERY_TYPE acts as for optab_for_tree_code.  */
+
+bool
+directly_supported_p (code_helper code, tree type, optab_subtype query_type)
+{
+  if (code.is_tree_code ())
+    {
+      direct_optab optab = optab_for_tree_code (tree_code (code), type,
+						query_type);
+      return (optab != unknown_optab
+	      && optab_handler (optab, TYPE_MODE (type)) != CODE_FOR_nothing);
+    }
+  gcc_assert (query_type == optab_default
+	      || (query_type == optab_vector && VECTOR_TYPE_P (type))
+	      || (query_type == optab_scalar && !VECTOR_TYPE_P (type)));
+  internal_fn ifn = associated_internal_fn (combined_fn (code), type);
+  return (direct_internal_fn_p (ifn)
+	  && direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED));
+}
+
+/* A wrapper around the internal-fn.cc versions of get_conditional_internal_fn
+   for a code_helper CODE operating on type TYPE.  */
+
+internal_fn
+get_conditional_internal_fn (code_helper code, tree type)
+{
+  if (code.is_tree_code ())
+    return get_conditional_internal_fn (tree_code (code));
+  auto cfn = combined_fn (code);
+  return get_conditional_internal_fn (associated_internal_fn (cfn, type));
+}
diff --git a/gcc/gimple-match-head.cc b/gcc/gimple-match-head.cc
index ae0c5c8a74fd9f1acdb616014941b11961e96c04..64591da3465b5102269f18817175a5c9c9a232e4 100644
--- a/gcc/gimple-match-head.cc
+++ b/gcc/gimple-match-head.cc
@@ -48,1090 +48,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-range.h"
 #include "langhooks.h"
 
-/* Forward declarations of the private auto-generated matchers.
-   They expect valueized operands in canonical order and do not
-   perform simplification of all-constant operands.  */
-static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
-			     code_helper, tree, tree);
-static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
-			     code_helper, tree, tree, tree);
-static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
-			     code_helper, tree, tree, tree, tree);
-static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
-			     code_helper, tree, tree, tree, tree, tree);
-static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
-			     code_helper, tree, tree, tree, tree, tree, tree);
-static bool gimple_resimplify1 (gimple_seq *, gimple_match_op *,
-				tree (*)(tree));
-static bool gimple_resimplify2 (gimple_seq *, gimple_match_op *,
-				tree (*)(tree));
-static bool gimple_resimplify3 (gimple_seq *, gimple_match_op *,
-				tree (*)(tree));
-static bool gimple_resimplify4 (gimple_seq *, gimple_match_op *,
-				tree (*)(tree));
-static bool gimple_resimplify5 (gimple_seq *, gimple_match_op *,
-				tree (*)(tree));
-
-const unsigned int gimple_match_op::MAX_NUM_OPS;
-
-/* Return whether T is a constant that we'll dispatch to fold to
-   evaluate fully constant expressions.  */
-
-static inline bool
-constant_for_folding (tree t)
-{
-  return (CONSTANT_CLASS_P (t)
-	  /* The following is only interesting to string builtins.  */
-	  || (TREE_CODE (t) == ADDR_EXPR
-	      && TREE_CODE (TREE_OPERAND (t, 0)) == STRING_CST));
-}
-
-/* Try to convert conditional operation ORIG_OP into an IFN_COND_*
-   operation.  Return true on success, storing the new operation in NEW_OP.  */
-
-static bool
-convert_conditional_op (gimple_match_op *orig_op,
-			gimple_match_op *new_op)
-{
-  internal_fn ifn;
-  if (orig_op->code.is_tree_code ())
-    ifn = get_conditional_internal_fn ((tree_code) orig_op->code);
-  else
-    {
-      auto cfn = combined_fn (orig_op->code);
-      if (!internal_fn_p (cfn))
-	return false;
-      ifn = get_conditional_internal_fn (as_internal_fn (cfn));
-    }
-  if (ifn == IFN_LAST)
-    return false;
-  unsigned int num_ops = orig_op->num_ops;
-  new_op->set_op (as_combined_fn (ifn), orig_op->type, num_ops + 2);
-  new_op->ops[0] = orig_op->cond.cond;
-  for (unsigned int i = 0; i < num_ops; ++i)
-    new_op->ops[i + 1] = orig_op->ops[i];
-  tree else_value = orig_op->cond.else_value;
-  if (!else_value)
-    else_value = targetm.preferred_else_value (ifn, orig_op->type,
-					       num_ops, orig_op->ops);
-  new_op->ops[num_ops + 1] = else_value;
-  return true;
-}
-
-/* RES_OP is the result of a simplification.  If it is conditional,
-   try to replace it with the equivalent UNCOND form, such as an
-   IFN_COND_* call or a VEC_COND_EXPR.  Also try to resimplify the
-   result of the replacement if appropriate, adding any new statements to
-   SEQ and using VALUEIZE as the valueization function.  Return true if
-   this resimplification occurred and resulted in at least one change.  */
-
-static bool
-maybe_resimplify_conditional_op (gimple_seq *seq, gimple_match_op *res_op,
-				 tree (*valueize) (tree))
-{
-  if (!res_op->cond.cond)
-    return false;
-
-  if (!res_op->cond.else_value
-      && res_op->code.is_tree_code ())
-    {
-      /* The "else" value doesn't matter.  If the "then" value is a
-	 gimple value, just use it unconditionally.  This isn't a
-	 simplification in itself, since there was no operation to
-	 build in the first place.  */
-      if (gimple_simplified_result_is_gimple_val (res_op))
-	{
-	  res_op->cond.cond = NULL_TREE;
-	  return false;
-	}
-
-      /* Likewise if the operation would not trap.  */
-      bool honor_trapv = (INTEGRAL_TYPE_P (res_op->type)
-			  && TYPE_OVERFLOW_TRAPS (res_op->type));
-      tree_code op_code = (tree_code) res_op->code;
-      bool op_could_trap;
-
-      /* COND_EXPR will trap if, and only if, the condition
-	 traps and hence we have to check this.  For all other operations, we
-	 don't need to consider the operands.  */
-      if (op_code == COND_EXPR)
-	op_could_trap = generic_expr_could_trap_p (res_op->ops[0]);
-      else
-	op_could_trap = operation_could_trap_p ((tree_code) res_op->code,
-						FLOAT_TYPE_P (res_op->type),
-						honor_trapv,
-						res_op->op_or_null (1));
-
-      if (!op_could_trap)
-	{
-	  res_op->cond.cond = NULL_TREE;
-	  return false;
-	}
-    }
-
-  /* If the "then" value is a gimple value and the "else" value matters,
-     create a VEC_COND_EXPR between them, then see if it can be further
-     simplified.  */
-  gimple_match_op new_op;
-  if (res_op->cond.else_value
-      && VECTOR_TYPE_P (res_op->type)
-      && gimple_simplified_result_is_gimple_val (res_op))
-    {
-      new_op.set_op (VEC_COND_EXPR, res_op->type,
-		     res_op->cond.cond, res_op->ops[0],
-		     res_op->cond.else_value);
-      *res_op = new_op;
-      return gimple_resimplify3 (seq, res_op, valueize);
-    }
-
-  /* Otherwise try rewriting the operation as an IFN_COND_* call.
-     Again, this isn't a simplification in itself, since it's what
-     RES_OP already described.  */
-  if (convert_conditional_op (res_op, &new_op))
-    *res_op = new_op;
-
-  return false;
-}
-
-/* Helper that matches and simplifies the toplevel result from
-   a gimple_simplify run (where we don't want to build
-   a stmt in case it's used in in-place folding).  Replaces
-   RES_OP with a simplified and/or canonicalized result and
-   returns whether any change was made.  */
-
-static bool
-gimple_resimplify1 (gimple_seq *seq, gimple_match_op *res_op,
-		    tree (*valueize)(tree))
-{
-  if (constant_for_folding (res_op->ops[0]))
-    {
-      tree tem = NULL_TREE;
-      if (res_op->code.is_tree_code ())
-	{
-	  auto code = tree_code (res_op->code);
-	  if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code))
-	      && TREE_CODE_LENGTH (code) == 1)
-	    tem = const_unop (code, res_op->type, res_op->ops[0]);
-	}
-      else
-	tem = fold_const_call (combined_fn (res_op->code), res_op->type,
-			       res_op->ops[0]);
-      if (tem != NULL_TREE
-	  && CONSTANT_CLASS_P (tem))
-	{
-	  if (TREE_OVERFLOW_P (tem))
-	    tem = drop_tree_overflow (tem);
-	  res_op->set_value (tem);
-	  maybe_resimplify_conditional_op (seq, res_op, valueize);
-	  return true;
-	}
-    }
-
-  /* Limit recursion, there are cases like PR80887 and others, for
-     example when value-numbering presents us with unfolded expressions
-     that we are really not prepared to handle without eventual
-     oscillation like ((_50 + 0) + 8) where _50 gets mapped to _50
-     itself as available expression.  */
-  static unsigned depth;
-  if (depth > 10)
-    {
-      if (dump_file && (dump_flags & TDF_FOLDING))
-	fprintf (dump_file, "Aborting expression simplification due to "
-		 "deep recursion\n");
-      return false;
-    }
-
-  ++depth;
-  gimple_match_op res_op2 (*res_op);
-  if (gimple_simplify (&res_op2, seq, valueize,
-		       res_op->code, res_op->type, res_op->ops[0]))
-    {
-      --depth;
-      *res_op = res_op2;
-      return true;
-    }
-  --depth;
-
-  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
-    return true;
-
-  return false;
-}
-
-/* Helper that matches and simplifies the toplevel result from
-   a gimple_simplify run (where we don't want to build
-   a stmt in case it's used in in-place folding).  Replaces
-   RES_OP with a simplified and/or canonicalized result and
-   returns whether any change was made.  */
-
-static bool
-gimple_resimplify2 (gimple_seq *seq, gimple_match_op *res_op,
-		    tree (*valueize)(tree))
-{
-  if (constant_for_folding (res_op->ops[0])
-      && constant_for_folding (res_op->ops[1]))
-    {
-      tree tem = NULL_TREE;
-      if (res_op->code.is_tree_code ())
-	{
-	  auto code = tree_code (res_op->code);
-	  if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code))
-	      && TREE_CODE_LENGTH (code) == 2)
-	    tem = const_binop (code, res_op->type,
-			       res_op->ops[0], res_op->ops[1]);
-	}
-      else
-	tem = fold_const_call (combined_fn (res_op->code), res_op->type,
-			       res_op->ops[0], res_op->ops[1]);
-      if (tem != NULL_TREE
-	  && CONSTANT_CLASS_P (tem))
-	{
-	  if (TREE_OVERFLOW_P (tem))
-	    tem = drop_tree_overflow (tem);
-	  res_op->set_value (tem);
-	  maybe_resimplify_conditional_op (seq, res_op, valueize);
-	  return true;
-	}
-    }
-
-  /* Canonicalize operand order.  */
-  bool canonicalized = false;
-  bool is_comparison
-    = (res_op->code.is_tree_code ()
-       && TREE_CODE_CLASS (tree_code (res_op->code)) == tcc_comparison);
-  if ((is_comparison || commutative_binary_op_p (res_op->code, res_op->type))
-      && tree_swap_operands_p (res_op->ops[0], res_op->ops[1]))
-    {
-      std::swap (res_op->ops[0], res_op->ops[1]);
-      if (is_comparison)
-	res_op->code = swap_tree_comparison (tree_code (res_op->code));
-      canonicalized = true;
-    }
-
-  /* Limit recursion, see gimple_resimplify1.  */
-  static unsigned depth;
-  if (depth > 10)
-    {
-      if (dump_file && (dump_flags & TDF_FOLDING))
-	fprintf (dump_file, "Aborting expression simplification due to "
-		 "deep recursion\n");
-      return false;
-    }
-
-  ++depth;
-  gimple_match_op res_op2 (*res_op);
-  if (gimple_simplify (&res_op2, seq, valueize,
-		       res_op->code, res_op->type,
-		       res_op->ops[0], res_op->ops[1]))
-    {
-      --depth;
-      *res_op = res_op2;
-      return true;
-    }
-  --depth;
-
-  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
-    return true;
-
-  return canonicalized;
-}
-
-/* Helper that matches and simplifies the toplevel result from
-   a gimple_simplify run (where we don't want to build
-   a stmt in case it's used in in-place folding).  Replaces
-   RES_OP with a simplified and/or canonicalized result and
-   returns whether any change was made.  */
-
-static bool
-gimple_resimplify3 (gimple_seq *seq, gimple_match_op *res_op,
-		    tree (*valueize)(tree))
-{
-  if (constant_for_folding (res_op->ops[0])
-      && constant_for_folding (res_op->ops[1])
-      && constant_for_folding (res_op->ops[2]))
-    {
-      tree tem = NULL_TREE;
-      if (res_op->code.is_tree_code ())
-	{
-	  auto code = tree_code (res_op->code);
-	  if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code))
-	      && TREE_CODE_LENGTH (code) == 3)
-	    tem = fold_ternary/*_to_constant*/ (code, res_op->type,
-						res_op->ops[0], res_op->ops[1],
-						res_op->ops[2]);
-	}
-      else
-	tem = fold_const_call (combined_fn (res_op->code), res_op->type,
-			       res_op->ops[0], res_op->ops[1], res_op->ops[2]);
-      if (tem != NULL_TREE
-	  && CONSTANT_CLASS_P (tem))
-	{
-	  if (TREE_OVERFLOW_P (tem))
-	    tem = drop_tree_overflow (tem);
-	  res_op->set_value (tem);
-	  maybe_resimplify_conditional_op (seq, res_op, valueize);
-	  return true;
-	}
-    }
-
-  /* Canonicalize operand order.  */
-  bool canonicalized = false;
-  int argno = first_commutative_argument (res_op->code, res_op->type);
-  if (argno >= 0
-      && tree_swap_operands_p (res_op->ops[argno], res_op->ops[argno + 1]))
-    {
-      std::swap (res_op->ops[argno], res_op->ops[argno + 1]);
-      canonicalized = true;
-    }
-
-  /* Limit recursion, see gimple_resimplify1.  */
-  static unsigned depth;
-  if (depth > 10)
-    {
-      if (dump_file && (dump_flags & TDF_FOLDING))
-	fprintf (dump_file, "Aborting expression simplification due to "
-		 "deep recursion\n");
-      return false;
-    }
-
-  ++depth;
-  gimple_match_op res_op2 (*res_op);
-  if (gimple_simplify (&res_op2, seq, valueize,
-		       res_op->code, res_op->type,
-		       res_op->ops[0], res_op->ops[1], res_op->ops[2]))
-    {
-      --depth;
-      *res_op = res_op2;
-      return true;
-    }
-  --depth;
-
-  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
-    return true;
-
-  return canonicalized;
-}
-
-/* Helper that matches and simplifies the toplevel result from
-   a gimple_simplify run (where we don't want to build
-   a stmt in case it's used in in-place folding).  Replaces
-   RES_OP with a simplified and/or canonicalized result and
-   returns whether any change was made.  */
-
-static bool
-gimple_resimplify4 (gimple_seq *seq, gimple_match_op *res_op,
-		    tree (*valueize)(tree))
-{
-  /* No constant folding is defined for four-operand functions.  */
-
-  /* Canonicalize operand order.  */
-  bool canonicalized = false;
-  int argno = first_commutative_argument (res_op->code, res_op->type);
-  if (argno >= 0
-      && tree_swap_operands_p (res_op->ops[argno], res_op->ops[argno + 1]))
-    {
-      std::swap (res_op->ops[argno], res_op->ops[argno + 1]);
-      canonicalized = true;
-    }
-
-  /* Limit recursion, see gimple_resimplify1.  */
-  static unsigned depth;
-  if (depth > 10)
-    {
-      if (dump_file && (dump_flags & TDF_FOLDING))
-	fprintf (dump_file, "Aborting expression simplification due to "
-		 "deep recursion\n");
-      return false;
-    }
-
-  ++depth;
-  gimple_match_op res_op2 (*res_op);
-  if (gimple_simplify (&res_op2, seq, valueize,
-		       res_op->code, res_op->type,
-		       res_op->ops[0], res_op->ops[1], res_op->ops[2],
-		       res_op->ops[3]))
-    {
-      --depth;
-      *res_op = res_op2;
-      return true;
-    }
-  --depth;
-
-  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
-    return true;
-
-  return canonicalized;
-}
-
-/* Helper that matches and simplifies the toplevel result from
-   a gimple_simplify run (where we don't want to build
-   a stmt in case it's used in in-place folding).  Replaces
-   RES_OP with a simplified and/or canonicalized result and
-   returns whether any change was made.  */
-
-static bool
-gimple_resimplify5 (gimple_seq *seq, gimple_match_op *res_op,
-		    tree (*valueize)(tree))
-{
-  /* No constant folding is defined for five-operand functions.  */
-
-  /* Canonicalize operand order.  */
-  bool canonicalized = false;
-  int argno = first_commutative_argument (res_op->code, res_op->type);
-  if (argno >= 0
-      && tree_swap_operands_p (res_op->ops[argno], res_op->ops[argno + 1]))
-    {
-      std::swap (res_op->ops[argno], res_op->ops[argno + 1]);
-      canonicalized = true;
-    }
-
-  gimple_match_op res_op2 (*res_op);
-  if (gimple_simplify (&res_op2, seq, valueize,
-		       res_op->code, res_op->type,
-		       res_op->ops[0], res_op->ops[1], res_op->ops[2],
-		       res_op->ops[3], res_op->ops[4]))
-    {
-      *res_op = res_op2;
-      return true;
-    }
-
-  if (maybe_resimplify_conditional_op (seq, res_op, valueize))
-    return true;
-
-  return canonicalized;
-}
-
-/* Match and simplify the toplevel valueized operation THIS.
-   Replaces THIS with a simplified and/or canonicalized result and
-   returns whether any change was made.  */
-
-bool
-gimple_match_op::resimplify (gimple_seq *seq, tree (*valueize)(tree))
-{
-  switch (num_ops)
-    {
-    case 1:
-      return gimple_resimplify1 (seq, this, valueize);
-    case 2:
-      return gimple_resimplify2 (seq, this, valueize);
-    case 3:
-      return gimple_resimplify3 (seq, this, valueize);
-    case 4:
-      return gimple_resimplify4 (seq, this, valueize);
-    case 5:
-      return gimple_resimplify5 (seq, this, valueize);
-    default:
-      gcc_unreachable ();
-    }
-}
-
-/* If in GIMPLE the operation described by RES_OP should be single-rhs,
-   build a GENERIC tree for that expression and update RES_OP accordingly.  */
-
-void
-maybe_build_generic_op (gimple_match_op *res_op)
-{
-  tree_code code = (tree_code) res_op->code;
-  tree val;
-  switch (code)
-    {
-    case REALPART_EXPR:
-    case IMAGPART_EXPR:
-    case VIEW_CONVERT_EXPR:
-      val = build1 (code, res_op->type, res_op->ops[0]);
-      res_op->set_value (val);
-      break;
-    case BIT_FIELD_REF:
-      val = build3 (code, res_op->type, res_op->ops[0], res_op->ops[1],
-		    res_op->ops[2]);
-      REF_REVERSE_STORAGE_ORDER (val) = res_op->reverse;
-      res_op->set_value (val);
-      break;
-    default:;
-    }
-}
-
-tree (*mprts_hook) (gimple_match_op *);
-
-/* Try to build RES_OP, which is known to be a call to FN.  Return null
-   if the target doesn't support the function.  */
-
-static gcall *
-build_call_internal (internal_fn fn, gimple_match_op *res_op)
-{
-  if (direct_internal_fn_p (fn))
-    {
-      tree_pair types = direct_internal_fn_types (fn, res_op->type,
-						  res_op->ops);
-      if (!direct_internal_fn_supported_p (fn, types, OPTIMIZE_FOR_BOTH))
-	return NULL;
-    }
-  return gimple_build_call_internal (fn, res_op->num_ops,
-				     res_op->op_or_null (0),
-				     res_op->op_or_null (1),
-				     res_op->op_or_null (2),
-				     res_op->op_or_null (3),
-				     res_op->op_or_null (4));
-}
-
-/* Push the exploded expression described by RES_OP as a statement to
-   SEQ if necessary and return a gimple value denoting the value of the
-   expression.  If RES is not NULL then the result will be always RES
-   and even gimple values are pushed to SEQ.  */
-
-tree
-maybe_push_res_to_seq (gimple_match_op *res_op, gimple_seq *seq, tree res)
-{
-  tree *ops = res_op->ops;
-  unsigned num_ops = res_op->num_ops;
-
-  /* The caller should have converted conditional operations into an UNCOND
-     form and resimplified as appropriate.  The conditional form only
-     survives this far if that conversion failed.  */
-  if (res_op->cond.cond)
-    return NULL_TREE;
-
-  if (res_op->code.is_tree_code ())
-    {
-      if (!res
-	  && gimple_simplified_result_is_gimple_val (res_op))
-	return ops[0];
-      if (mprts_hook)
-	{
-	  tree tem = mprts_hook (res_op);
-	  if (tem)
-	    return tem;
-	}
-    }
-
-  if (!seq)
-    return NULL_TREE;
-
-  /* Play safe and do not allow abnormals to be mentioned in
-     newly created statements.  */
-  for (unsigned int i = 0; i < num_ops; ++i)
-    if (TREE_CODE (ops[i]) == SSA_NAME
-	&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ops[i]))
-      return NULL_TREE;
-
-  if (num_ops > 0 && COMPARISON_CLASS_P (ops[0]))
-    for (unsigned int i = 0; i < 2; ++i)
-      if (TREE_CODE (TREE_OPERAND (ops[0], i)) == SSA_NAME
-	  && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (ops[0], i)))
-	return NULL_TREE;
-
-  if (res_op->code.is_tree_code ())
-    {
-      auto code = tree_code (res_op->code);
-      if (!res)
-	{
-	  if (gimple_in_ssa_p (cfun))
-	    res = make_ssa_name (res_op->type);
-	  else
-	    res = create_tmp_reg (res_op->type);
-	}
-      maybe_build_generic_op (res_op);
-      gimple *new_stmt = gimple_build_assign (res, code,
-					      res_op->op_or_null (0),
-					      res_op->op_or_null (1),
-					      res_op->op_or_null (2));
-      gimple_seq_add_stmt_without_update (seq, new_stmt);
-      return res;
-    }
-  else
-    {
-      gcc_assert (num_ops != 0);
-      auto fn = combined_fn (res_op->code);
-      gcall *new_stmt = NULL;
-      if (internal_fn_p (fn))
-	{
-	  /* Generate the given function if we can.  */
-	  internal_fn ifn = as_internal_fn (fn);
-	  new_stmt = build_call_internal (ifn, res_op);
-	  if (!new_stmt)
-	    return NULL_TREE;
-	}
-      else
-	{
-	  /* Find the function we want to call.  */
-	  tree decl = builtin_decl_implicit (as_builtin_fn (fn));
-	  if (!decl)
-	    return NULL;
-
-	  /* We can't and should not emit calls to non-const functions.  */
-	  if (!(flags_from_decl_or_type (decl) & ECF_CONST))
-	    return NULL;
-
-	  new_stmt = gimple_build_call (decl, num_ops,
-					res_op->op_or_null (0),
-					res_op->op_or_null (1),
-					res_op->op_or_null (2),
-					res_op->op_or_null (3),
-					res_op->op_or_null (4));
-	}
-      if (!res)
-	{
-	  if (gimple_in_ssa_p (cfun))
-	    res = make_ssa_name (res_op->type);
-	  else
-	    res = create_tmp_reg (res_op->type);
-	}
-      gimple_call_set_lhs (new_stmt, res);
-      gimple_seq_add_stmt_without_update (seq, new_stmt);
-      return res;
-    }
-}
-
-
-/* Public API overloads follow for operation being tree_code or
-   built_in_function and for one to three operands or arguments.
-   They return NULL_TREE if nothing could be simplified or
-   the resulting simplified value with parts pushed to SEQ.
-   If SEQ is NULL then if the simplification needs to create
-   new stmts it will fail.  If VALUEIZE is non-NULL then all
-   SSA names will be valueized using that hook prior to
-   applying simplifications.  */
-
-/* Unary ops.  */
-
-tree
-gimple_simplify (enum tree_code code, tree type,
-		 tree op0,
-		 gimple_seq *seq, tree (*valueize)(tree))
-{
-  if (constant_for_folding (op0))
-    {
-      tree res = const_unop (code, type, op0);
-      if (res != NULL_TREE
-	  && CONSTANT_CLASS_P (res))
-	return res;
-    }
-
-  gimple_match_op res_op;
-  if (!gimple_simplify (&res_op, seq, valueize, code, type, op0))
-    return NULL_TREE;
-  return maybe_push_res_to_seq (&res_op, seq);
-}
-
-/* Binary ops.  */
-
-tree
-gimple_simplify (enum tree_code code, tree type,
-		 tree op0, tree op1,
-		 gimple_seq *seq, tree (*valueize)(tree))
-{
-  if (constant_for_folding (op0) && constant_for_folding (op1))
-    {
-      tree res = const_binop (code, type, op0, op1);
-      if (res != NULL_TREE
-	  && CONSTANT_CLASS_P (res))
-	return res;
-    }
-
-  /* Canonicalize operand order both for matching and fallback stmt
-     generation.  */
-  if ((commutative_tree_code (code)
-       || TREE_CODE_CLASS (code) == tcc_comparison)
-      && tree_swap_operands_p (op0, op1))
-    {
-      std::swap (op0, op1);
-      if (TREE_CODE_CLASS (code) == tcc_comparison)
-	code = swap_tree_comparison (code);
-    }
-
-  gimple_match_op res_op;
-  if (!gimple_simplify (&res_op, seq, valueize, code, type, op0, op1))
-    return NULL_TREE;
-  return maybe_push_res_to_seq (&res_op, seq);
-}
-
-/* Ternary ops.  */
-
-tree
-gimple_simplify (enum tree_code code, tree type,
-		 tree op0, tree op1, tree op2,
-		 gimple_seq *seq, tree (*valueize)(tree))
-{
-  if (constant_for_folding (op0) && constant_for_folding (op1)
-      && constant_for_folding (op2))
-    {
-      tree res = fold_ternary/*_to_constant */ (code, type, op0, op1, op2);
-      if (res != NULL_TREE
-	  && CONSTANT_CLASS_P (res))
-	return res;
-    }
-
-  /* Canonicalize operand order both for matching and fallback stmt
-     generation.  */
-  if (commutative_ternary_tree_code (code)
-      && tree_swap_operands_p (op0, op1))
-    std::swap (op0, op1);
-
-  gimple_match_op res_op;
-  if (!gimple_simplify (&res_op, seq, valueize, code, type, op0, op1, op2))
-    return NULL_TREE;
-  return maybe_push_res_to_seq (&res_op, seq);
-}
-
-/* Builtin or internal function with one argument.  */
-
-tree
-gimple_simplify (combined_fn fn, tree type,
-		 tree arg0,
-		 gimple_seq *seq, tree (*valueize)(tree))
-{
-  if (constant_for_folding (arg0))
-    {
-      tree res = fold_const_call (fn, type, arg0);
-      if (res && CONSTANT_CLASS_P (res))
-	return res;
-    }
-
-  gimple_match_op res_op;
-  if (!gimple_simplify (&res_op, seq, valueize, fn, type, arg0))
-    return NULL_TREE;
-  return maybe_push_res_to_seq (&res_op, seq);
-}
-
-/* Builtin or internal function with two arguments.  */
-
-tree
-gimple_simplify (combined_fn fn, tree type,
-		 tree arg0, tree arg1,
-		 gimple_seq *seq, tree (*valueize)(tree))
-{
-  if (constant_for_folding (arg0)
-      && constant_for_folding (arg1))
-    {
-      tree res = fold_const_call (fn, type, arg0, arg1);
-      if (res && CONSTANT_CLASS_P (res))
-	return res;
-    }
-
-  gimple_match_op res_op;
-  if (!gimple_simplify (&res_op, seq, valueize, fn, type, arg0, arg1))
-    return NULL_TREE;
-  return maybe_push_res_to_seq (&res_op, seq);
-}
-
-/* Builtin or internal function with three arguments.  */
-
-tree
-gimple_simplify (combined_fn fn, tree type,
-		 tree arg0, tree arg1, tree arg2,
-		 gimple_seq *seq, tree (*valueize)(tree))
-{
-  if (constant_for_folding (arg0)
-      && constant_for_folding (arg1)
-      && constant_for_folding (arg2))
-    {
-      tree res = fold_const_call (fn, type, arg0, arg1, arg2);
-      if (res && CONSTANT_CLASS_P (res))
-	return res;
-    }
-
-  gimple_match_op res_op;
-  if (!gimple_simplify (&res_op, seq, valueize, fn, type, arg0, arg1, arg2))
-    return NULL_TREE;
-  return maybe_push_res_to_seq (&res_op, seq);
-}
-
-/* Helper for gimple_simplify valueizing OP using VALUEIZE and setting
-   VALUEIZED to true if valueization changed OP.  */
-
-static inline tree
-do_valueize (tree op, tree (*valueize)(tree), bool &valueized)
-{
-  if (valueize && TREE_CODE (op) == SSA_NAME)
-    {
-      tree tem = valueize (op);
-      if (tem && tem != op)
-	{
-	  op = tem;
-	  valueized = true;
-	}
-    }
-  return op;
-}
-
-/* If RES_OP is a call to a conditional internal function, try simplifying
-   the associated unconditional operation and using the result to build
-   a new conditional operation.  For example, if RES_OP is:
-
-     IFN_COND_ADD (COND, A, B, ELSE)
-
-   try simplifying (plus A B) and using the result to build a replacement
-   for the whole IFN_COND_ADD.
-
-   Return true if this approach led to a simplification, otherwise leave
-   RES_OP unchanged (and so suitable for other simplifications).  When
-   returning true, add any new statements to SEQ and use VALUEIZE as the
-   valueization function.
-
-   RES_OP is known to be a call to IFN.  */
-
-static bool
-try_conditional_simplification (internal_fn ifn, gimple_match_op *res_op,
-				gimple_seq *seq, tree (*valueize) (tree))
-{
-  code_helper op;
-  tree_code code = conditional_internal_fn_code (ifn);
-  if (code != ERROR_MARK)
-    op = code;
-  else
-    {
-      ifn = get_unconditional_internal_fn (ifn);
-      if (ifn == IFN_LAST)
-	return false;
-      op = as_combined_fn (ifn);
-    }
-
-  unsigned int num_ops = res_op->num_ops;
-  gimple_match_op cond_op (gimple_match_cond (res_op->ops[0],
-					      res_op->ops[num_ops - 1]),
-			   op, res_op->type, num_ops - 2);
-
-  memcpy (cond_op.ops, res_op->ops + 1, (num_ops - 1) * sizeof *cond_op.ops);
-  switch (num_ops - 2)
-    {
-    case 1:
-      if (!gimple_resimplify1 (seq, &cond_op, valueize))
-	return false;
-      break;
-    case 2:
-      if (!gimple_resimplify2 (seq, &cond_op, valueize))
-	return false;
-      break;
-    case 3:
-      if (!gimple_resimplify3 (seq, &cond_op, valueize))
-	return false;
-      break;
-    default:
-      gcc_unreachable ();
-    }
-  *res_op = cond_op;
-  maybe_resimplify_conditional_op (seq, res_op, valueize);
-  return true;
-}
-
-/* Common subroutine of gimple_extract_op and gimple_simplify.  Try to
-   describe STMT in RES_OP, returning true on success.  Before recording
-   an operand, call:
-
-   - VALUEIZE_CONDITION for a COND_EXPR condition
-   - VALUEIZE_OP for every other top-level operand
-
-   Both routines take a tree argument and returns a tree.  */
-
-template<typename ValueizeOp, typename ValueizeCondition>
-inline bool
-gimple_extract (gimple *stmt, gimple_match_op *res_op,
-		ValueizeOp valueize_op,
-		ValueizeCondition valueize_condition)
-{
-  switch (gimple_code (stmt))
-    {
-    case GIMPLE_ASSIGN:
-      {
-	enum tree_code code = gimple_assign_rhs_code (stmt);
-	tree type = TREE_TYPE (gimple_assign_lhs (stmt));
-	switch (gimple_assign_rhs_class (stmt))
-	  {
-	  case GIMPLE_SINGLE_RHS:
-	    if (code == REALPART_EXPR
-		|| code == IMAGPART_EXPR
-		|| code == VIEW_CONVERT_EXPR)
-	      {
-		tree op0 = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
-		res_op->set_op (code, type, valueize_op (op0));
-		return true;
-	      }
-	    else if (code == BIT_FIELD_REF)
-	      {
-		tree rhs1 = gimple_assign_rhs1 (stmt);
-		tree op0 = valueize_op (TREE_OPERAND (rhs1, 0));
-		res_op->set_op (code, type, op0,
-				TREE_OPERAND (rhs1, 1),
-				TREE_OPERAND (rhs1, 2),
-				REF_REVERSE_STORAGE_ORDER (rhs1));
-		return true;
-	      }
-	    else if (code == SSA_NAME)
-	      {
-		tree op0 = gimple_assign_rhs1 (stmt);
-		res_op->set_op (TREE_CODE (op0), type, valueize_op (op0));
-		return true;
-	      }
-	    break;
-	  case GIMPLE_UNARY_RHS:
-	    {
-	      tree rhs1 = gimple_assign_rhs1 (stmt);
-	      res_op->set_op (code, type, valueize_op (rhs1));
-	      return true;
-	    }
-	  case GIMPLE_BINARY_RHS:
-	    {
-	      tree rhs1 = valueize_op (gimple_assign_rhs1 (stmt));
-	      tree rhs2 = valueize_op (gimple_assign_rhs2 (stmt));
-	      res_op->set_op (code, type, rhs1, rhs2);
-	      return true;
-	    }
-	  case GIMPLE_TERNARY_RHS:
-	    {
-	      tree rhs1 = gimple_assign_rhs1 (stmt);
-	      if (code == COND_EXPR && COMPARISON_CLASS_P (rhs1))
-		rhs1 = valueize_condition (rhs1);
-	      else
-		rhs1 = valueize_op (rhs1);
-	      tree rhs2 = valueize_op (gimple_assign_rhs2 (stmt));
-	      tree rhs3 = valueize_op (gimple_assign_rhs3 (stmt));
-	      res_op->set_op (code, type, rhs1, rhs2, rhs3);
-	      return true;
-	    }
-	  default:
-	    gcc_unreachable ();
-	  }
-	break;
-      }
-
-    case GIMPLE_CALL:
-      /* ???  This way we can't simplify calls with side-effects.  */
-      if (gimple_call_lhs (stmt) != NULL_TREE
-	  && gimple_call_num_args (stmt) >= 1
-	  && gimple_call_num_args (stmt) <= 5)
-	{
-	  combined_fn cfn;
-	  if (gimple_call_internal_p (stmt))
-	    cfn = as_combined_fn (gimple_call_internal_fn (stmt));
-	  else
-	    {
-	      tree fn = gimple_call_fn (stmt);
-	      if (!fn)
-		return false;
-
-	      fn = valueize_op (fn);
-	      if (TREE_CODE (fn) != ADDR_EXPR
-		  || TREE_CODE (TREE_OPERAND (fn, 0)) != FUNCTION_DECL)
-		return false;
-
-	      tree decl = TREE_OPERAND (fn, 0);
-	      if (DECL_BUILT_IN_CLASS (decl) != BUILT_IN_NORMAL
-		  || !gimple_builtin_call_types_compatible_p (stmt, decl))
-		return false;
-
-	      cfn = as_combined_fn (DECL_FUNCTION_CODE (decl));
-	    }
-
-	  unsigned int num_args = gimple_call_num_args (stmt);
-	  res_op->set_op (cfn, TREE_TYPE (gimple_call_lhs (stmt)), num_args);
-	  for (unsigned i = 0; i < num_args; ++i)
-	    res_op->ops[i] = valueize_op (gimple_call_arg (stmt, i));
-	  return true;
-	}
-      break;
-
-    case GIMPLE_COND:
-      {
-	tree lhs = valueize_op (gimple_cond_lhs (stmt));
-	tree rhs = valueize_op (gimple_cond_rhs (stmt));
-	res_op->set_op (gimple_cond_code (stmt), boolean_type_node, lhs, rhs);
-	return true;
-      }
-
-    default:
-      break;
-    }
-
-  return false;
-}
-
-/* Try to describe STMT in RES_OP, returning true on success.
-   For GIMPLE_CONDs, describe the condition that is being tested.
-   For GIMPLE_ASSIGNs, describe the rhs of the assignment.
-   For GIMPLE_CALLs, describe the call.  */
-
-bool
-gimple_extract_op (gimple *stmt, gimple_match_op *res_op)
-{
-  auto nop = [](tree op) { return op; };
-  return gimple_extract (stmt, res_op, nop, nop);
-}
-
-/* The main STMT based simplification entry.  It is used by the fold_stmt
-   and the fold_stmt_to_constant APIs.  */
-
-bool
-gimple_simplify (gimple *stmt, gimple_match_op *res_op, gimple_seq *seq,
-		 tree (*valueize)(tree), tree (*top_valueize)(tree))
-{
-  bool valueized = false;
-  auto valueize_op = [&](tree op)
-    {
-      return do_valueize (op, top_valueize, valueized);
-    };
-  auto valueize_condition = [&](tree op) -> tree
-    {
-      bool cond_valueized = false;
-      tree lhs = do_valueize (TREE_OPERAND (op, 0), top_valueize,
-			      cond_valueized);
-      tree rhs = do_valueize (TREE_OPERAND (op, 1), top_valueize,
-			      cond_valueized);
-      gimple_match_op res_op2 (res_op->cond, TREE_CODE (op),
-			       TREE_TYPE (op), lhs, rhs);
-      if ((gimple_resimplify2 (seq, &res_op2, valueize)
-	   || cond_valueized)
-	  && res_op2.code.is_tree_code ())
-	{
-	  auto code = tree_code (res_op2.code);
-	  if (TREE_CODE_CLASS (code) == tcc_comparison)
-	    {
-	      valueized = true;
-	      return build2 (code, TREE_TYPE (op),
-			     res_op2.ops[0], res_op2.ops[1]);
-	    }
-	  else if (code == SSA_NAME
-		   || code == INTEGER_CST
-		   || code == VECTOR_CST)
-	    {
-	      valueized = true;
-	      return res_op2.ops[0];
-	    }
-	}
-      return valueize_op (op);
-    };
-
-  if (!gimple_extract (stmt, res_op, valueize_op, valueize_condition))
-    return false;
-
-  if (res_op->code.is_internal_fn ())
-    {
-      internal_fn ifn = internal_fn (res_op->code);
-      if (try_conditional_simplification (ifn, res_op, seq, valueize))
-	return true;
-    }
-
-  if (!res_op->reverse
-      && res_op->num_ops
-      && res_op->resimplify (seq, valueize))
-    return true;
-
-  return valueized;
-}
-
-/* Helper for the autogenerated code, valueize OP.  */
-
-inline tree
-do_valueize (tree (*valueize)(tree), tree op)
-{
-  if (valueize && TREE_CODE (op) == SSA_NAME)
-    {
-      tree tem = valueize (op);
-      if (tem)
-	return tem;
-    }
-  return op;
-}
-
 /* Helper for the autogenerated code, get at the definition of NAME when
    VALUEIZE allows that.  */
 
@@ -1306,112 +222,8 @@ optimize_successive_divisions_p (tree divisor, tree inner_div)
   return true;
 }
 
-/* Return a canonical form for CODE when operating on TYPE.  The idea
-   is to remove redundant ways of representing the same operation so
-   that code_helpers can be hashed and compared for equality.
-
-   The only current canonicalization is to replace built-in functions
-   with internal functions, in cases where internal-fn.def defines
-   such an internal function.
-
-   Note that the new code_helper cannot necessarily be used in place of
-   the original code_helper.  For example, the new code_helper might be
-   an internal function that the target does not support.  */
-
-code_helper
-canonicalize_code (code_helper code, tree type)
-{
-  if (code.is_fn_code ())
-    return associated_internal_fn (combined_fn (code), type);
-  return code;
-}
-
-/* Return true if CODE is a binary operation and if CODE is commutative when
-   operating on type TYPE.  */
-
-bool
-commutative_binary_op_p (code_helper code, tree type)
-{
-  if (code.is_tree_code ())
-    return commutative_tree_code (tree_code (code));
-  auto cfn = combined_fn (code);
-  return commutative_binary_fn_p (associated_internal_fn (cfn, type));
-}
-
-/* Return true if CODE represents a ternary operation and if the first two
-   operands are commutative when CODE is operating on TYPE.  */
-
-bool
-commutative_ternary_op_p (code_helper code, tree type)
-{
-  if (code.is_tree_code ())
-    return commutative_ternary_tree_code (tree_code (code));
-  auto cfn = combined_fn (code);
-  return commutative_ternary_fn_p (associated_internal_fn (cfn, type));
-}
-
-/* If CODE is commutative in two consecutive operands, return the
-   index of the first, otherwise return -1.  */
-
-int
-first_commutative_argument (code_helper code, tree type)
-{
-  if (code.is_tree_code ())
-    {
-      auto tcode = tree_code (code);
-      if (commutative_tree_code (tcode)
-	  || commutative_ternary_tree_code (tcode))
-	return 0;
-      return -1;
-    }
-  auto cfn = combined_fn (code);
-  return first_commutative_argument (associated_internal_fn (cfn, type));
-}
-
-/* Return true if CODE is a binary operation that is associative when
-   operating on type TYPE.  */
-
-bool
-associative_binary_op_p (code_helper code, tree type)
-{
-  if (code.is_tree_code ())
-    return associative_tree_code (tree_code (code));
-  auto cfn = combined_fn (code);
-  return associative_binary_fn_p (associated_internal_fn (cfn, type));
-}
-
-/* Return true if the target directly supports operation CODE on type TYPE.
-   QUERY_TYPE acts as for optab_for_tree_code.  */
-
-bool
-directly_supported_p (code_helper code, tree type, optab_subtype query_type)
-{
-  if (code.is_tree_code ())
-    {
-      direct_optab optab = optab_for_tree_code (tree_code (code), type,
-						query_type);
-      return (optab != unknown_optab
-	      && optab_handler (optab, TYPE_MODE (type)) != CODE_FOR_nothing);
-    }
-  gcc_assert (query_type == optab_default
-	      || (query_type == optab_vector && VECTOR_TYPE_P (type))
-	      || (query_type == optab_scalar && !VECTOR_TYPE_P (type)));
-  internal_fn ifn = associated_internal_fn (combined_fn (code), type);
-  return (direct_internal_fn_p (ifn)
-	  && direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED));
-}
-
-/* A wrapper around the internal-fn.cc versions of get_conditional_internal_fn
-   for a code_helper CODE operating on type TYPE.  */
-
-internal_fn
-get_conditional_internal_fn (code_helper code, tree type)
-{
-  if (code.is_tree_code ())
-    return get_conditional_internal_fn (tree_code (code));
-  auto cfn = combined_fn (code);
-  return get_conditional_internal_fn (associated_internal_fn (cfn, type));
-}
+/* A wrapper for debug functions to minimize parsing overhead in the generated
+   files.  */
 
 /* Helper method for debug printing to reducing string parsing overhead.  Keep
    in sync with version in generic-match-head.cc.  */
diff --git a/gcc/gimple-match.h b/gcc/gimple-match.h
index b20585dca4b0204ec4b332a89c2737a82d3f7d4d..969aecf2d2bfef29abd1888c6dbad76465223e97 100644
--- a/gcc/gimple-match.h
+++ b/gcc/gimple-match.h
@@ -314,11 +314,36 @@ gimple_simplified_result_is_gimple_val (const gimple_match_op *op)
 	  && is_gimple_val (op->ops[0]));
 }
 
-extern tree (*mprts_hook) (gimple_match_op *);
-
 bool gimple_extract_op (gimple *, gimple_match_op *);
 bool gimple_simplify (gimple *, gimple_match_op *, gimple_seq *,
-		      tree (*)(tree), tree (*)(tree));
+		     tree (*)(tree), tree (*)(tree));
+bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
+		     code_helper, tree, tree);
+bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
+		     code_helper, tree, tree, tree);
+bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
+		     code_helper, tree, tree, tree, tree);
+bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
+		     code_helper, tree, tree, tree, tree, tree);
+bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree),
+		     code_helper, tree, tree, tree, tree, tree, tree);
+
+/* Functions that are needed by gimple-match but that are exported and used in
+   other places in the compiler.  */
+
+tree gimple_simplify (enum tree_code, tree, tree, gimple_seq *,
+		     tree (*)(tree));
+tree gimple_simplify (enum tree_code, tree, tree, tree, gimple_seq *,
+		     tree (*)(tree));
+tree gimple_simplify (enum tree_code, tree, tree, tree, tree, gimple_seq *,
+		     tree (*)(tree));
+tree gimple_simplify (combined_fn, tree, tree, gimple_seq *,
+		     tree (*)(tree));
+tree gimple_simplify (combined_fn, tree, tree, tree, gimple_seq *,
+		     tree (*)(tree));
+tree gimple_simplify (combined_fn, tree, tree, tree, tree, gimple_seq *,
+		     tree (*)(tree));
+
 tree maybe_push_res_to_seq (gimple_match_op *, gimple_seq *,
 			    tree res = NULL_TREE);
 void maybe_build_generic_op (gimple_match_op *);
@@ -328,6 +353,8 @@ bool commutative_ternary_op_p (code_helper, tree);
 int first_commutative_argument (code_helper, tree);
 bool associative_binary_op_p (code_helper, tree);
 code_helper canonicalize_code (code_helper, tree);
+tree do_valueize (tree, tree (*)(tree), bool &);
+tree do_valueize (tree (*)(tree), tree);
 
 #ifdef GCC_OPTABS_TREE_H
 bool directly_supported_p (code_helper, tree, optab_subtype = optab_default);
@@ -335,4 +362,7 @@ bool directly_supported_p (code_helper, tree, optab_subtype = optab_default);
 
 internal_fn get_conditional_internal_fn (code_helper, tree);
 
+extern tree (*mprts_hook) (gimple_match_op *);
+
+
 #endif  /* GCC_GIMPLE_MATCH_H */