[PATCH 1/2] Handle component_ref to a structre/union field including flexible array member [PR101832]

[Qing Zhao <qing.zhao@oracle.com>]

GCC extension accepts the case when a struct with a flexible array member
is embedded into another struct (possibly recursively).
__builtin_object_size should treat such struct as flexible size per
-fstrict-flex-arrays.

	PR tree-optimization/101832

gcc/ChangeLog:

	PR tree-optimization/101832
	* tree-object-size.cc (flexible_size_type_p): New function.
	(addr_object_size): Handle structure/union type when it has
	flexible size.

gcc/testsuite/ChangeLog:

	PR tree-optimization/101832
	* gcc.dg/builtin-object-size-pr101832-2.c: New test.
	* gcc.dg/builtin-object-size-pr101832-3.c: New test.
	* gcc.dg/builtin-object-size-pr101832-4.c: New test.
	* gcc.dg/builtin-object-size-pr101832.c: New test.
---
 .../gcc.dg/builtin-object-size-pr101832-2.c   | 135 ++++++++++++++++++
 .../gcc.dg/builtin-object-size-pr101832-3.c   | 135 ++++++++++++++++++
 .../gcc.dg/builtin-object-size-pr101832-4.c   | 135 ++++++++++++++++++
 .../gcc.dg/builtin-object-size-pr101832.c     | 119 +++++++++++++++
 gcc/tree-object-size.cc                       | 115 +++++++++++----
 5 files changed, 611 insertions(+), 28 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/builtin-object-size-pr101832-2.c
 create mode 100644 gcc/testsuite/gcc.dg/builtin-object-size-pr101832-3.c
 create mode 100644 gcc/testsuite/gcc.dg/builtin-object-size-pr101832-4.c
 create mode 100644 gcc/testsuite/gcc.dg/builtin-object-size-pr101832.c

diff --git a/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-2.c b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-2.c
new file mode 100644
index 00000000000..f38babc5415
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-2.c
@@ -0,0 +1,135 @@
+/* PR 101832: 
+   GCC extension accepts the case when a struct with a flexible array member
+   is embedded into another struct (possibly recursively).
+   __builtin_object_size will treat such struct as flexible size per
+   -fstrict-flex-arrays.  */ 
+/* { dg-do run } */
+/* { dg-options "-O2 -fstrict-flex-arrays=1" } */
+
+#include <stdio.h>
+
+unsigned n_fails = 0;
+
+#define expect(p, _v) do { \
+  size_t v = _v; \
+  if (p == v) \
+    printf("ok:  %s == %zd\n", #p, p); \
+  else {\
+    printf("WAT: %s == %zd (expected %zd)\n", #p, p, v); \
+    n_fails++; \
+  } \
+} while (0);
+
+struct A {
+  int n;
+  char data[];/* Content following header */
+};
+
+struct B {
+  int m;
+  struct A a;
+};
+
+struct C {
+  int q;
+  struct B b;
+};
+
+struct A0 {
+  int n;
+  char data[0];/* Content following header */
+};
+
+struct B0 {
+  int m;
+  struct A0 a;
+};
+
+struct C0 {
+  int q;
+  struct B0 b;
+};
+
+struct A1 {
+  int n;
+  char data[1];/* Content following header */
+};
+
+struct B1 {
+  int m;
+  struct A1 a;
+};
+
+struct C1 {
+  int q;
+  struct B1 b;
+};
+
+struct An {
+  int n;
+  char data[8];/* Content following header */
+};
+
+struct Bn {
+  int m;
+  struct An a;
+};
+
+struct Cn {
+  int q;
+  struct Bn b;
+};
+
+volatile void *magic1, *magic2;
+
+int main(int argc, char *argv[])
+{
+    struct B *outer;
+    struct C *outest;
+
+    /* Make sure optimization can't find some other object size. */
+    outer = (void *)magic1;
+    outest = (void *)magic2;
+
+    expect(__builtin_object_size(&outer->a, 1), -1);
+    expect(__builtin_object_size(&outest->b, 1), -1);
+    expect(__builtin_object_size(&outest->b.a, 1), -1);
+
+    struct B0 *outer0;
+    struct C0 *outest0;
+
+    /* Make sure optimization can't find some other object size. */
+    outer0 = (void *)magic1;
+    outest0 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer0->a, 1), -1);
+    expect(__builtin_object_size(&outest0->b, 1), -1);
+    expect(__builtin_object_size(&outest0->b.a, 1), -1);
+
+    struct B1 *outer1;
+    struct C1 *outest1;
+
+    /* Make sure optimization can't find some other object size. */
+    outer1 = (void *)magic1;
+    outest1 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer1->a, 1), -1);
+    expect(__builtin_object_size(&outest1->b, 1), -1);
+    expect(__builtin_object_size(&outest1->b.a, 1), -1);
+
+    struct Bn *outern;
+    struct Cn *outestn;
+
+    /* Make sure optimization can't find some other object size. */
+    outern = (void *)magic1;
+    outestn = (void *)magic2;
+
+    expect(__builtin_object_size(&outern->a, 1), sizeof(outern->a));
+    expect(__builtin_object_size(&outestn->b, 1), sizeof(outestn->b));
+    expect(__builtin_object_size(&outestn->b.a, 1), sizeof(outestn->b.a));
+
+    if (n_fails > 0)
+      __builtin_abort ();
+
+    return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-3.c b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-3.c
new file mode 100644
index 00000000000..aaae99b8d67
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-3.c
@@ -0,0 +1,135 @@
+/* PR 101832: 
+   GCC extension accepts the case when a struct with a flexible array member
+   is embedded into another struct (possibly recursively).
+   __builtin_object_size will treat such struct as flexible size per
+   -fstrict-flex-arrays.  */
+/* { dg-do run } */
+/* { dg-options "-O2 -fstrict-flex-arrays=2" } */
+
+#include <stdio.h>
+
+unsigned n_fails = 0;
+
+#define expect(p, _v) do { \
+  size_t v = _v; \
+  if (p == v) \
+    printf("ok:  %s == %zd\n", #p, p); \
+  else {\
+    printf("WAT: %s == %zd (expected %zd)\n", #p, p, v); \
+    n_fails++; \
+  } \
+} while (0);
+
+struct A {
+  int n;
+  char data[];/* Content following header */
+};
+
+struct B {
+  int m;
+  struct A a;
+};
+
+struct C {
+  int q;
+  struct B b;
+};
+
+struct A0 {
+  int n;
+  char data[0];/* Content following header */
+};
+
+struct B0 {
+  int m;
+  struct A0 a;
+};
+
+struct C0 {
+  int q;
+  struct B0 b;
+};
+
+struct A1 {
+  int n;
+  char data[1];/* Content following header */
+};
+
+struct B1 {
+  int m;
+  struct A1 a;
+};
+
+struct C1 {
+  int q;
+  struct B1 b;
+};
+
+struct An {
+  int n;
+  char data[8];/* Content following header */
+};
+
+struct Bn {
+  int m;
+  struct An a;
+};
+
+struct Cn {
+  int q;
+  struct Bn b;
+};
+
+volatile void *magic1, *magic2;
+
+int main(int argc, char *argv[])
+{
+    struct B *outer;
+    struct C *outest;
+
+    /* Make sure optimization can't find some other object size. */
+    outer = (void *)magic1;
+    outest = (void *)magic2;
+
+    expect(__builtin_object_size(&outer->a, 1), -1);
+    expect(__builtin_object_size(&outest->b, 1), -1);
+    expect(__builtin_object_size(&outest->b.a, 1), -1);
+
+    struct B0 *outer0;
+    struct C0 *outest0;
+
+    /* Make sure optimization can't find some other object size. */
+    outer0 = (void *)magic1;
+    outest0 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer0->a, 1), -1);
+    expect(__builtin_object_size(&outest0->b, 1), -1);
+    expect(__builtin_object_size(&outest0->b.a, 1), -1);
+
+    struct B1 *outer1;
+    struct C1 *outest1;
+
+    /* Make sure optimization can't find some other object size. */
+    outer1 = (void *)magic1;
+    outest1 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer1->a, 1), sizeof(outer1->a));
+    expect(__builtin_object_size(&outest1->b, 1), sizeof(outest1->b));
+    expect(__builtin_object_size(&outest1->b.a, 1), sizeof(outest1->b.a));
+
+    struct Bn *outern;
+    struct Cn *outestn;
+
+    /* Make sure optimization can't find some other object size. */
+    outern = (void *)magic1;
+    outestn = (void *)magic2;
+
+    expect(__builtin_object_size(&outern->a, 1), sizeof(outern->a));
+    expect(__builtin_object_size(&outestn->b, 1), sizeof(outestn->b));
+    expect(__builtin_object_size(&outestn->b.a, 1), sizeof(outestn->b.a));
+
+    if (n_fails > 0)
+      __builtin_abort ();
+
+    return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-4.c b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-4.c
new file mode 100644
index 00000000000..424264e2acd
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832-4.c
@@ -0,0 +1,135 @@
+/* PR 101832: 
+   GCC extension accepts the case when a struct with a flexible array member
+   is embedded into another struct (possibly recursively).
+   __builtin_object_size will treat such struct as flexible size per
+   -fstrict-flex-arrays.  */
+/* { dg-do run } */
+/* { dg-options "-O2 -fstrict-flex-arrays=3" } */
+
+#include <stdio.h>
+
+unsigned n_fails = 0;
+
+#define expect(p, _v) do { \
+  size_t v = _v; \
+  if (p == v) \
+    printf("ok:  %s == %zd\n", #p, p); \
+  else {\
+    printf("WAT: %s == %zd (expected %zd)\n", #p, p, v); \
+    n_fails++; \
+  } \
+} while (0);
+
+struct A {
+  int n;
+  char data[];/* Content following header */
+};
+
+struct B {
+  int m;
+  struct A a;
+};
+
+struct C {
+  int q;
+  struct B b;
+};
+
+struct A0 {
+  int n;
+  char data[0];/* Content following header */
+};
+
+struct B0 {
+  int m;
+  struct A0 a;
+};
+
+struct C0 {
+  int q;
+  struct B0 b;
+};
+
+struct A1 {
+  int n;
+  char data[1];/* Content following header */
+};
+
+struct B1 {
+  int m;
+  struct A1 a;
+};
+
+struct C1 {
+  int q;
+  struct B1 b;
+};
+
+struct An {
+  int n;
+  char data[8];/* Content following header */
+};
+
+struct Bn {
+  int m;
+  struct An a;
+};
+
+struct Cn {
+  int q;
+  struct Bn b;
+};
+
+volatile void *magic1, *magic2;
+
+int main(int argc, char *argv[])
+{
+    struct B *outer;
+    struct C *outest;
+
+    /* Make sure optimization can't find some other object size. */
+    outer = (void *)magic1;
+    outest = (void *)magic2;
+
+    expect(__builtin_object_size(&outer->a, 1), -1);
+    expect(__builtin_object_size(&outest->b, 1), -1);
+    expect(__builtin_object_size(&outest->b.a, 1), -1);
+
+    struct B0 *outer0;
+    struct C0 *outest0;
+
+    /* Make sure optimization can't find some other object size. */
+    outer0 = (void *)magic1;
+    outest0 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer0->a, 1), sizeof(outer0->a));
+    expect(__builtin_object_size(&outest0->b, 1), sizeof(outest0->b));
+    expect(__builtin_object_size(&outest0->b.a, 1), sizeof(outest0->b.a));
+
+    struct B1 *outer1;
+    struct C1 *outest1;
+
+    /* Make sure optimization can't find some other object size. */
+    outer1 = (void *)magic1;
+    outest1 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer1->a, 1), sizeof(outer1->a));
+    expect(__builtin_object_size(&outest1->b, 1), sizeof(outest1->b));
+    expect(__builtin_object_size(&outest1->b.a, 1), sizeof(outest1->b.a));
+
+    struct Bn *outern;
+    struct Cn *outestn;
+
+    /* Make sure optimization can't find some other object size. */
+    outern = (void *)magic1;
+    outestn = (void *)magic2;
+
+    expect(__builtin_object_size(&outern->a, 1), sizeof(outern->a));
+    expect(__builtin_object_size(&outestn->b, 1), sizeof(outestn->b));
+    expect(__builtin_object_size(&outestn->b.a, 1), sizeof(outestn->b.a));
+
+    if (n_fails > 0)
+      __builtin_abort ();
+
+    return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/builtin-object-size-pr101832.c b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832.c
new file mode 100644
index 00000000000..8ed6980edf0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/builtin-object-size-pr101832.c
@@ -0,0 +1,119 @@
+/* PR 101832: 
+   GCC extension accepts the case when a struct with a flexible array member
+   is embedded into another struct (possibly recursively).
+   __builtin_object_size will treat such struct as flexible size per
+   -fstrict-flex-arrays.  */
+/* { dg-do run } */
+/* { dg-options "-O2" } */
+
+#include <stdio.h>
+
+unsigned n_fails = 0;
+
+#define expect(p, _v) do { \
+  size_t v = _v; \
+  if (p == v) \
+    printf("ok:  %s == %zd\n", #p, p); \
+  else {\
+    printf("WAT: %s == %zd (expected %zd)\n", #p, p, v); \
+    n_fails++; \
+  } \
+} while (0);
+
+struct A {
+  int n;
+  char data[];/* Content following header */
+};
+
+struct B {
+  int m;
+  struct A a;
+};
+
+struct C {
+  int q;
+  struct B b;
+};
+
+struct A0 {
+  int n;
+  char data[0];/* Content following header */
+};
+
+struct B0 {
+  int m;
+  struct A0 a;
+};
+
+struct C0 {
+  int q;
+  struct B0 b;
+};
+
+struct A1 {
+  int n;
+  char data[1];/* Content following header */
+};
+
+struct B1 {
+  int m;
+  struct A1 a;
+};
+
+struct C1 {
+  int q;
+  struct B1 b;
+};
+
+struct An {
+  int n;
+  char data[8];/* Content following header */
+};
+
+struct Bn {
+  int m;
+  struct An a;
+};
+
+struct Cn {
+  int q;
+  struct Bn b;
+};
+
+volatile void *magic1, *magic2;
+
+int main(int argc, char *argv[])
+{
+    struct B *outer = (void *)magic1;
+    struct C *outest = (void *)magic2;
+
+    expect(__builtin_object_size(&outer->a, 1), -1);
+    expect(__builtin_object_size(&outest->b, 1), -1);
+    expect(__builtin_object_size(&outest->b.a, 1), -1);
+
+    struct B0 *outer0 = (void *)magic1;
+    struct C0 *outest0 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer0->a, 1), -1);
+    expect(__builtin_object_size(&outest0->b, 1), -1);
+    expect(__builtin_object_size(&outest0->b.a, 1), -1);
+
+    struct B1 *outer1 = (void *)magic1;
+    struct C1 *outest1 = (void *)magic2;
+
+    expect(__builtin_object_size(&outer1->a, 1), -1);
+    expect(__builtin_object_size(&outest1->b, 1), -1);
+    expect(__builtin_object_size(&outest1->b.a, 1), -1);
+
+    struct Bn *outern = (void *)magic1;
+    struct Cn *outestn = (void *)magic2;
+
+    expect(__builtin_object_size(&outern->a, 1), -1);
+    expect(__builtin_object_size(&outestn->b, 1), -1);
+    expect(__builtin_object_size(&outestn->b.a, 1), -1);
+
+    if (n_fails > 0)
+      __builtin_abort ();
+
+    return 0;
+}
diff --git a/gcc/tree-object-size.cc b/gcc/tree-object-size.cc
index 9a936a91983..56b78ca2a8c 100644
--- a/gcc/tree-object-size.cc
+++ b/gcc/tree-object-size.cc
@@ -500,6 +500,42 @@ decl_init_size (tree decl, bool min)
   return size;
 }
 
+/* Determine whether TYPE is a structure with a flexible array member
+   per -fstrict-flex-array or a union containing such a structure
+   (possibly recursively).  */
+static bool
+flexible_size_type_p (const_tree type)
+{
+  tree x = NULL_TREE;
+  tree last = NULL_TREE;
+  switch (TREE_CODE (type))
+    {
+    case RECORD_TYPE:
+      for (x = TYPE_FIELDS (type); x != NULL_TREE; x = DECL_CHAIN (x))
+	if (TREE_CODE (x) == FIELD_DECL)
+	  last = x;
+      if (last == NULL_TREE)
+	return false;
+      if (TREE_CODE (TREE_TYPE (last)) == ARRAY_TYPE
+	  && !DECL_NOT_FLEXARRAY (last))
+	return true;
+      else if (TREE_CODE (TREE_TYPE (last)) == RECORD_TYPE
+	       || TREE_CODE (TREE_TYPE (last)) == UNION_TYPE)
+	return flexible_size_type_p (TREE_TYPE (last));
+      return false;
+    case UNION_TYPE:
+      for (x = TYPE_FIELDS (type); x != NULL_TREE; x = DECL_CHAIN (x))
+	{
+	  if (TREE_CODE (x) == FIELD_DECL
+	      && flexible_array_type_p (TREE_TYPE (x)))
+	    return true;
+	}
+      return false;
+    default:
+      return false;
+  }
+}
+
 /* Compute __builtin_object_size for PTR, which is a ADDR_EXPR.
    OBJECT_SIZE_TYPE is the second argument from __builtin_object_size.
    If unknown, return size_unknown (object_size_type).  */
@@ -633,45 +669,68 @@ addr_object_size (struct object_size_info *osi, const_tree ptr,
 		    v = NULL_TREE;
 		    break;
 		  case COMPONENT_REF:
-		    if (TREE_CODE (TREE_TYPE (v)) != ARRAY_TYPE)
+		    /* When the ref is not to an array, a record or a union, it
+		       will not have flexible size, compute the object size
+		       directly.  */
+		    if ((TREE_CODE (TREE_TYPE (v)) != ARRAY_TYPE)
+			&& (TREE_CODE (TREE_TYPE (v)) != RECORD_TYPE)
+			&& (TREE_CODE (TREE_TYPE (v)) != UNION_TYPE))
 		      {
 			v = NULL_TREE;
 			break;
 		      }
-		    is_flexible_array_mem_ref = array_ref_flexible_size_p (v);
-		    while (v != pt_var && TREE_CODE (v) == COMPONENT_REF)
-		      if (TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
-			  != UNION_TYPE
-			  && TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
-			  != QUAL_UNION_TYPE)
-			break;
-		      else
-			v = TREE_OPERAND (v, 0);
-		    if (TREE_CODE (v) == COMPONENT_REF
-			&& TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
-			   == RECORD_TYPE)
+		    /* if the record or union does not have flexible size
+		       compute the object size directly.  */
+		    if (TREE_CODE (TREE_TYPE (v)) == RECORD_TYPE
+			|| TREE_CODE (TREE_TYPE (v)) == UNION_TYPE)
 		      {
-			/* compute object size only if v is not a
-			   flexible array member.  */
-			if (!is_flexible_array_mem_ref)
+			if (!flexible_size_type_p (TREE_TYPE (v)))
 			  {
 			    v = NULL_TREE;
 			    break;
 			  }
-			v = TREE_OPERAND (v, 0);
+			else
+			  v = TREE_OPERAND (v, 0);
 		      }
-		    while (v != pt_var && TREE_CODE (v) == COMPONENT_REF)
-		      if (TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
-			  != UNION_TYPE
-			  && TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
-			  != QUAL_UNION_TYPE)
-			break;
-		      else
-			v = TREE_OPERAND (v, 0);
-		    if (v != pt_var)
-		      v = NULL_TREE;
 		    else
-		      v = pt_var;
+		      {
+			/* Now the ref is to an array type.  */
+			is_flexible_array_mem_ref
+			  = array_ref_flexible_size_p (v);
+			while (v != pt_var && TREE_CODE (v) == COMPONENT_REF)
+			if (TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
+			      != UNION_TYPE
+			    && TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
+				 != QUAL_UNION_TYPE)
+			  break;
+			else
+			  v = TREE_OPERAND (v, 0);
+			if (TREE_CODE (v) == COMPONENT_REF
+			    && TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
+				 == RECORD_TYPE)
+			  {
+			    /* compute object size only if v is not a
+			       flexible array member.  */
+			    if (!is_flexible_array_mem_ref)
+			      {
+				v = NULL_TREE;
+				break;
+			      }
+			    v = TREE_OPERAND (v, 0);
+			  }
+			while (v != pt_var && TREE_CODE (v) == COMPONENT_REF)
+			  if (TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
+				!= UNION_TYPE
+			      && TREE_CODE (TREE_TYPE (TREE_OPERAND (v, 0)))
+				   != QUAL_UNION_TYPE)
+			    break;
+			  else
+			    v = TREE_OPERAND (v, 0);
+			if (v != pt_var)
+			  v = NULL_TREE;
+			else
+			  v = pt_var;
+		      }
 		    break;
 		  default:
 		    v = pt_var;
-- 
2.31.1