PATCH: Altivec context-sensitive keywords

PATCH: Altivec context-sensitive keywords

[Ben Elliston]

This patch is a revised attempt to implement context-sensitive keywords
for the PowerPC port.  There are some libcpp changes, some rs6000
backend changes, and some minor testsuite changes.

To overcome many of the concerns about the interaction of
context-sensitive keywords with ISO C compliance, the feature is now
controlled by !flag_iso.  When compiling with -maltivec, the
context-sensitive keyword support is enabled (and the compiler defines
__APPLE_ALTIVEC__, as required by the PIM).  When _APPLE_ALTIVEC__ is
defined, <altivec.h> will suppress the macros that "implement" these
keywords.

With -maltivec -ansi, the context-sensitive keywords are disabled and to
use the Altivec keywords, you must resort to #including <altivec.h> (the
predefine method).

The two testsuite changes in gcc.dg/vmx are made due to the reasoning
stated here:
  http://gcc.gnu.org/ml/gcc-patches/2008-04/msg01616.html

Rather than remove the use of bool variables from this test, the tests
are now compiled with -ansi to ensure that the predefine method is used.

Tested with a bootstrap on powerpc64-linux and a regression testsuite
run on powerpc-linux and powerpc64-linux.  Okay for mainline?

Cheers, Ben

gcc/
        * coretypes.h (struct cpp_token): Forward declare.
        * doc/extend.texi (PowerPC AltiVec Built-in Functions): Document
        the context-sensitive keyword method.
        * config/rs6000/rs6000-c.c (__vector_keyword, vector_keyword,
        __pixel_keyword, pixel_keyword, __bool_keyword, bool_keyword,
        expand_bool_pixel): New cpp_hashnodes.
        (altivec_categorize_keyword): New function.
        (DSC): Convenience macro, borrowed from libcpp.
        (init_vector_keywords): New function.
        (rs6000_macro_to_expand): Likewise.
        (rs6000_cpu_cpp_builtins): Enable context-sensitive macros if not
        compiling an ISO C dialect.

gcc/testsuite/
        * gcc.target/powerpc/altivec-macros.c: New test.
        * gcc.target/powerpc/altviec-26.c: Likewise.
        * gcc.dg/vmx/1b-06.c: Pass -ansi -maltivec options.
        * gcc.dg/vmx/1b-07.c: Likewise.

libcpp/
        * include/cpplib.h (NODE_CONDITIONAL): New.
        (struct cpp_callbacks): New macro_to_expand field.
        (struct cpp_hashnode): Increase CPP flags from char to int.
        (_cpp_peek_token, _cpp_backup_tokens_direct): Prototype.
        lex.c (_cpp_peek_token): New function.
        (_cpp_temp_token): Protect pre-existing lookaheads.
        * macro.c (cpp_get_token): Expand any conditional macros.
        (_cpp_backup_tokens_direct): New.
        (_cpp_backup_tokens): Call _cpp_backup_tokens_direct.
        (warn_of_redefinition): Silently allow redefined conditional
        macros.
        (_cpp_create_definition): Remove the conditional flag when a user
        defines one of the conditional macros.

Index: gcc/doc/extend.texi
===================================================================
--- gcc/doc/extend.texi	(revision 137138)
+++ gcc/doc/extend.texi	(working copy)
@@ -9117,9 +9117,10 @@ always specify the signedness.
 
 @item
 Compiling with @option{-maltivec} adds keywords @code{__vector},
-@code{__pixel}, and @code{__bool}.  Macros @option{vector},
-@code{pixel}, and @code{bool} are defined in @code{<altivec.h>} and can
-be undefined.
+@code{vector}, @code{__pixel}, @code{pixel}, @code{__bool} and
+@code{bool}.  When compiling ISO C, the context-sensitive substitution
+of the keywords @code{vector}, @code{pixel} and @code{bool} is
+disabled.  To use them, you must include @code{<altivec.h>} instead.
 
 @item
 GCC allows using a @code{typedef} name as the type specifier for a
Index: gcc/testsuite/gcc.target/powerpc/altivec-macros.c
===================================================================
--- gcc/testsuite/gcc.target/powerpc/altivec-macros.c	(revision 0)
+++ gcc/testsuite/gcc.target/powerpc/altivec-macros.c	(revision 0)
@@ -0,0 +1,59 @@
+/* Copyright (C) 2007 Free Software Foundation, Inc.  */
+
+/* { dg-do preprocess } */
+/* { dg-options "-maltivec" } */
+
+/* Redefinition of conditional macros.  */
+/* No warning should be generated.  */
+
+#define __vector __new_vector
+#define __pixel __new_pixel
+#define __bool __new_bool
+#define vector new_vector
+#define pixel new_pixel
+#define bool new_bool
+
+/* Definition of conditional macros.  */
+/* No warning should be generated.  */
+
+#undef __vector
+#define __vector __new_vector
+
+#undef __pixel
+#define __pixel __new_pixel
+
+#undef __bool
+#define __bool __new_bool
+
+#undef vector
+#define vector new_vector
+
+#undef pixel
+#define pixel new_pixel
+
+#undef bool
+#define bool new_bool
+
+/* Re-definition of "unconditional" macros.  */
+/* Warnings should be generated as usual.  */
+
+#define __vector	__newer_vector
+#define __pixel		__newer_pixel
+#define __bool		__newer_bool
+#define vector		newer_vector
+#define pixel		newer_pixel
+#define bool		newer_bool
+
+/* { dg-warning "redefined" "__vector redefined"  { target *-*-* } 40 } */
+/* { dg-warning "redefined" "__pixel redefined"   { target *-*-* } 41 } */
+/* { dg-warning "redefined" "__bool redefined"    { target *-*-* } 42 } */
+/* { dg-warning "redefined" "vector redefined"    { target *-*-* } 43 } */
+/* { dg-warning "redefined" "pixel redefined"     { target *-*-* } 44 } */
+/* { dg-warning "redefined" "bool redefined"      { target *-*-* } 45 } */
+
+/* { dg-warning "previous"  "prev __vector defn"  { target *-*-* } 20 } */
+/* { dg-warning "previous"  "prev __pixel defn"   { target *-*-* } 23 } */
+/* { dg-warning "previous"  "prev __bool defn"    { target *-*-* } 26 } */
+/* { dg-warning "previous"  "prev vector defn"    { target *-*-* } 29 } */
+/* { dg-warning "previous"  "prev pixel defn"     { target *-*-* } 32 } */
+/* { dg-warning "previous"  "prev bool defn"      { target *-*-* } 35 } */
Index: gcc/testsuite/gcc.target/powerpc/altivec-26.c
===================================================================
--- gcc/testsuite/gcc.target/powerpc/altivec-26.c	(revision 0)
+++ gcc/testsuite/gcc.target/powerpc/altivec-26.c	(revision 0)
@@ -0,0 +1,11 @@
+/* { dg-do compile { target powerpc*-*-* } } */
+/* { dg-require-effective-target powerpc_altivec_ok } */
+/* { dg-options "-maltivec" } */
+
+/* A compiler implementing context-sensitive keywords must define this
+   preprocessor macro so that altivec.h does not provide the vector,
+   pixel, etc. macros.  */
+
+#ifndef __APPLE_ALTIVEC__
+#error __APPLE_ALTIVEC__ not pre-defined
+#endif
Index: gcc/testsuite/gcc.dg/vmx/1b-07.c
===================================================================
--- gcc/testsuite/gcc.dg/vmx/1b-07.c	(revision 137138)
+++ gcc/testsuite/gcc.dg/vmx/1b-07.c	(working copy)
@@ -1,4 +1,6 @@
 /* { dg-do compile } */
+/* { dg-options "-ansi -maltivec" } */
+
 #include <altivec.h>
 vector char bool _0 ;
 vector bool char _8 ;
Index: gcc/testsuite/gcc.dg/vmx/1b-06.c
===================================================================
--- gcc/testsuite/gcc.dg/vmx/1b-06.c	(revision 137138)
+++ gcc/testsuite/gcc.dg/vmx/1b-06.c	(working copy)
@@ -1,4 +1,6 @@
 /* { dg-do compile } */
+/* { dg-options "-ansi -maltivec" } */
+
 #include <altivec.h>
 vector char bool _4 ;
 vector char unsigned _31 ;
Index: gcc/coretypes.h
===================================================================
--- gcc/coretypes.h	(revision 137138)
+++ gcc/coretypes.h	(working copy)
@@ -60,9 +60,11 @@ enum ir_type {
 
 /* Provide forward struct declaration so that we don't have to include
    all of cpplib.h whenever a random prototype includes a pointer.
-   Note that the cpp_reader typedef remains part of cpplib.h.  */
+   Note that the cpp_reader and cpp_token typedefs remain part of
+   cpplib.h.  */
 
 struct cpp_reader;
+struct cpp_token;
 
 /* The thread-local storage model associated with a given VAR_DECL
    or SYMBOL_REF.  This isn't used much, but both trees and RTL refer
Index: gcc/config/rs6000/rs6000-c.c
===================================================================
--- gcc/config/rs6000/rs6000-c.c	(revision 137138)
+++ gcc/config/rs6000/rs6000-c.c	(working copy)
@@ -84,6 +84,142 @@ rs6000_pragma_longcall (cpp_reader *pfil
 #define builtin_define(TXT) cpp_define (pfile, TXT)
 #define builtin_assert(TXT) cpp_assert (pfile, TXT)
 
+/* Keep the AltiVec keywords handy for fast comparisons.  */
+static GTY(()) cpp_hashnode *__vector_keyword;
+static GTY(()) cpp_hashnode *vector_keyword;
+static GTY(()) cpp_hashnode *__pixel_keyword;
+static GTY(()) cpp_hashnode *pixel_keyword;
+static GTY(()) cpp_hashnode *__bool_keyword;
+static GTY(()) cpp_hashnode *bool_keyword;
+
+static GTY(()) cpp_hashnode *expand_bool_pixel;  /* Preserved across calls.  */
+
+static cpp_hashnode *
+altivec_categorize_keyword (const cpp_token *tok)
+{
+  if (tok->type == CPP_NAME)
+    {
+      cpp_hashnode *ident = tok->val.node;
+
+      if (ident == vector_keyword || ident == __vector_keyword)
+	return __vector_keyword;
+
+      if (ident == pixel_keyword || ident ==  __pixel_keyword)
+	return __pixel_keyword;
+
+      if (ident == bool_keyword || ident == __bool_keyword)
+	return __bool_keyword;
+
+      return ident;
+    }
+
+  return 0;
+}
+
+/*  From libcpp.  */
+#define DSC(str) (const unsigned char *)str, sizeof str - 1
+
+static void
+init_vector_keywords (cpp_reader *pfile)
+{
+  /* Keywords without two leading underscores are context-sensitive,
+     and hence implemented as conditional macros, controlled by the
+     rs6000_macro_to_expand() function above.  */
+
+  __vector_keyword = cpp_lookup (pfile, DSC ("__vector"));
+  __vector_keyword->flags |= NODE_CONDITIONAL;
+
+  __pixel_keyword = cpp_lookup (pfile, DSC ("__pixel"));
+  __pixel_keyword->flags |= NODE_CONDITIONAL;
+
+  __bool_keyword = cpp_lookup (pfile, DSC ("__bool"));
+  __bool_keyword->flags |= NODE_CONDITIONAL;
+
+  vector_keyword = cpp_lookup (pfile, DSC ("vector"));
+  vector_keyword->flags |= NODE_CONDITIONAL;
+
+  pixel_keyword = cpp_lookup (pfile, DSC ("pixel"));
+  pixel_keyword->flags |= NODE_CONDITIONAL;
+
+  bool_keyword = cpp_lookup (pfile, DSC ("bool"));
+  bool_keyword->flags |= NODE_CONDITIONAL;
+}
+
+/* Called to decide whether a conditional macro should be expanded.
+   Since we have exactly one such macro (i.e, 'vector'), we do not
+   need to examine the 'tok' parameter.  */
+
+static cpp_hashnode *
+rs6000_macro_to_expand (cpp_reader *pfile, const cpp_token *tok)
+{
+  static bool vector_keywords_init = false;
+  cpp_hashnode *expand_this = tok->val.node;
+  cpp_hashnode *ident;
+
+  if (!vector_keywords_init)
+    {
+      init_vector_keywords (pfile);
+      vector_keywords_init = true;
+    }
+
+  ident = altivec_categorize_keyword (tok);
+
+  if (ident == __vector_keyword)
+    {
+      tok = _cpp_peek_token (pfile, 0);
+      ident = altivec_categorize_keyword (tok);
+
+      if (ident ==  __pixel_keyword || ident == __bool_keyword)
+	{
+	  expand_this = __vector_keyword;
+	  expand_bool_pixel = ident;
+	}
+      else if (ident)
+	{
+	  enum rid rid_code = (enum rid)(ident->rid_code);
+	  if (ident->type == NT_MACRO)
+	    {
+	      (void)cpp_get_token (pfile);
+	      tok = _cpp_peek_token (pfile, 0);
+	      ident = altivec_categorize_keyword (tok);
+	      if (ident)
+		rid_code = (enum rid)(ident->rid_code);
+	    }
+
+	  if (rid_code == RID_UNSIGNED || rid_code == RID_LONG
+	      || rid_code == RID_SHORT || rid_code == RID_SIGNED
+	      || rid_code == RID_INT || rid_code == RID_CHAR
+	      || rid_code == RID_FLOAT)
+	    {
+	      expand_this = __vector_keyword;
+	      /* If the next keyword is bool or pixel, it
+		 will need to be expanded as well.  */
+	      tok = _cpp_peek_token (pfile, 1);
+	      ident = altivec_categorize_keyword (tok);
+
+	      if (ident ==  __pixel_keyword || ident == __bool_keyword)
+		expand_bool_pixel = ident;
+	      else
+		{
+		  /* Try two tokens down, too.  */
+		  tok = _cpp_peek_token (pfile, 2);
+		  ident = altivec_categorize_keyword (tok);
+		  if (ident ==  __pixel_keyword || ident == __bool_keyword)
+		    expand_bool_pixel = ident;
+		}
+	    }
+	}
+    }
+  else if (expand_bool_pixel
+	   && (ident ==  __pixel_keyword || ident == __bool_keyword))
+    {
+      expand_this = expand_bool_pixel;
+      expand_bool_pixel = 0;
+    }
+
+  return expand_this;
+}
+
 void
 rs6000_cpu_cpp_builtins (cpp_reader *pfile)
 {
@@ -120,6 +256,20 @@ rs6000_cpu_cpp_builtins (cpp_reader *pfi
       builtin_define ("__vector=__attribute__((altivec(vector__)))");
       builtin_define ("__pixel=__attribute__((altivec(pixel__))) unsigned short");
       builtin_define ("__bool=__attribute__((altivec(bool__))) unsigned");
+
+      if (!flag_iso)
+	{
+	  /* Define this when supporting context-sensitive keywords.  */
+	  builtin_define ("__APPLE_ALTIVEC__");
+	  
+	  builtin_define ("vector=vector");
+	  builtin_define ("pixel=pixel");
+	  builtin_define ("bool=bool");
+	  init_vector_keywords (pfile);
+
+	  /* Enable context-sensitive macros.  */
+	  cpp_get_callbacks (pfile)->macro_to_expand = rs6000_macro_to_expand;
+	}
     }
   if (TARGET_SPE)
     builtin_define ("__SPE__");
Index: libcpp/macro.c
===================================================================
--- libcpp/macro.c	(revision 137138)
+++ libcpp/macro.c	(working copy)
@@ -1240,16 +1240,21 @@ cpp_get_token (cpp_reader *pfile)
 
       if (!(node->flags & NODE_DISABLED))
 	{
-	  int ret;
+	  int ret = 0;
 	  /* If not in a macro context, and we're going to start an
 	     expansion, record the location.  */
 	  if (can_set && !context->macro)
 	    pfile->invocation_location = result->src_loc;
 	  if (pfile->state.prevent_expansion)
 	    break;
-	  ret = enter_macro_context (pfile, node, result);
-	  if (ret)
-	    {
+
+	  /* Conditional macros require that a predicate be evaluated
+	     first.  */
+	  if (((!(node->flags & NODE_CONDITIONAL))
+	       || (pfile->cb.macro_to_expand
+		   && (node = pfile->cb.macro_to_expand (pfile, result))))
+	      && (ret = enter_macro_context (pfile, node, result)))
+ 	    {
 	      if (pfile->state.in_directive || ret == 2)
 		continue;
 	      return padding_token (pfile, result);
@@ -1327,26 +1332,31 @@ cpp_scan_nooutput (cpp_reader *pfile)
   pfile->state.prevent_expansion--;
 }
 
+/* Step back one or more tokens obtained from the lexer.  */
+void
+_cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)
+{
+  pfile->lookaheads += count;
+  while (count--)
+    {
+      pfile->cur_token--;
+      if (pfile->cur_token == pfile->cur_run->base
+          /* Possible with -fpreprocessed and no leading #line.  */
+          && pfile->cur_run->prev != NULL)
+        {
+          pfile->cur_run = pfile->cur_run->prev;
+          pfile->cur_token = pfile->cur_run->limit;
+        }
+    }
+}
+
 /* Step back one (or more) tokens.  Can only step back more than 1 if
    they are from the lexer, and not from macro expansion.  */
 void
 _cpp_backup_tokens (cpp_reader *pfile, unsigned int count)
 {
   if (pfile->context->prev == NULL)
-    {
-      pfile->lookaheads += count;
-      while (count--)
-	{
-	  pfile->cur_token--;
-	  if (pfile->cur_token == pfile->cur_run->base
-	      /* Possible with -fpreprocessed and no leading #line.  */
-	      && pfile->cur_run->prev != NULL)
-	    {
-	      pfile->cur_run = pfile->cur_run->prev;
-	      pfile->cur_token = pfile->cur_run->limit;
-	    }
-	}
-    }
+    _cpp_backup_tokens_direct (pfile, count);
   else
     {
       if (count != 1)
@@ -1372,6 +1382,11 @@ warn_of_redefinition (cpp_reader *pfile,
   if (node->flags & NODE_WARN)
     return true;
 
+  /* Redefinitions of conditional (context-sensitive) macros, on
+     the other hand, must be allowed silently.  */
+  if (node->flags & NODE_CONDITIONAL)
+    return false;
+
   /* Redefinition of a macro is allowed if and only if the old and new
      definitions are the same.  (6.10.3 paragraph 2).  */
   macro1 = node->value.macro;
@@ -1804,6 +1819,10 @@ _cpp_create_definition (cpp_reader *pfil
       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_CONSTANT_MACROS"))
     node->flags |= NODE_WARN;
 
+  /* If user defines one of the conditional macros, remove the
+     conditional flag */
+  node->flags &= ~NODE_CONDITIONAL;
+
   return ok;
 }
 
Index: libcpp/include/cpplib.h
===================================================================
--- libcpp/include/cpplib.h	(revision 137138)
+++ libcpp/include/cpplib.h	(working copy)
@@ -484,6 +484,10 @@ struct cpp_callbacks
   void (*read_pch) (cpp_reader *, const char *, int, const char *);
   missing_header_cb missing_header;
 
+  /* Context-sensitive macro support.  Returns macro (if any) that should
+     be expanded.  */
+  cpp_hashnode * (*macro_to_expand) (cpp_reader *, const cpp_token *);
+
   /* Called to emit a diagnostic if client_diagnostic option is true.
      This callback receives the translated message.  */
   void (*error) (cpp_reader *, int, const char *, va_list *)
@@ -558,6 +562,7 @@ extern const char *progname;
 #define NODE_DISABLED	(1 << 5)	/* A disabled macro.  */
 #define NODE_MACRO_ARG	(1 << 6)	/* Used during #define processing.  */
 #define NODE_USED	(1 << 7)	/* Dumped with -dU.  */
+#define NODE_CONDITIONAL (1 << 8)	/* Conditional macro */
 
 /* Different flavors of hash node.  */
 enum node_type
@@ -630,7 +635,7 @@ struct cpp_hashnode GTY(())
 					   Otherwise, a NODE_OPERATOR.  */
   unsigned char rid_code;		/* Rid code - for front ends.  */
   ENUM_BITFIELD(node_type) type : 8;	/* CPP node type.  */
-  unsigned char flags;			/* CPP flags.  */
+  unsigned int flags;			/* CPP flags.  */
 
   union _cpp_hashnode_value GTY ((desc ("CPP_HASHNODE_VALUE_IDX (%1)"))) value;
 };
@@ -717,6 +722,8 @@ extern const cpp_token *cpp_get_token_wi
 extern const unsigned char *cpp_macro_definition (cpp_reader *,
 						  const cpp_hashnode *);
 extern void _cpp_backup_tokens (cpp_reader *, unsigned int);
+extern const cpp_token *_cpp_peek_token (cpp_reader *, int);
+extern void _cpp_backup_tokens_direct (cpp_reader *, unsigned int);
 
 /* Evaluate a CPP_CHAR or CPP_WCHAR token.  */
 extern cppchar_t cpp_interpret_charconst (cpp_reader *, const cpp_token *,
Index: libcpp/lex.c
===================================================================
--- libcpp/lex.c	(revision 137138)
+++ libcpp/lex.c	(working copy)
@@ -734,6 +734,45 @@ next_tokenrun (tokenrun *run)
   return run->next;
 }
 
+/* Look ahead in the input stream.  */
+const cpp_token *
+_cpp_peek_token (cpp_reader *pfile, int index)
+{
+  cpp_context *context = pfile->context;
+  const cpp_token *peektok;
+  int count;
+
+  /* First, scan through any pending cpp_context objects.  */
+  while (context->prev)
+    {
+      ptrdiff_t sz = (context->direct_p
+                      ? LAST (context).token - FIRST (context).token
+                      : LAST (context).ptoken - FIRST (context).ptoken);
+
+      if (index < (int) sz)
+        return (context->direct_p
+                ? FIRST (context).token + index
+                : *(FIRST (context).ptoken + index));
+
+      index -= (int) sz;
+      context = context->prev;
+    }
+
+  /* We will have to read some new tokens after all (and do so
+     without invalidating preceding tokens).  */
+  count = index;
+  pfile->keep_tokens++;
+
+  do
+    peektok = _cpp_lex_token (pfile);
+  while (index--);
+
+  _cpp_backup_tokens_direct (pfile, count + 1);
+  pfile->keep_tokens--;
+
+  return peektok;
+}
+
 /* Allocate a single token that is invalidated at the same time as the
    rest of the tokens on the line.  Has its line and col set to the
    same as the last lexed token, so that diagnostics appear in the
@@ -742,9 +781,30 @@ cpp_token *
 _cpp_temp_token (cpp_reader *pfile)
 {
   cpp_token *old, *result;
+  ptrdiff_t sz = pfile->cur_run->limit - pfile->cur_token;
+  ptrdiff_t la = (ptrdiff_t) pfile->lookaheads;
 
   old = pfile->cur_token - 1;
-  if (pfile->cur_token == pfile->cur_run->limit)
+  /* Any pre-existing lookaheads must not be clobbered.  */
+  if (la)
+    {
+      if (sz <= la)
+        {
+          tokenrun *next = next_tokenrun (pfile->cur_run);
+
+          if (sz < la)
+            memmove (next->base + 1, next->base,
+                     (la - sz) * sizeof (cpp_token));
+
+          next->base[0] = pfile->cur_run->limit[-1];
+        }
+
+      if (sz > 1)
+        memmove (pfile->cur_token + 1, pfile->cur_token,
+                 MIN (la, sz - 1) * sizeof (cpp_token));
+    }
+
+  if (!sz && pfile->cur_token == pfile->cur_run->limit)
     {
       pfile->cur_run = next_tokenrun (pfile->cur_run);
       pfile->cur_token = pfile->cur_run->base;

Re: PATCH: Altivec context-sensitive keywords

[Tom Tromey]

>>>>> "Ben" == Ben Elliston <bje@au1.ibm.com> writes:

Ben> This patch is a revised attempt to implement context-sensitive keywords
Ben> for the PowerPC port.  There are some libcpp changes, some rs6000
Ben> backend changes, and some minor testsuite changes.

I looked at this.  BTW for those following along, the previous patch
is here:

    http://gcc.gnu.org/ml/gcc-patches/2007-12/msg00306.html

Ben> To overcome many of the concerns about the interaction of
Ben> context-sensitive keywords with ISO C compliance, the feature is now
Ben> controlled by !flag_iso.

I read through this a bit but I don't see how the new patch addresses
test #3 from Joseph:

    http://gcc.gnu.org/ml/gcc-patches/2007-12/msg00339.html

Would you mind explaining that?  I feel like I must be missing
something.

Ben> +static GTY(()) cpp_hashnode *__vector_keyword;

Does this work?  I thought that since cpp_hashnode was embedded in
another structure, you need special marking magic to make the GC work
here.

Maybe it does not matter since we only turn on identifier GC after
parsing is complete.  If so, that seems a bit fragile to me.

Ben> +  /* Keywords without two leading underscores are context-sensitive,
Ben> +     and hence implemented as conditional macros, controlled by the
Ben> +     rs6000_macro_to_expand() function above.  */

s/above/below/

Ben> +  static bool vector_keywords_init = false;
Ben> +  if (!vector_keywords_init)
Ben> +    {
Ben> +      init_vector_keywords (pfile);
Ben> +      vector_keywords_init = true;
Ben> +    }

I think this must be unnecessary... init_vector_keywords is called
elsewhere, and IIUC, rs6000_macro_to_expand can only be called for
conditional macros -- and if we have not called init_vector_keywords
already, then there won't be any of these.

Ben> +void
Ben> +_cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)

I would prefer, if possible, not to have new public cpp functions that
frob cpp internal state like this.  The cpp code is pretty obscure
already; code paths like this make it even harder to reason about.

At the very least I think public methods should not have a leading "_".

Ben> @@ -630,7 +635,7 @@ struct cpp_hashnode GTY(())
Ben>  					   Otherwise, a NODE_OPERATOR.  */
Ben>    unsigned char rid_code;		/* Rid code - for front ends.  */
Ben>    ENUM_BITFIELD(node_type) type : 8;	/* CPP node type.  */
Ben> -  unsigned char flags;			/* CPP flags.  */
Ben> +  unsigned int flags;			/* CPP flags.  */

Do you know how this affects memory use?  I don't know how many
cpp_hashnodes are created ... this looks like a tightly-packed
structure, and I am wary about expanding it.

We could make a new NT_CONDITIONAL_MACRO type (maybe hard).  Or, I
think we can steal a bit from the type field -- if this does not make
the code big and/or slow.

Ben> +/* Look ahead in the input stream.  */
Ben> +const cpp_token *
Ben> +_cpp_peek_token (cpp_reader *pfile, int index)

Remove the leading "_".

It would be good to have test cases for combinations of vector and the
other conditional macros at EOF and in other weird places (#pragma or
_Pragma comes to mind).  I've fixed a number of crashes and whatnot
related to buffer manipulation at these boundaries... 


As long as the context-sensitivity remains lexical, as in this patch,
I'm ok with it.  But note that, on the incremental compiler branch,
the C compiler lexes the entire CU at once (as the C++ FE does on
trunk).  So, if hooks into the C parser are needed, this approach will
break the incremental compiler.

Since I'd prefer to avoid this, if parser hooks are needed, I would
prefer to see context-sensitive keywords.  I realize they are
unpopular, but at least they would support lexing AOT.

FWIW I think context-sensitive keywords would be easier to understand
even than the current patch, simply because they would be explicit
code in the parser rather than a callback buried in cpp.  Also, this
approach would not require new state-manipulating public cpp
functions.

Tom

Re: PATCH: Altivec context-sensitive keywords

[Andrew Pinski]

On Thu, Jun 26, 2008 at 4:04 AM, Ben Elliston <bje@au1.ibm.com> wrote:
> To overcome many of the concerns about the interaction of
> context-sensitive keywords with ISO C compliance, the feature is now
> controlled by !flag_iso.  When compiling with -maltivec, the
> context-sensitive keyword support is enabled (and the compiler defines
> __APPLE_ALTIVEC__, as required by the PIM).  When _APPLE_ALTIVEC__ is
> defined, <altivec.h> will suppress the macros that "implement" these
> keywords.

I would not call the macro __APPLE_ALTIVEC__, but instead
__CONTEX_SENSITIVE_VMX__ (or ALTIVEC for that matter).

> The two testsuite changes in gcc.dg/vmx are made due to the reasoning
> stated here:
>  http://gcc.gnu.org/ml/gcc-patches/2008-04/msg01616.html
>
> Rather than remove the use of bool variables from this test, the tests
> are now compiled with -ansi to ensure that the predefine method is used.

I think it is best if you run those testcase for both cases (the
context sensitive case deleting the bool variants) so there is at
least better testing of both cases.

Thanks,
Andrew Pinski

Re: PATCH: Altivec context-sensitive keywords

[Ben Elliston]

On Thu, 2008-06-26 at 16:13 -0700, Andrew Pinski wrote:

> I would not call the macro __APPLE_ALTIVEC__, but instead
> __CONTEX_SENSITIVE_VMX__ (or ALTIVEC for that matter).

I didn't choose the name of that macro; it has been mentioned in
<altivec.h> since 2004 (and should be considered a programmer-visible
interface that we should be reluctant to change).  A Google search
suggests that at least some people have used it in existing source code.

Ben

Re: PATCH: Altivec context-sensitive keywords

[Ben Elliston]

Hi Tom

Thanks for the review.  I've addressed the easy parts ;-)

> Ben> +  /* Keywords without two leading underscores are context-sensitive,
> Ben> +     and hence implemented as conditional macros, controlled by the
> Ben> +     rs6000_macro_to_expand() function above.  */
> 
> s/above/below/

Fixed.

> Ben> +  static bool vector_keywords_init = false;
> Ben> +  if (!vector_keywords_init)
> Ben> +    {
> Ben> +      init_vector_keywords (pfile);
> Ben> +      vector_keywords_init = true;
> Ben> +    }
> 
> I think this must be unnecessary... init_vector_keywords is called
> elsewhere, and IIUC, rs6000_macro_to_expand can only be called for
> conditional macros -- and if we have not called init_vector_keywords
> already, then there won't be any of these.

You're right; fixed.

> Ben> +/* Look ahead in the input stream.  */
> Ben> +const cpp_token *
> Ben> +_cpp_peek_token (cpp_reader *pfile, int index)
> 
> Remove the leading "_".

Fixed.

> It would be good to have test cases for combinations of vector and the
> other conditional macros at EOF and in other weird places (#pragma or
> _Pragma comes to mind).  I've fixed a number of crashes and whatnot
> related to buffer manipulation at these boundaries... 

I've added tests for #pragma and _Pragma.  Sure enough, there are
problems around EOF -- I will fix those.

Cheers, Ben

Re: PATCH: Altivec context-sensitive keywords

[Ben Elliston]

On Thu, 2008-06-26 at 13:22 -0600, Tom Tromey wrote:

> I read through this a bit but I don't see how the new patch addresses
> test #3 from Joseph:
> 
>     http://gcc.gnu.org/ml/gcc-patches/2007-12/msg00339.html
> 
> Would you mind explaining that?  I feel like I must be missing
> something.

When compiling with -std=c99, the context sensitive keyword support is
disabled, so a diagnostic is issued.  As far as I can tell, this meets
Joseph's requirements.

> Ben> +static GTY(()) cpp_hashnode *__vector_keyword;
> 
> Does this work?  I thought that since cpp_hashnode was embedded in
> another structure, you need special marking magic to make the GC work
> here.

I'm really unfamiliar with the GTY machinery -- can you elaborate?

> Ben> +void
> Ben> +_cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)
> 
> I would prefer, if possible, not to have new public cpp functions that
> frob cpp internal state like this.  The cpp code is pretty obscure
> already; code paths like this make it even harder to reason about.

I've moved the prototype for this function into internal.h and removed
it from include/cpplib.h (leaving the leading "_").  Is that OK?

> It would be good to have test cases for combinations of vector and the
> other conditional macros at EOF and in other weird places (#pragma or
> _Pragma comes to mind).  I've fixed a number of crashes and whatnot
> related to buffer manipulation at these boundaries... 

Done.

Cheers, Ben

Re: PATCH: Altivec context-sensitive keywords

[Tom Tromey]

>>>>> "Ben" == Ben Elliston <bje@au1.ibm.com> writes:

Ben> +static GTY(()) cpp_hashnode *__vector_keyword;

Tom> Does this work?  I thought that since cpp_hashnode was embedded in
Tom> another structure, you need special marking magic to make the GC work
Tom> here.

Ben> I'm really unfamiliar with the GTY machinery -- can you elaborate?

Sure.  Some background: the GC relies on the GTY markers to know how
to mark things.  And, in general, it expects pointers to point to the
start of objects.  If you have an interior pointer you have to use the
nested_ptr GTY keyword to tell the GC how to mark the containing
object.  See gcc/doc/gty.texi for info on this.

I think a cpp_hashnode only appears as an element of an identifier
tree -- see c-common.h:c_common_identifier.  There is some weird
machinery to support this, both in libcpp and in gcc via the
HT_IDENT_TO_GCC_IDENT and GCC_IDENT_TO_HT_IDENT macros.

Also see the GTY marker for the 'node' field in cpplib.h:cpp_token.

Offhand I would guess the simplest way to get the effect you want is
to change these variables to be of type 'tree', look them up using
get_identifier(), and then cast them to 'struct c_common_identifier *'.
This is probably technically wrong somehow but the C FE must already
be doing this.

There may be some other workable approach, e.g. just using the proper
GTY marker.

Anything you do here is likely to end up kind of ugly.  I'm not sure
one is obviously nicer.

Ben> +void
Ben> +_cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)

Ben> I've moved the prototype for this function into internal.h and removed
Ben> it from include/cpplib.h (leaving the leading "_").  Is that OK?

Yeah, thanks.

Tom

Re: PATCH: Altivec context-sensitive keywords

[Ben Elliston]

Here is a revised patch.  I think it takes care of everything except for
the GTY(()) parts.

How does this look otherwise, Tom?

Ben

gcc/
        * coretypes.h (struct cpp_token): Forward declare.
        * doc/extend.texi (PowerPC AltiVec Built-in Functions): Document
        the context-sensitive keyword method.
        * config/rs6000/rs6000-c.c (__vector_keyword, vector_keyword,
        __pixel_keyword, pixel_keyword, __bool_keyword, bool_keyword,
        expand_bool_pixel): New cpp_hashnodes.
        (altivec_categorize_keyword): New function.
        (DSC): Convenience macro, borrowed from libcpp.
        (init_vector_keywords): New function.
        (rs6000_macro_to_expand): Likewise.
        (rs6000_cpu_cpp_builtins): Enable context-sensitive macros if not
        compiling an ISO C dialect.

gcc/testsuite/
        * gcc.target/powerpc/altivec-macros.c: New test.
        * gcc.target/powerpc/altviec-26.c: Likewise.
        * gcc.dg/vmx/1b-06.c: Remove bool variable.
        * gcc.dg/vmx/1b-07.c: Likewise.
        * gcc.dg/vmx/1b-06-ansi.c: New test for the pre-define method.
        * gcc.dg/vmx/1b-07-ansi.c: Likewise.

libcpp/
        * include/cpplib.h (NODE_CONDITIONAL): New.
        (struct cpp_callbacks): New macro_to_expand field.
        (struct cpp_hashnode): Adjust size of flags and type fields.
        (cpp_peek_token): Prototype.
        * lex.c (cpp_peek_token): New function.
        (_cpp_temp_token): Protect pre-existing lookaheads.
        * macro.c (cpp_get_token): Expand any conditional macros.
        (_cpp_backup_tokens_direct): New.
        (_cpp_backup_tokens): Call _cpp_backup_tokens_direct.
        (warn_of_redefinition): Silently allow redefined conditional
        macros.
        (_cpp_create_definition): Remove the conditional flag when a user
        defines one of the conditional macros.
        * internal.h (_cpp_backup_tokens_direct): New prototype.

Index: gcc/doc/extend.texi
===================================================================
--- gcc/doc/extend.texi	(revision 137307)
+++ gcc/doc/extend.texi	(working copy)
@@ -9120,9 +9120,10 @@ always specify the signedness.
 
 @item
 Compiling with @option{-maltivec} adds keywords @code{__vector},
-@code{__pixel}, and @code{__bool}.  Macros @option{vector},
-@code{pixel}, and @code{bool} are defined in @code{<altivec.h>} and can
-be undefined.
+@code{vector}, @code{__pixel}, @code{pixel}, @code{__bool} and
+@code{bool}.  When compiling ISO C, the context-sensitive substitution
+of the keywords @code{vector}, @code{pixel} and @code{bool} is
+disabled.  To use them, you must include @code{<altivec.h>} instead.
 
 @item
 GCC allows using a @code{typedef} name as the type specifier for a
Index: gcc/testsuite/gcc.target/powerpc/altivec-macros.c
===================================================================
--- gcc/testsuite/gcc.target/powerpc/altivec-macros.c	(revision 0)
+++ gcc/testsuite/gcc.target/powerpc/altivec-macros.c	(revision 0)
@@ -0,0 +1,63 @@
+/* Copyright (C) 2007 Free Software Foundation, Inc.  */
+
+/* { dg-do preprocess } */
+/* { dg-options "-maltivec" } */
+
+/* Conditional macros should not be expanded by pragmas.  */
+#pragma __vector
+_Pragma ("__vector")
+
+/* Redefinition of conditional macros.  */
+/* No warning should be generated.  */
+
+#define __vector __new_vector
+#define __pixel __new_pixel
+#define __bool __new_bool
+#define vector new_vector
+#define pixel new_pixel
+#define bool new_bool
+
+/* Definition of conditional macros.  */
+/* No warning should be generated.  */
+
+#undef __vector
+#define __vector __new_vector
+
+#undef __pixel
+#define __pixel __new_pixel
+
+#undef __bool
+#define __bool __new_bool
+
+#undef vector
+#define vector new_vector
+
+#undef pixel
+#define pixel new_pixel
+
+#undef bool
+#define bool new_bool
+
+/* Re-definition of "unconditional" macros.  */
+/* Warnings should be generated as usual.  */
+
+#define __vector	__newer_vector
+#define __pixel		__newer_pixel
+#define __bool		__newer_bool
+#define vector		newer_vector
+#define pixel		newer_pixel
+#define bool		newer_bool
+
+/* { dg-warning "redefined" "__vector redefined"  { target *-*-* } 44 } */
+/* { dg-warning "redefined" "__pixel redefined"   { target *-*-* } 45 } */
+/* { dg-warning "redefined" "__bool redefined"    { target *-*-* } 46 } */
+/* { dg-warning "redefined" "vector redefined"    { target *-*-* } 47 } */
+/* { dg-warning "redefined" "pixel redefined"     { target *-*-* } 48 } */
+/* { dg-warning "redefined" "bool redefined"      { target *-*-* } 49 } */
+
+/* { dg-warning "previous"  "prev __vector defn"  { target *-*-* } 24 } */
+/* { dg-warning "previous"  "prev __pixel defn"   { target *-*-* } 27 } */
+/* { dg-warning "previous"  "prev __bool defn"    { target *-*-* } 30 } */
+/* { dg-warning "previous"  "prev vector defn"    { target *-*-* } 33 } */
+/* { dg-warning "previous"  "prev pixel defn"     { target *-*-* } 36 } */
+/* { dg-warning "previous"  "prev bool defn"      { target *-*-* } 39 } */
Index: gcc/testsuite/gcc.target/powerpc/altivec-26.c
===================================================================
--- gcc/testsuite/gcc.target/powerpc/altivec-26.c	(revision 0)
+++ gcc/testsuite/gcc.target/powerpc/altivec-26.c	(revision 0)
@@ -0,0 +1,11 @@
+/* { dg-do compile { target powerpc*-*-* } } */
+/* { dg-require-effective-target powerpc_altivec_ok } */
+/* { dg-options "-maltivec" } */
+
+/* A compiler implementing context-sensitive keywords must define this
+   preprocessor macro so that altivec.h does not provide the vector,
+   pixel, etc. macros.  */
+
+#ifndef __APPLE_ALTIVEC__
+#error __APPLE_ALTIVEC__ not pre-defined
+#endif
Index: gcc/testsuite/gcc.dg/vmx/1b-07-ansi.c
===================================================================
--- gcc/testsuite/gcc.dg/vmx/1b-07-ansi.c	(revision 0)
+++ gcc/testsuite/gcc.dg/vmx/1b-07-ansi.c	(revision 0)
@@ -0,0 +1,59 @@
+/* { dg-do compile } */
+/* { dg-options "-ansi -maltivec" } */
+
+#include <altivec.h>
+vector char bool _0 ;
+vector bool char _8 ;
+vector char unsigned _56 ;
+vector unsigned char _64 ;
+vector char signed _112 ;
+vector signed char _120 ;
+/* bool is permitted in the predefine method, as it is expanded
+   unconditionally to int.  */
+bool _168 ;
+vector pixel _170 ;
+vector int bool _178 ;
+vector bool int _186 ;
+vector short bool _234 ;
+vector bool short _242 ;
+vector unsigned int _290 ;
+vector int unsigned _298 ;
+vector unsigned short _346 ;
+vector short unsigned _354 ;
+vector signed int _402 ;
+vector int signed _410 ;
+vector signed short _458 ;
+vector short signed _466 ;
+vector int bool _514 ;
+vector int bool _544 ;
+vector int bool _559 ;
+vector bool int _589 ;
+vector int short bool _874 ;
+vector int bool short _889 ;
+vector short int bool _904 ;
+vector short bool int _919 ;
+vector bool int short _934 ;
+vector bool short int _949 ;
+vector unsigned int _1234 ;
+vector int unsigned _1249 ;
+vector unsigned int _1279 ;
+vector int unsigned _1294 ;
+vector unsigned int _1309 ;
+vector int unsigned short _1594 ;
+vector int short unsigned _1609 ;
+vector unsigned int short _1624 ;
+vector unsigned short int _1639 ;
+vector short int unsigned _1654 ;
+vector short unsigned int _1669 ;
+vector signed int _1954 ;
+vector int signed _1969 ;
+vector signed int _1999 ;
+vector int signed _2014 ;
+vector signed int _2029 ;
+vector int signed short _2314 ;
+vector int short signed _2329 ;
+vector signed int short _2344 ;
+vector signed short int _2359 ;
+vector short int signed _2374 ;
+vector short signed int _2389 ;
+vector float _2674 ;
Index: gcc/testsuite/gcc.dg/vmx/1b-07.c
===================================================================
--- gcc/testsuite/gcc.dg/vmx/1b-07.c	(revision 137307)
+++ gcc/testsuite/gcc.dg/vmx/1b-07.c	(working copy)
@@ -6,7 +6,6 @@ vector char unsigned _56 ;
 vector unsigned char _64 ;
 vector char signed _112 ;
 vector signed char _120 ;
-bool _168 ;
 vector pixel _170 ;
 vector int bool _178 ;
 vector bool int _186 ;
Index: gcc/testsuite/gcc.dg/vmx/1b-06-ansi.c
===================================================================
--- gcc/testsuite/gcc.dg/vmx/1b-06-ansi.c	(revision 0)
+++ gcc/testsuite/gcc.dg/vmx/1b-06-ansi.c	(revision 0)
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-ansi -maltivec" } */
+
+#include <altivec.h>
+vector char bool _4 ;
+vector char unsigned _31 ;
+vector char signed _59 ;
+/* bool is permitted in the predefine method, as it is expanded
+   unconditionally to int.  */
+bool _84 ;
+vector pixel _89 ;
+vector int bool _95 ;
+vector short bool _102 ;
+vector unsigned int _122 ;
+vector unsigned short _129 ;
+vector signed int _150 ;
+vector signed short _157 ;
+vector int bool _179 ;
+vector int short bool _186 ;
+vector unsigned int _206 ;
+vector int unsigned short _213 ;
+vector signed int _234 ;
+vector int signed short _241 ;
+vector float _339 ;
Index: gcc/testsuite/gcc.dg/vmx/1b-06.c
===================================================================
--- gcc/testsuite/gcc.dg/vmx/1b-06.c	(revision 137307)
+++ gcc/testsuite/gcc.dg/vmx/1b-06.c	(working copy)
@@ -3,7 +3,6 @@
 vector char bool _4 ;
 vector char unsigned _31 ;
 vector char signed _59 ;
-bool _84 ;
 vector pixel _89 ;
 vector int bool _95 ;
 vector short bool _102 ;
Index: gcc/testsuite/ChangeLog
===================================================================
--- gcc/testsuite/ChangeLog	(revision 137307)
+++ gcc/testsuite/ChangeLog	(working copy)
@@ -1,3 +1,12 @@
+2008-06-30  Ben Elliston  <bje@au.ibm.com>
+
+	* gcc.target/powerpc/altivec-macros.c: New test.
+	* gcc.target/powerpc/altviec-26.c: Likewise.
+	* gcc.dg/vmx/1b-06.c: Remove bool variable.
+	* gcc.dg/vmx/1b-07.c: Likewise.
+	* gcc.dg/vmx/1b-06-ansi.c: New test for the pre-define method.
+	* gcc.dg/vmx/1b-07-ansi.c: Likewise.
+
 2008-06-30  Andy Hutchinson  <hutchinsonandy@aim.com>
 
 	PR target/36598
Index: gcc/coretypes.h
===================================================================
--- gcc/coretypes.h	(revision 137307)
+++ gcc/coretypes.h	(working copy)
@@ -60,9 +60,11 @@ enum ir_type {
 
 /* Provide forward struct declaration so that we don't have to include
    all of cpplib.h whenever a random prototype includes a pointer.
-   Note that the cpp_reader typedef remains part of cpplib.h.  */
+   Note that the cpp_reader and cpp_token typedefs remain part of
+   cpplib.h.  */
 
 struct cpp_reader;
+struct cpp_token;
 
 /* The thread-local storage model associated with a given VAR_DECL
    or SYMBOL_REF.  This isn't used much, but both trees and RTL refer
Index: gcc/config/rs6000/rs6000-c.c
===================================================================
--- gcc/config/rs6000/rs6000-c.c	(revision 137307)
+++ gcc/config/rs6000/rs6000-c.c	(working copy)
@@ -84,6 +84,135 @@ rs6000_pragma_longcall (cpp_reader *pfil
 #define builtin_define(TXT) cpp_define (pfile, TXT)
 #define builtin_assert(TXT) cpp_assert (pfile, TXT)
 
+/* Keep the AltiVec keywords handy for fast comparisons.  */
+static GTY(()) cpp_hashnode *__vector_keyword;
+static GTY(()) cpp_hashnode *vector_keyword;
+static GTY(()) cpp_hashnode *__pixel_keyword;
+static GTY(()) cpp_hashnode *pixel_keyword;
+static GTY(()) cpp_hashnode *__bool_keyword;
+static GTY(()) cpp_hashnode *bool_keyword;
+
+static GTY(()) cpp_hashnode *expand_bool_pixel;  /* Preserved across calls.  */
+
+static cpp_hashnode *
+altivec_categorize_keyword (const cpp_token *tok)
+{
+  if (tok->type == CPP_NAME)
+    {
+      cpp_hashnode *ident = tok->val.node;
+
+      if (ident == vector_keyword || ident == __vector_keyword)
+	return __vector_keyword;
+
+      if (ident == pixel_keyword || ident ==  __pixel_keyword)
+	return __pixel_keyword;
+
+      if (ident == bool_keyword || ident == __bool_keyword)
+	return __bool_keyword;
+
+      return ident;
+    }
+
+  return 0;
+}
+
+/*  From libcpp.  */
+#define DSC(str) (const unsigned char *)str, sizeof str - 1
+
+static void
+init_vector_keywords (cpp_reader *pfile)
+{
+  /* Keywords without two leading underscores are context-sensitive,
+     and hence implemented as conditional macros, controlled by the
+     rs6000_macro_to_expand() function below.  */
+
+  __vector_keyword = cpp_lookup (pfile, DSC ("__vector"));
+  __vector_keyword->flags |= NODE_CONDITIONAL;
+
+  __pixel_keyword = cpp_lookup (pfile, DSC ("__pixel"));
+  __pixel_keyword->flags |= NODE_CONDITIONAL;
+
+  __bool_keyword = cpp_lookup (pfile, DSC ("__bool"));
+  __bool_keyword->flags |= NODE_CONDITIONAL;
+
+  vector_keyword = cpp_lookup (pfile, DSC ("vector"));
+  vector_keyword->flags |= NODE_CONDITIONAL;
+
+  pixel_keyword = cpp_lookup (pfile, DSC ("pixel"));
+  pixel_keyword->flags |= NODE_CONDITIONAL;
+
+  bool_keyword = cpp_lookup (pfile, DSC ("bool"));
+  bool_keyword->flags |= NODE_CONDITIONAL;
+}
+
+/* Called to decide whether a conditional macro should be expanded.
+   Since we have exactly one such macro (i.e, 'vector'), we do not
+   need to examine the 'tok' parameter.  */
+
+static cpp_hashnode *
+rs6000_macro_to_expand (cpp_reader *pfile, const cpp_token *tok)
+{
+  cpp_hashnode *expand_this = tok->val.node;
+  cpp_hashnode *ident;
+
+  ident = altivec_categorize_keyword (tok);
+
+  if (ident == __vector_keyword)
+    {
+      tok = cpp_peek_token (pfile, 0);
+      ident = altivec_categorize_keyword (tok);
+
+      if (ident ==  __pixel_keyword || ident == __bool_keyword)
+	{
+	  expand_this = __vector_keyword;
+	  expand_bool_pixel = ident;
+	}
+      else if (ident)
+	{
+	  enum rid rid_code = (enum rid)(ident->rid_code);
+	  if (ident->type == NT_MACRO)
+	    {
+	      (void)cpp_get_token (pfile);
+	      tok = cpp_peek_token (pfile, 0);
+	      ident = altivec_categorize_keyword (tok);
+	      if (ident)
+		rid_code = (enum rid)(ident->rid_code);
+	    }
+
+	  if (rid_code == RID_UNSIGNED || rid_code == RID_LONG
+	      || rid_code == RID_SHORT || rid_code == RID_SIGNED
+	      || rid_code == RID_INT || rid_code == RID_CHAR
+	      || rid_code == RID_FLOAT)
+	    {
+	      expand_this = __vector_keyword;
+	      /* If the next keyword is bool or pixel, it
+		 will need to be expanded as well.  */
+	      tok = cpp_peek_token (pfile, 1);
+	      ident = altivec_categorize_keyword (tok);
+
+	      if (ident ==  __pixel_keyword || ident == __bool_keyword)
+		expand_bool_pixel = ident;
+	      else
+		{
+		  /* Try two tokens down, too.  */
+		  tok = cpp_peek_token (pfile, 2);
+		  ident = altivec_categorize_keyword (tok);
+		  if (ident ==  __pixel_keyword || ident == __bool_keyword)
+		    expand_bool_pixel = ident;
+		}
+	    }
+	}
+    }
+  else if (expand_bool_pixel
+	   && (ident ==  __pixel_keyword || ident == __bool_keyword))
+    {
+      expand_this = expand_bool_pixel;
+      expand_bool_pixel = 0;
+    }
+
+  return expand_this;
+}
+
 void
 rs6000_cpu_cpp_builtins (cpp_reader *pfile)
 {
@@ -120,6 +249,20 @@ rs6000_cpu_cpp_builtins (cpp_reader *pfi
       builtin_define ("__vector=__attribute__((altivec(vector__)))");
       builtin_define ("__pixel=__attribute__((altivec(pixel__))) unsigned short");
       builtin_define ("__bool=__attribute__((altivec(bool__))) unsigned");
+
+      if (!flag_iso)
+	{
+	  /* Define this when supporting context-sensitive keywords.  */
+	  builtin_define ("__APPLE_ALTIVEC__");
+	  
+	  builtin_define ("vector=vector");
+	  builtin_define ("pixel=pixel");
+	  builtin_define ("bool=bool");
+	  init_vector_keywords (pfile);
+
+	  /* Enable context-sensitive macros.  */
+	  cpp_get_callbacks (pfile)->macro_to_expand = rs6000_macro_to_expand;
+	}
     }
   if (TARGET_SPE)
     builtin_define ("__SPE__");
Index: libcpp/macro.c
===================================================================
--- libcpp/macro.c	(revision 137307)
+++ libcpp/macro.c	(working copy)
@@ -1240,16 +1240,21 @@ cpp_get_token (cpp_reader *pfile)
 
       if (!(node->flags & NODE_DISABLED))
 	{
-	  int ret;
+	  int ret = 0;
 	  /* If not in a macro context, and we're going to start an
 	     expansion, record the location.  */
 	  if (can_set && !context->macro)
 	    pfile->invocation_location = result->src_loc;
 	  if (pfile->state.prevent_expansion)
 	    break;
-	  ret = enter_macro_context (pfile, node, result);
-	  if (ret)
-	    {
+
+	  /* Conditional macros require that a predicate be evaluated
+	     first.  */
+	  if (((!(node->flags & NODE_CONDITIONAL))
+	       || (pfile->cb.macro_to_expand
+		   && (node = pfile->cb.macro_to_expand (pfile, result))))
+	      && (ret = enter_macro_context (pfile, node, result)))
+ 	    {
 	      if (pfile->state.in_directive || ret == 2)
 		continue;
 	      return padding_token (pfile, result);
@@ -1327,26 +1332,31 @@ cpp_scan_nooutput (cpp_reader *pfile)
   pfile->state.prevent_expansion--;
 }
 
+/* Step back one or more tokens obtained from the lexer.  */
+void
+_cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)
+{
+  pfile->lookaheads += count;
+  while (count--)
+    {
+      pfile->cur_token--;
+      if (pfile->cur_token == pfile->cur_run->base
+          /* Possible with -fpreprocessed and no leading #line.  */
+          && pfile->cur_run->prev != NULL)
+        {
+          pfile->cur_run = pfile->cur_run->prev;
+          pfile->cur_token = pfile->cur_run->limit;
+        }
+    }
+}
+
 /* Step back one (or more) tokens.  Can only step back more than 1 if
    they are from the lexer, and not from macro expansion.  */
 void
 _cpp_backup_tokens (cpp_reader *pfile, unsigned int count)
 {
   if (pfile->context->prev == NULL)
-    {
-      pfile->lookaheads += count;
-      while (count--)
-	{
-	  pfile->cur_token--;
-	  if (pfile->cur_token == pfile->cur_run->base
-	      /* Possible with -fpreprocessed and no leading #line.  */
-	      && pfile->cur_run->prev != NULL)
-	    {
-	      pfile->cur_run = pfile->cur_run->prev;
-	      pfile->cur_token = pfile->cur_run->limit;
-	    }
-	}
-    }
+    _cpp_backup_tokens_direct (pfile, count);
   else
     {
       if (count != 1)
@@ -1372,6 +1382,11 @@ warn_of_redefinition (cpp_reader *pfile,
   if (node->flags & NODE_WARN)
     return true;
 
+  /* Redefinitions of conditional (context-sensitive) macros, on
+     the other hand, must be allowed silently.  */
+  if (node->flags & NODE_CONDITIONAL)
+    return false;
+
   /* Redefinition of a macro is allowed if and only if the old and new
      definitions are the same.  (6.10.3 paragraph 2).  */
   macro1 = node->value.macro;
@@ -1804,6 +1819,10 @@ _cpp_create_definition (cpp_reader *pfil
       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_CONSTANT_MACROS"))
     node->flags |= NODE_WARN;
 
+  /* If user defines one of the conditional macros, remove the
+     conditional flag */
+  node->flags &= ~NODE_CONDITIONAL;
+
   return ok;
 }
 
Index: libcpp/include/cpplib.h
===================================================================
--- libcpp/include/cpplib.h	(revision 137307)
+++ libcpp/include/cpplib.h	(working copy)
@@ -484,6 +484,10 @@ struct cpp_callbacks
   void (*read_pch) (cpp_reader *, const char *, int, const char *);
   missing_header_cb missing_header;
 
+  /* Context-sensitive macro support.  Returns macro (if any) that should
+     be expanded.  */
+  cpp_hashnode * (*macro_to_expand) (cpp_reader *, const cpp_token *);
+
   /* Called to emit a diagnostic if client_diagnostic option is true.
      This callback receives the translated message.  */
   void (*error) (cpp_reader *, int, const char *, va_list *)
@@ -558,6 +562,7 @@ extern const char *progname;
 #define NODE_DISABLED	(1 << 5)	/* A disabled macro.  */
 #define NODE_MACRO_ARG	(1 << 6)	/* Used during #define processing.  */
 #define NODE_USED	(1 << 7)	/* Dumped with -dU.  */
+#define NODE_CONDITIONAL (1 << 8)	/* Conditional macro */
 
 /* Different flavors of hash node.  */
 enum node_type
@@ -629,8 +634,8 @@ struct cpp_hashnode GTY(())
 					   then index into directive table.
 					   Otherwise, a NODE_OPERATOR.  */
   unsigned char rid_code;		/* Rid code - for front ends.  */
-  ENUM_BITFIELD(node_type) type : 8;	/* CPP node type.  */
-  unsigned char flags;			/* CPP flags.  */
+  ENUM_BITFIELD(node_type) type : 7;	/* CPP node type.  */
+  unsigned int flags : 9;		/* CPP flags.  */
 
   union _cpp_hashnode_value GTY ((desc ("CPP_HASHNODE_VALUE_IDX (%1)"))) value;
 };
@@ -717,6 +722,7 @@ extern const cpp_token *cpp_get_token_wi
 extern const unsigned char *cpp_macro_definition (cpp_reader *,
 						  const cpp_hashnode *);
 extern void _cpp_backup_tokens (cpp_reader *, unsigned int);
+extern const cpp_token *cpp_peek_token (cpp_reader *, int);
 
 /* Evaluate a CPP_CHAR or CPP_WCHAR token.  */
 extern cppchar_t cpp_interpret_charconst (cpp_reader *, const cpp_token *,
Index: libcpp/internal.h
===================================================================
--- libcpp/internal.h	(revision 137307)
+++ libcpp/internal.h	(working copy)
@@ -532,6 +532,7 @@ extern const unsigned char *_cpp_builtin
 extern int _cpp_warn_if_unused_macro (cpp_reader *, cpp_hashnode *, void *);
 extern void _cpp_push_token_context (cpp_reader *, cpp_hashnode *,
 				     const cpp_token *, unsigned int);
+extern void _cpp_backup_tokens_direct (cpp_reader *, unsigned int);
 
 /* In identifiers.c */
 extern void _cpp_init_hashtable (cpp_reader *, hash_table *);
Index: libcpp/lex.c
===================================================================
--- libcpp/lex.c	(revision 137307)
+++ libcpp/lex.c	(working copy)
@@ -734,6 +734,49 @@ next_tokenrun (tokenrun *run)
   return run->next;
 }
 
+/* Look ahead in the input stream.  */
+const cpp_token *
+cpp_peek_token (cpp_reader *pfile, int index)
+{
+  cpp_context *context = pfile->context;
+  const cpp_token *peektok;
+  int kount;
+
+  /* First, scan through any pending cpp_context objects.  */
+  while (context->prev)
+    {
+      ptrdiff_t sz = (context->direct_p
+                      ? LAST (context).token - FIRST (context).token
+                      : LAST (context).ptoken - FIRST (context).ptoken);
+
+      if (index < (int) sz)
+        return (context->direct_p
+                ? FIRST (context).token + index
+                : *(FIRST (context).ptoken + index));
+
+      index -= (int) sz;
+      context = context->prev;
+    }
+
+  /* We will have to read some new tokens after all (and do so
+     without invalidating preceding tokens).  */
+  kount = index;
+  pfile->keep_tokens++;
+
+  do
+    {
+      peektok = _cpp_lex_token (pfile);
+      if (peektok->type == CPP_EOF)
+	return peektok;
+    }
+  while (index--);
+
+  _cpp_backup_tokens_direct (pfile, kount + 1);
+  pfile->keep_tokens--;
+
+  return peektok;
+}
+
 /* Allocate a single token that is invalidated at the same time as the
    rest of the tokens on the line.  Has its line and col set to the
    same as the last lexed token, so that diagnostics appear in the
@@ -742,9 +785,30 @@ cpp_token *
 _cpp_temp_token (cpp_reader *pfile)
 {
   cpp_token *old, *result;
+  ptrdiff_t sz = pfile->cur_run->limit - pfile->cur_token;
+  ptrdiff_t la = (ptrdiff_t) pfile->lookaheads;
 
   old = pfile->cur_token - 1;
-  if (pfile->cur_token == pfile->cur_run->limit)
+  /* Any pre-existing lookaheads must not be clobbered.  */
+  if (la)
+    {
+      if (sz <= la)
+        {
+          tokenrun *next = next_tokenrun (pfile->cur_run);
+
+          if (sz < la)
+            memmove (next->base + 1, next->base,
+                     (la - sz) * sizeof (cpp_token));
+
+          next->base[0] = pfile->cur_run->limit[-1];
+        }
+
+      if (sz > 1)
+        memmove (pfile->cur_token + 1, pfile->cur_token,
+                 MIN (la, sz - 1) * sizeof (cpp_token));
+    }
+
+  if (!sz && pfile->cur_token == pfile->cur_run->limit)
     {
       pfile->cur_run = next_tokenrun (pfile->cur_run);
       pfile->cur_token = pfile->cur_run->base;