public inbox for gcc-cvs@sourceware.org
help / color / mirror / Atom feed
* [gcc/devel/rust/master] libproc_macro: Add TokenStream implementation
@ 2023-04-06 21:31 Thomas Schwinge
0 siblings, 0 replies; only message in thread
From: Thomas Schwinge @ 2023-04-06 21:31 UTC (permalink / raw)
To: gcc-cvs
https://gcc.gnu.org/g:e4cd0d726ed65e34cfcb8cd6a265095d89983520
commit e4cd0d726ed65e34cfcb8cd6a265095d89983520
Author: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>
Date: Fri Feb 17 16:40:01 2023 +0100
libproc_macro: Add TokenStream implementation
Add the TokenStream rust type implementation to libproc_macro. Also
implement IntoIter structure to iterator over a TokenStream.
ChangeLog:
* librust/proc_macro/rust/bridge.rs: Add group and token_stream
modules.
* librust/proc_macro/rust/group.rs: Add group internal
representation.
* librust/proc_macro/rust/lib.rs: Add TokenStream
implementation.
* librust/proc_macro/rust/literal.rs: Make internal
implementation visible in the whole crate.
* librust/proc_macro/rust/token_stream.rs: Implement IntoIter.
* librust/proc_macro/rust/bridge/group.rs: Add Group internals.
* librust/proc_macro/rust/bridge/token_stream.rs: Add
TokenStream internals.
Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>
Diff:
---
librust/proc_macro/rust/bridge.rs | 2 ++
librust/proc_macro/rust/bridge/group.rs | 9 +++++++
librust/proc_macro/rust/bridge/token_stream.rs | 35 ++++++++++++++++++++++++++
librust/proc_macro/rust/group.rs | 6 ++---
librust/proc_macro/rust/lib.rs | 21 ++++++++++++----
librust/proc_macro/rust/literal.rs | 2 +-
librust/proc_macro/rust/token_stream.rs | 28 +++++++++++++++++----
7 files changed, 89 insertions(+), 14 deletions(-)
diff --git a/librust/proc_macro/rust/bridge.rs b/librust/proc_macro/rust/bridge.rs
index ff2c49688b6..eb7f854bd2b 100644
--- a/librust/proc_macro/rust/bridge.rs
+++ b/librust/proc_macro/rust/bridge.rs
@@ -1,7 +1,9 @@
+pub mod group;
pub mod ident;
pub mod literal;
pub mod punct;
pub mod span;
+pub mod token_stream;
extern "C" {
fn bridge__is_available() -> bool;
diff --git a/librust/proc_macro/rust/bridge/group.rs b/librust/proc_macro/rust/bridge/group.rs
new file mode 100644
index 00000000000..6590f8952fa
--- /dev/null
+++ b/librust/proc_macro/rust/bridge/group.rs
@@ -0,0 +1,9 @@
+use bridge;
+use Delimiter;
+
+#[repr(C)]
+#[derive(Clone)]
+pub struct Group {
+ delimiter: Delimiter,
+ stream: bridge::token_stream::TokenStream,
+}
diff --git a/librust/proc_macro/rust/bridge/token_stream.rs b/librust/proc_macro/rust/bridge/token_stream.rs
new file mode 100644
index 00000000000..307ff6bd8b4
--- /dev/null
+++ b/librust/proc_macro/rust/bridge/token_stream.rs
@@ -0,0 +1,35 @@
+use bridge::{group::Group, ident::Ident, literal::Literal, punct::Punct};
+
+// TODO: There surely is a better way to achieve this. I don't like this
+// "duplication" of the TokenTree enumeration. But I cannot use the public
+// one since it's underlying types (public Group, Ident...) are not ffi safe.
+// Flattening all those struct might be a solution.
+#[repr(C)]
+#[derive(Clone)]
+pub enum TokenTree {
+ Group(Group),
+ Ident(Ident),
+ Punct(Punct),
+ Literal(Literal),
+}
+
+extern "C" {
+ fn TokenStream__new() -> TokenStream;
+}
+
+#[repr(C)]
+#[derive(Clone)]
+pub struct TokenStream {
+ pub(crate) data: *const TokenTree,
+ pub(crate) size: u64,
+}
+
+impl TokenStream {
+ pub fn new() -> Self {
+ unsafe { TokenStream__new() }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ 0 == self.size
+ }
+}
diff --git a/librust/proc_macro/rust/group.rs b/librust/proc_macro/rust/group.rs
index e0e3de98452..6512982cc32 100644
--- a/librust/proc_macro/rust/group.rs
+++ b/librust/proc_macro/rust/group.rs
@@ -1,8 +1,10 @@
+use bridge;
use std::fmt;
use Span;
use TokenStream;
/// Describes how a sequence of token trees is delimited.
+#[repr(C)]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum Delimiter {
/// The sequence is delimited by a parenthesis `(...)`.
@@ -17,9 +19,7 @@ pub enum Delimiter {
/// A delimited token stream.
#[derive(Clone)]
-pub struct Group {
- // Internal implementation details
-}
+pub struct Group(pub(crate) bridge::group::Group);
impl Group {
/// Creates a new `Group`.
diff --git a/librust/proc_macro/rust/lib.rs b/librust/proc_macro/rust/lib.rs
index 4e6802abeda..31aa6ea7c75 100644
--- a/librust/proc_macro/rust/lib.rs
+++ b/librust/proc_macro/rust/lib.rs
@@ -34,6 +34,19 @@ pub enum TokenTree {
Literal(Literal),
}
+type InternalTokenTree = bridge::token_stream::TokenTree;
+
+impl From<InternalTokenTree> for TokenTree {
+ fn from(value: InternalTokenTree) -> Self {
+ match value {
+ InternalTokenTree::Group(g) => TokenTree::Group(Group(g)),
+ InternalTokenTree::Ident(i) => TokenTree::Ident(Ident(i)),
+ InternalTokenTree::Punct(p) => TokenTree::Punct(Punct(p)),
+ InternalTokenTree::Literal(l) => TokenTree::Literal(Literal(l)),
+ }
+ }
+}
+
impl TokenTree {
/// Get the [`Span`] for this TokenTree.
pub fn span(&self) -> Span {
@@ -124,21 +137,19 @@ impl error::Error for LexError {}
/// is both the input and the output of `#[proc_macro]`,
/// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions.
#[derive(Clone)]
-pub struct TokenStream {
- // Internal implementation details
-}
+pub struct TokenStream(bridge::token_stream::TokenStream);
impl TokenStream {
// TODO: Add experimental API functions for this type
/// Creates an empty `TokenStream` containing no token trees.
pub fn new() -> Self {
- todo!("Implement this function")
+ TokenStream(bridge::token_stream::TokenStream::new())
}
/// Checks if this `TokenStream` is empty.
pub fn is_empty(&self) -> bool {
- todo!("Implement this function")
+ self.0.is_empty()
}
}
diff --git a/librust/proc_macro/rust/literal.rs b/librust/proc_macro/rust/literal.rs
index 2820da65021..4011201d998 100644
--- a/librust/proc_macro/rust/literal.rs
+++ b/librust/proc_macro/rust/literal.rs
@@ -21,7 +21,7 @@ use Span;
/// Boolean literals like `true` and `false` are `Ident`s and do not belong
/// here.
#[derive(Clone)]
-pub struct Literal(bridge::literal::Literal);
+pub struct Literal(pub(crate) bridge::literal::Literal);
impl Literal {
// TODO: Add experimental API functions for this type
diff --git a/librust/proc_macro/rust/token_stream.rs b/librust/proc_macro/rust/token_stream.rs
index 3de89dfa35f..c4caf9f9ff8 100644
--- a/librust/proc_macro/rust/token_stream.rs
+++ b/librust/proc_macro/rust/token_stream.rs
@@ -1,26 +1,40 @@
//! Public implementation details for the `TokenStream` type, such as iterators.
+use bridge;
+use std::convert::TryInto;
+
use TokenStream;
use TokenTree;
/// An iterator over [`TokenStream`]'s [`TokenTree`]s.
#[derive(Clone)]
pub struct IntoIter {
- // Internal implementation details
+ current: *const bridge::token_stream::TokenTree,
+ end: *const bridge::token_stream::TokenTree,
}
impl Iterator for IntoIter {
type Item = TokenTree;
fn next(&mut self) -> Option<TokenTree> {
- todo!("Implement this function")
+ if self.current == self.end {
+ return None;
+ } else {
+ let result = self.current;
+ self.current = unsafe { self.current.add(1) };
+ Some(unsafe { std::ptr::read(result) }.into())
+ }
}
fn size_hint(&self) -> (usize, Option<usize>) {
- todo!("Implement this function")
+ // TODO: I'm not a fan of those casts, once #![feature(ptr_sub_ptr)]
+ // is implemented we may replace this line by the following:
+ // self.end.sub_ptr(self.current)
+ let remaining = self.end as usize - self.current as usize;
+ (remaining, Some(remaining))
}
fn count(self) -> usize {
- todo!("Implement this function")
+ self.end as usize - self.current as usize
}
}
@@ -29,6 +43,10 @@ impl IntoIterator for TokenStream {
type IntoIter = IntoIter;
fn into_iter(self) -> IntoIter {
- todo!("Implement this function")
+ let capacity = self.0.size.try_into().unwrap();
+ IntoIter {
+ current: self.0.data,
+ end: unsafe { self.0.data.add(capacity) },
+ }
}
}
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2023-04-06 21:31 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-04-06 21:31 [gcc/devel/rust/master] libproc_macro: Add TokenStream implementation Thomas Schwinge
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).