From: Adhemerval Zanella <adhemerval.zanella@linaro.org>
To: Max Gautier <mg@max.gautier.name>, libc-alpha@sourceware.org
Subject: Re: [PATCH v5 3/4] iconv: make utf-7.c able to use variants
Date: Mon, 21 Mar 2022 09:24:04 -0300 [thread overview]
Message-ID: <cf57463b-23db-3d66-c118-d7f0e36ac9e8@linaro.org> (raw)
In-Reply-To: <YjdZYKh5dcGLr+0O@ol-mgautier>
On 20/03/2022 13:42, Max Gautier via Libc-alpha wrote:
> Add infrastructure in utf-7.c to handle variants. The approach comes from
> iso646.c
> The variant is defined at gconv_init time and is passed as a
> supplementary variable.
>
> Signed-off-by: Max Gautier <mg@max.gautier.name>
Patch looks ok, although it should be refactor to add 'enum variant' argument
on isdirect and isxdirect instead of relying on previous patch (to keep the
patch consistent).
> ---
> iconvdata/utf-7.c | 230 ++++++++++++++++++++++++++++++++++------------
> 1 file changed, 170 insertions(+), 60 deletions(-)
>
> diff --git a/iconvdata/utf-7.c b/iconvdata/utf-7.c
> index 15f3669ac8..b639d8ff3e 100644
> --- a/iconvdata/utf-7.c
> +++ b/iconvdata/utf-7.c
> @@ -29,6 +29,24 @@
> #include <stdlib.h>
>
>
> +enum variant
> +{
> + UTF7,
> +};
> +
> +/* Must be in the same order as enum variant above. */
> +static const char names[] =
> + "UTF-7//\0"
> + "\0";
> +
> +static uint32_t
> +shift_character (enum variant const var)
> +{
> + if (var == UTF7)
> + return '+';
> + else
> + abort ();
> +}
>
> static bool
> between (uint32_t const ch,
> @@ -38,23 +56,27 @@ between (uint32_t const ch,
> }
>
> /* The set of "direct characters":
> + FOR UTF-7
> A-Z a-z 0-9 ' ( ) , - . / : ? space tab lf cr
> */
>
> static bool
> isdirect (uint32_t ch, enum variant var)
> {
> - return (between (ch, 'A', 'Z')
> - || between (ch, 'a', 'z')
> - || between (ch, '0', '9')
> - || ch == '\'' || ch == '(' || ch == ')'
> - || between (ch, ',', '/')
> - || ch == ':' || ch == '?'
> - || ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r');
> + if (var == UTF7)
> + return (between (ch, 'A', 'Z')
> + || between (ch, 'a', 'z')
> + || between (ch, '0', '9')
> + || ch == '\'' || ch == '(' || ch == ')'
> + || between (ch, ',', '/')
> + || ch == ':' || ch == '?'
> + || ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r');
> + abort ();
> }
>
>
> /* The set of "direct and optional direct characters":
> + (UTF-7 only)
> A-Z a-z 0-9 ' ( ) , - . / : ? space tab lf cr
> ! " # $ % & * ; < = > @ [ ] ^ _ ` { | }
> */
> @@ -62,10 +84,15 @@ isdirect (uint32_t ch, enum variant var)
> static bool
> isxdirect (uint32_t ch, enum variant var)
> {
> - return (ch == '\t'
> - || ch == '\n'
> - || ch == '\r'
> - || (between (ch, ' ', '}') && ch != '+' && ch != '\\'));
> + if (isdirect (ch, var))
> + return true;
> + if (var != UTF7)
> + return false;
> + return between (ch, '!', '&')
> + || ch == '*'
> + || between (ch, ';', '@')
> + || (between (ch, '[', '`') && ch != '\\')
> + || between (ch, '{', '}');
> }
>
>
> @@ -85,7 +112,7 @@ needs_explicit_shift (uint32_t ch)
>
> /* Converts a value in the range 0..63 to a base64 encoded char. */
> static unsigned char
> -base64 (unsigned int i)
> +base64 (unsigned int i, enum variant var)
> {
> if (i < 26)
> return i + 'A';
> @@ -95,7 +122,7 @@ base64 (unsigned int i)
> return i - 52 + '0';
> else if (i == 62)
> return '+';
> - else if (i == 63)
> + else if (i == 63 && var == UTF7)
> return '/';
> else
> abort ();
> @@ -103,9 +130,8 @@ base64 (unsigned int i)
>
>
> /* Definitions used in the body of the `gconv' function. */
> -#define CHARSET_NAME "UTF-7//"
> -#define DEFINE_INIT 1
> -#define DEFINE_FINI 1
> +#define DEFINE_INIT 0
> +#define DEFINE_FINI 0
> #define FROM_LOOP from_utf7_loop
> #define TO_LOOP to_utf7_loop
> #define MIN_NEEDED_FROM 1
> @@ -113,11 +139,27 @@ base64 (unsigned int i)
> #define MIN_NEEDED_TO 4
> #define MAX_NEEDED_TO 4
> #define ONE_DIRECTION 0
> +#define FROM_DIRECTION (dir == from_utf7)
> #define PREPARE_LOOP \
> mbstate_t saved_state; \
> - mbstate_t *statep = data->__statep;
> -#define EXTRA_LOOP_ARGS , statep
> + mbstate_t *statep = data->__statep; \
> + enum direction dir = ((struct utf7_data *) step->__data)->dir; \
> + enum direction var = ((struct utf7_data *) step->__data)->var;
> +#define EXTRA_LOOP_ARGS , statep, var
> +
>
> +enum direction
> +{
> + illegal_dir,
> + from_utf7,
> + to_utf7
> +};
> +
> +struct utf7_data
> +{
> + enum direction dir;
> + enum variant var;
> +};
>
> /* Since we might have to reset input pointer we must be able to save
> and restore the state. */
> @@ -127,6 +169,70 @@ base64 (unsigned int i)
> else \
> *statep = saved_state
>
> +int
> +gconv_init (struct __gconv_step *step)
> +{
> + /* Determine which direction. */
> + struct utf7_data *new_data;
> + enum direction dir = illegal_dir;
> +
> + enum variant var = 0;
> + for (const char *name = names; *name != '\0';
> + name = __rawmemchr (name, '\0') + 1)
> + {
> + if (__strcasecmp (step->__from_name, name) == 0)
> + {
> + dir = from_utf7;
> + break;
> + }
> + else if (__strcasecmp (step->__to_name, name) == 0)
> + {
> + dir = to_utf7;
> + break;
> + }
> + ++var;
> + }
> +
> + if (__glibc_likely (dir != illegal_dir))
> + {
> + new_data = malloc (sizeof (*new_data));
> + if (new_data == NULL)
> + return __GCONV_NOMEM;
> +
> + new_data->dir = dir;
> + new_data->var = var;
> + step->__data = new_data;
> +
> + if (dir == from_utf7)
> + {
> + step->__min_needed_from = MIN_NEEDED_FROM;
> + step->__max_needed_from = MAX_NEEDED_FROM;
> + step->__min_needed_to = MIN_NEEDED_TO;
> + step->__max_needed_to = MAX_NEEDED_TO;
> + }
> + else
> + {
> + step->__min_needed_from = MIN_NEEDED_TO;
> + step->__max_needed_from = MAX_NEEDED_TO;
> + step->__min_needed_to = MIN_NEEDED_FROM;
> + step->__max_needed_to = MAX_NEEDED_FROM;
> + }
> + }
> + else
> + return __GCONV_NOCONV;
> +
> + step->__stateful = 1;
> +
> + return __GCONV_OK;
> +}
> +
> +void
> +gconv_end (struct __gconv_step *data)
> +{
> + free (data->__data);
> +}
> +
> +
>
> /* First define the conversion function from UTF-7 to UCS4.
> The state is structured as follows:
> @@ -154,13 +260,13 @@ base64 (unsigned int i)
> if ((statep->__count >> 3) == 0) \
> { \
> /* base64 encoding inactive. */ \
> - if (isxdirect (ch)) \
> + if (isxdirect (ch, var)) \
> { \
> inptr++; \
> put32 (outptr, ch); \
> outptr += 4; \
> } \
> - else if (__glibc_likely (ch == '+')) \
> + else if (__glibc_likely (ch == shift_character (var))) \
> { \
> if (__glibc_unlikely (inptr + 2 > inend)) \
> { \
> @@ -285,7 +391,7 @@ base64 (unsigned int i)
> } \
> }
> #define LOOP_NEED_FLAGS
> -#define EXTRA_LOOP_DECLS , mbstate_t *statep
> +#define EXTRA_LOOP_DECLS , mbstate_t *statep, enum variant var
> #include <iconv/loop.c>
>
>
> @@ -316,7 +422,7 @@ base64 (unsigned int i)
> if ((statep->__count & 0x18) == 0) \
> { \
> /* base64 encoding inactive */ \
> - if (isdirect (ch)) \
> + if (isdirect (ch, var)) \
> { \
> *outptr++ = (unsigned char) ch; \
> } \
> @@ -324,7 +430,7 @@ base64 (unsigned int i)
> { \
> size_t count; \
> \
> - if (ch == '+') \
> + if (ch == shift_character (var)) \
> count = 2; \
> else if (ch < 0x10000) \
> count = 3; \
> @@ -339,13 +445,13 @@ base64 (unsigned int i)
> break; \
> } \
> \
> - *outptr++ = '+'; \
> - if (ch == '+') \
> + *outptr++ = shift_character (var); \
> + if (ch == shift_character (var)) \
> *outptr++ = '-'; \
> else if (ch < 0x10000) \
> { \
> - *outptr++ = base64 (ch >> 10); \
> - *outptr++ = base64 ((ch >> 4) & 0x3f); \
> + *outptr++ = base64 (ch >> 10, var); \
> + *outptr++ = base64 ((ch >> 4) & 0x3f, var); \
> statep->__count = ((ch & 15) << 5) | (3 << 3); \
> } \
> else if (ch < 0x110000) \
> @@ -354,11 +460,11 @@ base64 (unsigned int i)
> uint32_t ch2 = 0xdc00 + ((ch - 0x10000) & 0x3ff); \
> \
> ch = (ch1 << 16) | ch2; \
> - *outptr++ = base64 (ch >> 26); \
> - *outptr++ = base64 ((ch >> 20) & 0x3f); \
> - *outptr++ = base64 ((ch >> 14) & 0x3f); \
> - *outptr++ = base64 ((ch >> 8) & 0x3f); \
> - *outptr++ = base64 ((ch >> 2) & 0x3f); \
> + *outptr++ = base64 (ch >> 26, var); \
> + *outptr++ = base64 ((ch >> 20) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 14) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 8) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 2) & 0x3f, var); \
> statep->__count = ((ch & 3) << 7) | (2 << 3); \
> } \
> else \
> @@ -368,7 +474,7 @@ base64 (unsigned int i)
> else \
> { \
> /* base64 encoding active */ \
> - if (isdirect (ch)) \
> + if (isdirect (ch, var)) \
> { \
> /* deactivate base64 encoding */ \
> size_t count; \
> @@ -382,7 +488,7 @@ base64 (unsigned int i)
> } \
> \
> if ((statep->__count & 0x18) >= 0x10) \
> - *outptr++ = base64 ((statep->__count >> 3) & ~3); \
> + *outptr++ = base64 ((statep->__count >> 3) & ~3, var); \
> if (needs_explicit_shift (ch)) \
> *outptr++ = '-'; \
> *outptr++ = (unsigned char) ch; \
> @@ -410,22 +516,24 @@ base64 (unsigned int i)
> switch ((statep->__count >> 3) & 3) \
> { \
> case 1: \
> - *outptr++ = base64 (ch >> 10); \
> - *outptr++ = base64 ((ch >> 4) & 0x3f); \
> + *outptr++ = base64 (ch >> 10, var); \
> + *outptr++ = base64 ((ch >> 4) & 0x3f, var); \
> statep->__count = ((ch & 15) << 5) | (3 << 3); \
> break; \
> case 2: \
> *outptr++ = \
> - base64 (((statep->__count >> 3) & ~3) | (ch >> 12)); \
> - *outptr++ = base64 ((ch >> 6) & 0x3f); \
> - *outptr++ = base64 (ch & 0x3f); \
> + base64 (((statep->__count >> 3) & ~3) | (ch >> 12), \
> + var); \
> + *outptr++ = base64 ((ch >> 6) & 0x3f, var); \
> + *outptr++ = base64 (ch & 0x3f, var); \
> statep->__count = (1 << 3); \
> break; \
> case 3: \
> *outptr++ = \
> - base64 (((statep->__count >> 3) & ~3) | (ch >> 14)); \
> - *outptr++ = base64 ((ch >> 8) & 0x3f); \
> - *outptr++ = base64 ((ch >> 2) & 0x3f); \
> + base64 (((statep->__count >> 3) & ~3) | (ch >> 14), \
> + var); \
> + *outptr++ = base64 ((ch >> 8) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 2) & 0x3f, var); \
> statep->__count = ((ch & 3) << 7) | (2 << 3); \
> break; \
> default: \
> @@ -441,30 +549,32 @@ base64 (unsigned int i)
> switch ((statep->__count >> 3) & 3) \
> { \
> case 1: \
> - *outptr++ = base64 (ch >> 26); \
> - *outptr++ = base64 ((ch >> 20) & 0x3f); \
> - *outptr++ = base64 ((ch >> 14) & 0x3f); \
> - *outptr++ = base64 ((ch >> 8) & 0x3f); \
> - *outptr++ = base64 ((ch >> 2) & 0x3f); \
> + *outptr++ = base64 (ch >> 26, var); \
> + *outptr++ = base64 ((ch >> 20) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 14) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 8) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 2) & 0x3f, var); \
> statep->__count = ((ch & 3) << 7) | (2 << 3); \
> break; \
> case 2: \
> *outptr++ = \
> - base64 (((statep->__count >> 3) & ~3) | (ch >> 28)); \
> - *outptr++ = base64 ((ch >> 22) & 0x3f); \
> - *outptr++ = base64 ((ch >> 16) & 0x3f); \
> - *outptr++ = base64 ((ch >> 10) & 0x3f); \
> - *outptr++ = base64 ((ch >> 4) & 0x3f); \
> + base64 (((statep->__count >> 3) & ~3) | (ch >> 28), \
> + var); \
> + *outptr++ = base64 ((ch >> 22) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 16) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 10) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 4) & 0x3f, var); \
> statep->__count = ((ch & 15) << 5) | (3 << 3); \
> break; \
> case 3: \
> *outptr++ = \
> - base64 (((statep->__count >> 3) & ~3) | (ch >> 30)); \
> - *outptr++ = base64 ((ch >> 24) & 0x3f); \
> - *outptr++ = base64 ((ch >> 18) & 0x3f); \
> - *outptr++ = base64 ((ch >> 12) & 0x3f); \
> - *outptr++ = base64 ((ch >> 6) & 0x3f); \
> - *outptr++ = base64 (ch & 0x3f); \
> + base64 (((statep->__count >> 3) & ~3) | (ch >> 30), \
> + var); \
> + *outptr++ = base64 ((ch >> 24) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 18) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 12) & 0x3f, var); \
> + *outptr++ = base64 ((ch >> 6) & 0x3f, var); \
> + *outptr++ = base64 (ch & 0x3f, var); \
> statep->__count = (1 << 3); \
> break; \
> default: \
> @@ -480,7 +590,7 @@ base64 (unsigned int i)
> inptr += 4; \
> }
> #define LOOP_NEED_FLAGS
> -#define EXTRA_LOOP_DECLS , mbstate_t *statep
> +#define EXTRA_LOOP_DECLS , mbstate_t *statep, enum variant var
> #include <iconv/loop.c>
>
>
> @@ -510,7 +620,7 @@ base64 (unsigned int i)
> { \
> /* Write out the shift sequence. */ \
> if ((state & 0x18) >= 0x10) \
> - *outbuf++ = base64 ((state >> 3) & ~3); \
> + *outbuf++ = base64 ((state >> 3) & ~3, var); \
> *outbuf++ = '-'; \
> \
> data->__statep->__count = 0; \
next prev parent reply other threads:[~2022-03-21 12:24 UTC|newest]
Thread overview: 60+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-08-19 23:06 [PATCH 0/5] iconv: module for MODIFIED-UTF-7 Max Gautier
2020-08-19 23:06 ` [PATCH 1/5] Copy utf-7 module to modified-utf-7 Max Gautier
2020-08-19 23:06 ` [PATCH 2/5] Update gconv-modules file Max Gautier
2020-08-19 23:07 ` [PATCH 3/5] Transform UTF-7 to MODIFIED-UTF-7 Max Gautier
2020-08-19 23:07 ` [PATCH 4/5] Make terminating base64 sequences mandatory Max Gautier
2020-08-19 23:07 ` [PATCH 5/5] Add test case for MODIFIED-UTF-7 Max Gautier
2020-08-20 7:18 ` Andreas Schwab
2020-08-20 15:40 ` [PATCH v2 " Max Gautier
2020-08-20 8:03 ` [PATCH 0/5] iconv: module " Florian Weimer
2020-08-20 15:19 ` Max Gautier
2020-08-20 15:58 ` Florian Weimer
2020-09-02 15:24 ` Max Gautier
2020-09-02 20:01 ` Adhemerval Zanella
2020-09-03 9:47 ` Max Gautier
2020-09-03 10:56 ` Andreas Schwab
2021-01-25 9:02 ` [PATCH v3 0/5] iconv: module for IMAP-UTF-7 Max Gautier
2021-01-25 9:02 ` [PATCH v3 1/5] Copy utf-7 module to modified-utf-7 Max Gautier
2021-01-25 9:31 ` Andreas Schwab
2021-01-25 13:51 ` Max Gautier
2021-02-07 9:42 ` Florian Weimer
2021-02-07 12:29 ` Max Gautier
2021-02-07 12:34 ` Florian Weimer
2021-12-09 9:31 ` [PATCH v4 0/4] iconv: Add support for UTF-7-IMAP Max Gautier
2021-12-09 9:31 ` [PATCH v4 1/4] iconv: Always encode "optional direct" UTF-7 characters Max Gautier
2022-03-07 12:10 ` Adhemerval Zanella
2021-12-09 9:31 ` [PATCH v4 2/4] iconv: Better mapping to RFC for UTF-7 Max Gautier
2022-03-07 12:14 ` Adhemerval Zanella
2022-03-20 16:41 ` [PATCH v5 " Max Gautier
2022-03-21 11:53 ` Adhemerval Zanella
2022-03-21 11:59 ` Adhemerval Zanella
2022-03-21 12:06 ` Adhemerval Zanella
2022-03-21 14:07 ` Max Gautier
2021-12-09 9:31 ` [PATCH v4 3/4] iconv: make utf-7.c able to use variants Max Gautier
2022-03-07 12:34 ` Adhemerval Zanella
2022-03-12 11:07 ` Max Gautier
2022-03-14 12:17 ` Adhemerval Zanella
2022-03-20 16:42 ` [PATCH v5 " Max Gautier
2022-03-21 12:24 ` Adhemerval Zanella [this message]
2021-12-09 9:31 ` [PATCH v4 4/4] iconv: Add UTF-7-IMAP variant in utf-7.c Max Gautier
2022-03-07 12:46 ` Adhemerval Zanella
2022-03-20 16:43 ` [PATCH v5 " Max Gautier
2022-03-21 12:24 ` Adhemerval Zanella
2021-12-17 13:15 ` [PATCH v4 0/4] iconv: Add support for UTF-7-IMAP Max Gautier
2022-01-24 14:19 ` Adhemerval Zanella
2022-02-10 13:16 ` Max Gautier
2022-02-10 13:17 ` Adhemerval Zanella
2022-03-04 8:53 ` Max Gautier
2022-01-17 14:07 ` Max Gautier
2022-01-24 9:17 ` Max Gautier
2021-01-25 9:02 ` [PATCH v3 2/5] Update gconv-modules file Max Gautier
2021-02-07 9:49 ` Florian Weimer
2021-01-25 9:02 ` [PATCH v3 3/5] Transform UTF-7 to IMAP-UTF-7 Max Gautier
2021-01-25 9:02 ` [PATCH v3 4/5] Make terminating base64 sequences mandatory Max Gautier
2021-02-07 9:45 ` Florian Weimer
2021-01-25 9:02 ` [PATCH v3 5/5] Add test case for IMAP-UTF-7 Max Gautier
2021-02-07 9:49 ` Florian Weimer
2021-03-16 14:39 ` [PATCH v3 5/5][pw utf test] " Siddhesh Poyarekar
2022-03-21 12:28 ` [PATCH v3 0/5] iconv: module " Adhemerval Zanella
2022-03-21 14:09 ` Max Gautier
2021-01-12 9:12 ` [PATCH 0/5] iconv: module for MODIFIED-UTF-7 Florian Weimer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=cf57463b-23db-3d66-c118-d7f0e36ac9e8@linaro.org \
--to=adhemerval.zanella@linaro.org \
--cc=libc-alpha@sourceware.org \
--cc=mg@max.gautier.name \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).