From 9c5d6b1adcf949269e3fceeaf31203921745d2c9 Mon Sep 17 00:00:00 2001
From: mintty <mintty@users.noreply.github.com>
Date: Mon, 14 Aug 2017 21:59:25 +0200
Subject: [PATCH 1/4] creation of width data, supporting Unicode updates

---
 newlib/libc/string/Makefile.widthdata |  47 +++
 newlib/libc/string/mkwide             |  49 +++
 newlib/libc/string/mkwidthA           |  20 +
 newlib/libc/string/uniset             | 678 ++++++++++++++++++++++++++++++++++
 4 files changed, 794 insertions(+)
 create mode 100644 newlib/libc/string/Makefile.widthdata
 create mode 100755 newlib/libc/string/mkwide
 create mode 100755 newlib/libc/string/mkwidthA
 create mode 100755 newlib/libc/string/uniset

diff --git a/newlib/libc/string/Makefile.widthdata b/newlib/libc/string/Makefile.widthdata
new file mode 100644
index 0000000..14adab5
--- /dev/null
+++ b/newlib/libc/string/Makefile.widthdata
@@ -0,0 +1,47 @@
+#############################################################################
+# generate Unicode width data for newlib/libc/string/wcwidth.c
+
+
+#############################################################################
+# table sets to be generated
+
+widthdata=combining.t ambiguous.t wide.t
+
+widthdata:	$(widthdata)
+
+
+#############################################################################
+# tools and data
+
+#WGET=wget -N -t 1 --timeout=55
+WGET=curl -R -O --connect-timeout 55
+WGET+=-z $@
+
+%.txt:
+	ln -s /usr/share/unicode/ucd/$@ . || $(WGET) http://unicode.org/Public/UNIDATA/$@
+
+uniset.tar.gz:
+	$(WGET) http://www.cl.cam.ac.uk/~mgk25/download/uniset.tar.gz
+
+uniset:	uniset.tar.gz
+	gzip -dc uniset.tar.gz | tar xvf - uniset
+
+
+#############################################################################
+# width data for libc/string/wcwidth.c
+
+combining.t:	uniset UnicodeData.txt Blocks.txt
+	PATH="${PATH}:." uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B +D7B0-D7C6 +D7CB-D7FB c > combining.t
+
+WIDTH-A:	uniset UnicodeData.txt Blocks.txt EastAsianWidth.txt
+	PATH="${PATH}:." sh ./mkwidthA
+
+ambiguous.t:	uniset WIDTH-A UnicodeData.txt Blocks.txt
+	PATH="${PATH}:." uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c > ambiguous.t
+
+wide.t:	uniset UnicodeData.txt Blocks.txt EastAsianWidth.txt
+	PATH="${PATH}:." sh ./mkwide
+
+
+#############################################################################
+# end
diff --git a/newlib/libc/string/mkwide b/newlib/libc/string/mkwide
new file mode 100755
index 0000000..55a0bab
--- /dev/null
+++ b/newlib/libc/string/mkwide
@@ -0,0 +1,49 @@
+#! /bin/sh
+
+# generate list of wide characters, with convex closure
+
+skipcheck=false
+
+if [ ! -r EastAsianWidth.txt ]
+then	ln -s /usr/share/unicode/ucd/EastAsianWidth.txt . || exit 1
+fi
+if [ ! -r UnicodeData.txt ]
+then	ln -s /usr/share/unicode/ucd/UnicodeData.txt . || exit 1
+fi
+if [ ! -r Blocks.txt ]
+then	ln -s /usr/share/unicode/ucd/Blocks.txt . || exit 1
+fi
+
+sed -e "s,^\([^;]*\);[NAH],\1," -e t -e d EastAsianWidth.txt > wide.na
+sed -e "s,^\([^;]*\);[WF],\1," -e t -e d EastAsianWidth.txt > wide.fw
+
+PATH="$PATH:." # for uniset
+
+nrfw=`uniset +wide.fw nr | sed -e 's,.*:,,'`
+echo FW $nrfw
+nrna=`uniset +wide.na nr | sed -e 's,.*:,,'`
+echo NAH $nrna
+
+extrablocks="2E80-303E"
+
+# check all blocks
+includes () {
+	nr=`uniset +wide.$2 -$1 nr | sed -e 's,.*:,,'`
+	test $nr != $3
+}
+echo "adding compact closure of wide ranges, this may take ~10min"
+for b in $extrablocks `sed -e 's,^\([0-9A-F]*\)\.\.\([0-9A-F]*\).*,\1-\2,' -e t -e d Blocks.txt`
+do	range=$b
+	echo checking $range $* >&2
+	if includes $range fw $nrfw && ! includes $range na $nrna
+	then	echo $range
+	fi
+done > wide.blocks
+
+(
+sed -e "s,^,//," -e 1q EastAsianWidth.txt
+sed -e "s,^,//," -e 1q Blocks.txt
+uniset `sed -e 's,^,+,' wide.blocks` +wide.fw c
+) > wide.t
+
+rm -f wide.na wide.fw wide.blocks
diff --git a/newlib/libc/string/mkwidthA b/newlib/libc/string/mkwidthA
new file mode 100755
index 0000000..343ab40
--- /dev/null
+++ b/newlib/libc/string/mkwidthA
@@ -0,0 +1,20 @@
+#! /bin/sh
+
+# generate WIDTH-A file, listing Unicode characters with width property
+# Ambiguous, from EastAsianWidth.txt
+
+if [ ! -r EastAsianWidth.txt ]
+then	ln -s /usr/share/unicode/ucd/EastAsianWidth.txt . || exit 1
+fi
+if [ ! -r UnicodeData.txt ]
+then	ln -s /usr/share/unicode/ucd/UnicodeData.txt . || exit 1
+fi
+if [ ! -r Blocks.txt ]
+then	ln -s /usr/share/unicode/ucd/Blocks.txt . || exit 1
+fi
+
+sed -e "s,^\([^;]*\);A,\1," -e t -e d EastAsianWidth.txt > width-a-new
+rm -f WIDTH-A
+echo "# UAX #11: East Asian Ambiguous" > WIDTH-A
+PATH="$PATH:." uniset +width-a-new compact >> WIDTH-A
+rm -f width-a-new
diff --git a/newlib/libc/string/uniset b/newlib/libc/string/uniset
new file mode 100755
index 0000000..415e219
--- /dev/null
+++ b/newlib/libc/string/uniset
@@ -0,0 +1,678 @@
+#!/usr/bin/perl
+# Uniset -- Unicode subset manager -- Markus Kuhn
+# http://www.cl.cam.ac.uk/~mgk25/download/uniset.tar.gz
+# $Id: uniset,v 1.18 2004-04-10 21:19:39+01 mgk25 Exp mgk25 $
+
+require 5.008;
+use open ':utf8';
+
+binmode(STDOUT, ":utf8");
+binmode(STDIN, ":utf8");
+
+my (%name, %invname, %category, %comment);
+
+print <<End if $#ARGV < 0;
+Uniset -- Unicode subset manager -- Markus Kuhn
+
+Uniset allows to merge and subtract Unicode subsets. It can output and
+analyse the resulting set in various formats.
+
+The following commands can be supplied to uniset on the command line:
+
+Commands to define a set of characters:
+
+  + filename   add the character set described in the file to the set
+  - filename   remove the character set described in the file from the set
+  +: filename  add the characters in the UTF-8 file to the set
+  -: filename  remove the characters in the UTF-8 file from the set
+  +xxxx..yyyy  add the range to the set (xxxx and yyyy are hex numbers)
+  -xxxx..yyyy  remove the range from the set (xxxx and yyyy are hex numbers)
+  +cat=Xx      add all Unicode characters with category code Xx
+  -cat=Xx      remove all Unicode characters with category code Xx
+  -cat!=Xx     remove all Unicode characters without category code Xx
+  clean        remove any elements that do not appear in the Unicode database
+  unknown      remove any elements that do appear in the Unicode database
+
+Command to output descriptions of the constructed set of characters:
+
+  table        write a full table with one line per character
+  compact      output the set in compact MES format
+  c            output the set as C interval array
+  nr           output the number of characters
+  sources      output a table that shows the number of characters contributed
+               by the various combinations of input sets added with +.
+  utf8-list    output a list of all characters encoded in UTF-8
+
+Commands to tailor the following output commands:
+
+  html         write HTML tables instead of plain text
+  ucs          add the unicode character itself to the table (UTF-8 in
+               plain table, numeric character reference in HTML)
+
+Formats of character set input files read by the + and - command:
+
+Empty lines, white space at the start and end of the line and any
+comment text following a \# are ignored. The following formats are
+recognized
+
+xx yyyy             xx is the hex code in an 8-bit character set and yyyy
+                    is the corresponding Unicode value. Both can optionally
+                    be prefixed by 0x. This is the format used in the
+                    files on <ftp://ftp.unicode.org/Public/MAPPINGS/>.
+
+yyyy                yyyy (optionally prefixed with 0x) is a Unicode character
+                    belonging to the specified subset.
+
+yyyy-yyyy           a range of Unicode characters belonging to
+yyyy..yyyy          the specified subset.
+
+xx yy yy yy-yy yy   xx denotes a row (high-byte) and the yy specify
+                    corresponding low bytes or with a hyphen also ranges of
+                    low bytes in the Unicode values that belong to this
+                    subset. This is also the format that is generated by
+                    the compact command.
+End
+exit 1 if $#ARGV < 0;
+
+
+# Subroutine to identify whether the ISO 10646/Unicode character code
+# ucs belongs into the East Asian Wide (W) or East Asian FullWidth
+# (F) category as defined in Unicode Technical Report #11.
+
+sub iswide ($) {
+    my $ucs = shift(@_);
+
+    return ($ucs >= 0x1100 &&
+	    ($ucs <= 0x115f ||                     # Hangul Jamo
+	     $ucs == 0x2329 || $ucs == 0x232a ||
+	     ($ucs >= 0x2e80 && $ucs <= 0xa4cf &&
+	      $ucs != 0x303f) ||                   # CJK .. Yi
+	     ($ucs >= 0xac00 && $ucs <= 0xd7a3) || # Hangul Syllables
+	     ($ucs >= 0xf900 && $ucs <= 0xfaff) || # CJK Comp. Ideographs
+	     ($ucs >= 0xfe30 && $ucs <= 0xfe6f) || # CJK Comp. Forms
+	     ($ucs >= 0xff00 && $ucs <= 0xff60) || # Fullwidth Forms
+	     ($ucs >= 0xffe0 && $ucs <= 0xffe6) ||
+	     ($ucs >= 0x20000 && $ucs <= 0x2fffd) ||
+	     ($ucs >= 0x30000 && $ucs <= 0x3fffd)));
+}
+
+# Return the Unicode name that belongs to a given character code
+
+# Jamo short names, see Unicode 3.0, table 4-4, page 86
+
+my @lname = ('G', 'GG', 'N', 'D', 'DD', 'R', 'M', 'B', 'BB', 'S', 'SS', '',
+	     'J', 'JJ', 'C', 'K', 'T', 'P', 'H'); # 1100..1112
+my @vname = ('A', 'AE', 'YA', 'YAE', 'EO', 'E', 'YEO', 'YE', 'O',
+	     'WA', 'WAE', 'OE', 'YO', 'U', 'WEO', 'WE', 'WI', 'YU',
+	     'EU', 'YI', 'I'); # 1161..1175
+my @tname = ('G', 'GG', 'GS', 'N', 'NJ', 'NH', 'D', 'L', 'LG', 'LM',
+	     'LB', 'LS', 'LT', 'LP', 'LH', 'M', 'B', 'BS', 'S', 'SS',
+	     'NG', 'J', 'C', 'K', 'T', 'P', 'H'); # 11a8..11c2
+
+sub name {
+    my $ucs = shift(@_);
+    
+    # The intervals used here reflect Unicode Version 3.2
+    if (($ucs >=  0x3400 && $ucs <=  0x4db5) ||
+	($ucs >=  0x4e00 && $ucs <=  0x9fa5) ||
+	($ucs >= 0x20000 && $ucs <= 0x2a6d6)) {
+	return "CJK UNIFIED IDEOGRAPH-" . sprintf("%04X", $ucs);
+    }
+    
+    if ($ucs >= 0xac00 && $ucs <= 0xd7a3) {
+	my $s = $ucs - 0xac00;
+	my $l = 0x1100 + int($s / (21 * 28));
+	my $v = 0x1161 + int(($s % (21 * 28)) / 28);
+	my $t = 0x11a7 + $s % 28;
+	return "HANGUL SYLLABLE " . 
+	    ($lname[int($s / (21 * 28))] .
+	     $vname[int(($s % (21 * 28)) / 28)] .
+	     $tname[$s % 28 - 1]);
+    }
+    
+    return $name{$ucs};
+}
+
+sub is_unicode {
+    my $ucs = shift(@_);
+
+    # The intervals used here reflect Unicode Version 3.2
+    if (($ucs >=  0x3400 && $ucs <=  0x4db5) ||
+	($ucs >=  0x4e00 && $ucs <=  0x9fa5) ||
+	($ucs >=  0xac00 && $ucs <=  0xd7a3) ||
+	($ucs >= 0x20000 && $ucs <= 0x2a6d6)) {
+	return 1;
+    }
+    
+    return exists $name{$ucs};
+}
+
+
+my $html = 0;
+my $image = 0;
+my $adducs = 0;
+my $unicodedata = "UnicodeData.txt";
+my $blockdata = "Blocks.txt";
+my $datadir = "$ENV{HOME}/local/lib/ucs";
+
+# read list of all Unicode names
+if (!open(UDATA, $unicodedata) && !open(UDATA, "$datadir/$unicodedata")) {
+    die ("Can't open Unicode database '$unicodedata':\n$!\n\n" .
+	 "Please make sure that you have downloaded the file\n" .
+	 "ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt\n");
+}
+while (<UDATA>) {
+    if (/^([0-9,A-F]{4,8});([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*);([^;]*)$/) {
+	next if $2 ne '<control>' && substr($2, 0, 1) eq '<';
+	$ucs = hex($1);
+        $name{$ucs} = $2;
+	$invname{$2} = $ucs;
+	$category{$ucs} = $3;
+        $comment{$ucs} = $12;
+    } else {
+        die("Syntax error in line '$_' in file '$unicodedata'");
+    }
+}
+close(UDATA);
+
+# read list of all Unicode blocks
+if (!open(UDATA, $blockdata) && !open(UDATA, "$datadir/$blockdata")) {
+    die ("Can't open Unicode blockname list '$blockdata':\n$!\n\n" .
+	 "Please make sure that you have downloaded the file\n" .
+	 "ftp://ftp.unicode.org/Public/UNIDATA/Blocks.txt\n");
+}
+my $blocks = 0;
+my (@blockstart, @blockend, @blockname);
+while (<UDATA>) {
+    if (/^\s*([0-9,A-F]{4,8})\s*\.\.\s*([0-9,A-F]{4,8})\s*;\s*(.*)$/) {
+        $blockstart[$blocks] = hex($1);
+	$blockend  [$blocks] = hex($2);
+        $blockname [$blocks] = $3;
+	$blocks++;
+    } elsif (/^\s*\#/ || /^\s*$/) {
+	# ignore comments and empty lines
+    } else {
+        die("Syntax error in line '$_' in file '$blockdata'");
+    }
+}
+close(UDATA);
+if ($blockend[$blocks-1] < 0x110000) {
+    $blockstart[$blocks] = 0x110000;
+    $blockend  [$blocks] = 0x7FFFFFFF;
+    $blockname [$blocks] = "Beyond Plane 16";
+    $blocks++;
+}
+
+# process command line arguments
+while ($_ = shift(@ARGV)) {
+    if (/^html$/) {
+	$html = 1;
+    } elsif (/^ucs$/) {
+	$adducs = 1;
+    } elsif (/^img$/) {
+	$html = 1;
+	$image = 1;
+    } elsif (/^template$/) {
+	$template = shift(@ARGV);
+	open(TEMPLATE, $template) || die("Can't open template file '$template': '$!'");
+	while (<TEMPLATE>) {
+	    if (/^\#\s*include\s+\"([^\"]*)\"\s*$/) {
+		open(INCLUDE, $1) || die("Can't open template include file '$1': '$!'");
+		while (<INCLUDE>) {
+		    print $_;
+		}
+		close(INCLUDE);
+	    } elsif (/^\#\s*quote\s+\"([^\"]*)\"\s*$/) {
+		open(INCLUDE, $1) || die("Can't open template include file '$1': '$!'");
+		while (<INCLUDE>) {
+		    s/&/&amp;/g;
+		    s/</&lt;/g;
+		    print $_;
+		}
+		close(INCLUDE);
+	    } else {
+		print $_;
+	    }
+	}
+	close(TEMPLATE);
+    } elsif (/^\+cat=(.+)$/) {
+	# add characters with given category
+	$cat = $1;
+	for $i (keys(%category)) {
+	    $used{$i} = "[${cat}]" if $category{$i} eq $cat;
+	}
+    } elsif (/^\-cat=(.+)$/) {
+	# remove characters with given category
+	$cat = $1;
+	for $i (keys(%category)) {
+	    delete $used{$i} if $category{$i} eq $cat;
+	}
+    } elsif (/^\-cat!=(.+)$/) {
+	# remove characters without given category
+	$cat = $1;
+	for $i (keys(%category)) {
+	    delete $used{$i} unless $category{$i} eq $cat;
+	}
+    } elsif (/^([+-]):(.*)/) {
+	$remove = $1 eq "-";
+	$setfile = $2;
+	$setfile = shift(@ARGV) if $setfile eq "";
+	push(@SETS, $setfile);
+	open(SET, $setfile) || die("Can't open set file '$setfile': '$!'");
+	$setname = $setfile;
+	while (<SET>) {
+	    while ($_) {
+		$i = ord($_);
+		$used{$i} .= "[${setname}]" unless $remove;
+		delete $used{$i} if $remove;
+		$_ = substr($_, 1);
+	    }
+	}
+	close SET;
+    } elsif (/^([+-])(.*)/) {
+	$remove = $1 eq "-";
+	$setfile = $2;
+	$setfile = "$setfile..$setfile" if $setfile =~ /^([0-9A-Fa-f]{4,8})$/;
+	if ($setfile =~ /^([0-9A-Fa-f]{4,8})(-|\.\.)([0-9A-Fa-f]{4,8})$/) {
+	    # handle intervall specification on command line
+	    $first = hex($1);
+	    $last = hex($3);
+	    for ($i = $first; $i <= $last; $i++) {
+		$used{$i} .= "[ARG]" unless $remove;
+		delete $used{$i} if $remove;
+	    }
+	    next;
+	}
+	$setfile = shift(@ARGV) if $setfile eq "";
+	push(@SETS, $setfile);
+	open(SET, $setfile) || die("Can't open set file '$setfile': '$!'");
+	$cedf = ($setfile =~ /cedf/); # detect Kosta Kosti's trans CEDF format by path name
+	$setname = $setfile;
+	$setname =~ s/([^.\[\]]*)\..*/$1/;
+	while (<SET>) {
+	    if (/^<code_set_name>/) {
+		# handle ISO 15897 (POSIX registry) charset mapping format
+		undef $comment_char;
+		undef $escape_char;
+		while (<SET>) {
+		    if ($comment_char && /^$comment_char/) {
+			# remove comments
+			$_ = $`;
+		    }
+		    next if (/^\032?\s*$/);                                             # skip empty lines
+		    if (/^<comment_char> (\S)$/) {
+			$comment_char = $1;
+		    } elsif (/^<escape_char> (\S)$/) {
+			$escape_char = $1;
+		    } elsif (/^(END )?CHARMAP$/) {
+			#ignore
+		    } elsif (/^<.*>\s*\/x([0-9A-F]{2})\s*<U([0-9A-F]{4,8})>/) {
+			$used{hex($2)} .= "[${setname}{$1}]" unless $remove;
+			delete $used{hex($2)} if $remove;
+		    } else {
+			die("Syntax error in line $. in file '$setfile':\n'$_'\n");
+		    }
+		}
+		next;
+	    } elsif (/^STARTFONT /) {
+		# handle X11 BDF file
+		while (<SET>) {
+		    if (/^ENCODING\s+([0-9]+)/) { 
+			$used{$1} .= "[${setname}]" unless $remove;
+			delete $used{$1} if $remove;
+		    }
+		}
+		next;
+	    }
+	    tr/a-z/A-Z/;           # make input uppercase
+	    if ($cedf) {
+		if ($. > 4) {
+		    if (/^([0-9A-F]{2})\t.?\t(.*)$/) {
+			# handle Kosta Kosti's trans CEDF format
+			next if (hex($1) < 32 || (hex($1) > 0x7e && hex($1) < 0xa0));
+			$ucs = $invname{$2};
+			die "unknown ISO 10646 name '$2' in '$setfile' line $..\n" if ! $ucs;
+			$used{$ucs} .= "[${setname}{$1}]" unless $remove;
+			delete $used{$ucs} if $remove;
+		    } else {
+			die("Syntax error in line $. in CEDF file '$setfile':\n'$_'\n");
+		    }
+		}
+		next;
+	    }
+	    if (/^\s*(0X|U\+|U-)?([0-9A-F]{2})\s+\#\s*UNDEFINED\s*$/) {
+		# ignore ftp.unicode.org mapping file lines with #UNDEFINED
+		next;
+	    }
+	    s/^([^\#]*)\#.*$/$1/;  # remove comments
+	    next if (/^\032?\s*$/);     # skip empty lines
+	    if (/^\s*(0X)?([0-9A-F-]{2})\s+(0X|U\+|U-)?([0-9A-F]{4,8})\s*$/) {
+		# handle entry from a ftp.unicode.org mapping file
+		$used{hex($4)} .= "[${setname}{$2}]" unless $remove;
+		delete $used{hex($4)} if $remove;
+	    } elsif (/^\s*(0X|U\+|U-)?([0-9A-F]{4,8})(\s*-\s*|\s*\.\.\s*|\s+)(0X|U\+|U-)?([0-9A-F]{4,8})\s*$/) {
+		# handle interval specification
+		$first = hex($2);
+		$last = hex($5);
+		for ($i = $first; $i <= $last; $i++) {
+		    $used{$i} .= "[${setname}]" unless $remove;
+		    delete $used{$i} if $remove;
+		}
+	    } elsif (/^\s*([0-9A-F]{2,6})(\s+[0-9A-F]{2},?|\s+[0-9A-F]{2}-[0-9A-F]{2},?)+/) {
+		# handle lines from P10 MES draft
+		$row = $1;
+		$cols = $_;
+		$cols =~ s/^\s*([0-9A-F]{2,6})\s*(.*)\s*$/$2/;
+		$cols =~ tr/,//d;
+		@cols = split(/\s+/, $cols);
+		for (@cols) {
+		    if (/^(..)$/) {
+			$first = hex("$row$1");
+			$last  = $first;
+		    } elsif (/^(..)-(..)$/) {
+			$first = hex("$row$1");
+			$last  = hex("$row$2");
+		    } else {
+			die ("this should never happen '$_'");
+		    }
+		    for ($i = $first; $i <= $last; $i++) {
+			$used{$i} .= "[${setname}]" unless $remove;
+			delete $used{$i} if $remove;
+		    }
+		}
+	    } elsif (/^\s*(0X|U\+|U-)?([0-9A-F]{4,8})\s*/) {
+		# handle single character
+		$used{hex($2)} .= "[${setname}]" unless $remove;
+		delete $used{hex($2)} if $remove;
+	    } else {
+		die("Syntax error in line $. in file '$setfile':\n'$_'\n") unless /^\s*(\#.*)?$/;
+	    }
+	}
+	close SET;
+    } elsif (/^loadimages$/ || /^loadbigimages$/) {
+	if (/^loadimages$/) {
+	    $prefix = "Small.Glyphs";
+	} else {
+	    $prefix = "Glyphs";
+	}
+	$total = 0;
+	for $i (keys(%used)) {
+	    next if ($name{$i} eq "<control>");
+	    $total++;
+	}
+	$count = 0;
+	$| = 1;
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    $count++;
+	    $j = sprintf("%04X", $i);
+	    $j =~ /(..)(..)/;
+	    $gif = "http://charts.unicode.org/Unicode.charts/$prefix/$1/U$j.gif";
+	    print("\r$count/$total: $gif");
+	    system("mkdir -p $prefix/$1; cd $prefix/$1; webcopy -u -s $gif &");
+	    select(undef, undef, undef, 0.2);
+	}
+	print("\n");
+	exit 0;
+    } elsif (/^giftable/) {
+	# form a table of glyphs (requires pbmtools installed)
+	$count = 0;
+	for $i (keys(%used)) {
+	    $count++ unless $name{$i} eq "<control>";
+	}
+	$width = int(sqrt($count/sqrt(2)) + 0.5);
+	$width = $1 if /^giftable([0-9]+)$/;
+	system("rm -f tmp-*.pnm table.pnm~ table.pnm");
+	$col = 0;
+	$row = 0;
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    $j = sprintf("%04X", $i);
+	    $j =~ /(..)(..)/;
+	    $gif = "Small.Glyphs/$1/U$j.gif";
+	    $pnm = sprintf("tmp-%02x.pnm", $col);
+	    $fallback = "Small.Glyphs/FF/UFFFD.gif";
+	    system("giftopnm $gif >$pnm || { rm $pnm ; giftopnm $fallback >$pnm ; }");
+	    if (++$col == $width) {
+		system("pnmcat -lr tmp-*.pnm | cat >tmp-row.pnm");
+		if ($row == 0) {
+		    system("mv tmp-row.pnm table.pnm");
+		} else {
+		    system("mv table.pnm table.pnm~; pnmcat -tb table.pnm~ tmp-row.pnm >table.pnm");
+		}
+		$row++;
+		$col = 0;
+		system("rm -f tmp-*.pnm table.pnm~");
+	    }
+	}
+	if ($col > 0) {
+	    system("pnmcat -lr tmp-*.pnm | cat >tmp-row.pnm");
+	    if ($row == 0) {
+		system("mv tmp-row.pnm table.pnm");
+	    } else {
+		system("mv table.pnm table.pnm~; pnmcat -tb -jleft -black table.pnm~ tmp-row.pnm >table.pnm");
+	    }
+	}
+	system("rm -f table.gif ; ppmtogif table.pnm > table.gif");
+	system("rm -f tmp-*.pnm table.pnm~ table.pnm");
+    } elsif (/^table$/) {
+	# go through all used names to print full table
+	print "<TABLE border=2>\n" if $html;
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    if ($html) {
+		$sources = $used{$i};
+		$sources =~ s/\]\[/, /g;
+		$sources =~ s/^\[//g;
+		$sources =~ s/\]$//g;
+		$sources =~ s/\{(..)\}/<SUB>$1<\/SUB>/g;
+		$j = sprintf("%04X", $i);
+		$j =~ /(..)(..)/;
+		$gif = "Small.Glyphs/$1/U$j.gif";
+		print "<TR>";
+		print "<TD><img width=32 height=32 src=\"$gif\">" if $image;
+		printf("<TD>&#%d;", $i) if $adducs;
+		print "<TD><SAMP>$j</SAMP><TD><SAMP>" . name($i);
+		print " ($comment{$i})" if $comment{$i};
+		print "</SAMP><TD><SMALL>$sources</SMALL>\n";
+	    } else {
+		printf("%04X \# ", $i);
+		print pack("U", $i) . " " if $adducs;
+		print name($i) ."\n";
+	    }
+	}
+	print "</TABLE>\n" if $html;
+    } elsif (/^imgblock$/) {
+	$width = 16;
+	$width = $1 if /giftable([0-9]+)/;
+	$col = 0;
+	$subline = "";
+	print "\n<P><TABLE cellspacing=0 cellpadding=0>";
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    print "<TR>" if $col == 0;
+	    $j = sprintf("%04X", $i);
+	    $j =~ /(..)(..)/;
+	    $gif = "Small.Glyphs/$1/U$j.gif";
+	    $alt = name($i);
+	    print "<TD><img width=32 height=32 src=\"$gif\" alt=\"$alt\">";
+	    $subline .= "<TD><SMALL><SAMP>$j</SAMP></SMALL>";
+	    if (++$col == $width) {
+		print "<TR align=center>$subline";
+		$col = 0;
+		$subline = "";
+	    }
+	}
+	print "<TR align=center>$subline" if ($col > 0);
+	print "</TABLE>\n";
+    } elsif (/^sources$/) {
+	# count how many characters are attributed to the various source set combinations
+	print "<P>Number of occurences of source character set combinations:\n<TABLE border=2>" if $html;
+	for $i (keys(%used)) {
+	    next if ($name{$i} eq "<control>");
+	    $sources = $used{$i};
+	    $sources =~ s/\]\[/, /g;
+	    $sources =~ s/^\[//g;
+	    $sources =~ s/\]$//g;
+	    $sources =~ s/\{(..)\}//g;
+	    $contribs{$sources} += 1;
+	}
+	for $j (keys(%contribs)) {
+	    print "<TR><TD>$contribs{$j}<TD>$j\n" if $html;
+	}
+	print "</TABLE>\n" if $html;
+    } elsif (/^compact$/) {
+	# print compact table in P10 MES format
+	print "<P>Compact representation of this character set:\n<TABLE border=2>" if $html;
+	print "<TR><TD><B>Rows</B><TD><B>Positions (Cells)</B>" if $html;
+	print "\n# Plane 00\n# Rows\tPositions (Cells)\n" unless $html;
+	$current_row = '';
+	$start_col = '';
+	$last_col = '';
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    $row = sprintf("%02X", $i >> 8);
+	    $col = sprintf("%02X", $i & 0xff);
+	    if ($row ne $current_row) {
+		if (($last_col ne '') and ($last_col ne $start_col)) {
+		    print "-$last_col";
+		    print "</SAMP>" if $html;
+		}
+		print "<TR><TD><SAMP>$row</SAMP><TD><SAMP>" if $html;
+		print "\n  $row\t" unless $html;
+		$len = 0;
+		$current_row = $row;
+		$start_col = '';
+	    }
+	    if ($start_col eq '') {
+		print "$col";
+		$len += 2;
+		$start_col = $col;
+		$last_col = $col;
+	    } elsif (hex($col) == hex($last_col) + 1) {
+		$last_col = $col;
+	    } else {
+		if ($last_col ne $start_col) {
+		    print "-$last_col";
+		    $len += 3;
+		}
+		if ($len > 60 && !$html) {
+		    print "\n  $row\t";
+		    $len = 0;
+		};
+		print " " if $len;
+		print "$col";
+		$len += 2 + !! $len;
+		$start_col = $col;
+		$last_col = $col;
+	    }
+	}
+	if (($last_col ne '') and ($last_col ne $start_col)) {
+	    print "-$last_col";
+	    print "</SAMP>" if $html;
+	}
+	print "\n" if ($current_row ne '');
+	print "</TABLE>\n" if $html;
+	print "\n";
+    } elsif (/^c$/) {
+	# print table as C interval array
+	print "{";
+	$last_i = '';
+	$columns = 3;
+	$col = $columns;
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    if ($last_i eq '') {
+		if (++$col > $columns) { $col = 1; print "\n "; }
+		printf(" { 0x%04X, ", $i);
+		$last_i = $i;
+	    } elsif ($i == $last_i + 1) {
+		$last_i = $i;
+	    } else {
+		printf("0x%04X },", $last_i);
+		if (++$col > $columns) { $col = 1; print "\n "; }
+		printf(" { 0x%04X, ", $i);
+		$last_i = $i;
+	    }
+	}
+	if ($last_i ne '') {
+	    printf("0x%04X }", $last_i);
+	}
+	print "\n};\n";
+    } elsif (/^utf8-list$/) {
+	$col = 0;
+	$block = 0;
+	$last = -1;
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    while ($blockend[$block] < $i && $block < $blocks - 1) {
+		$block++;
+	    }
+	    if ($last <= $blockend[$block-1] &&
+		$i < $blockstart[$block]) {
+		print "\n" if ($col);
+		printf "\nFree block (U+%04X-U+%04X):\n\n",
+		    $blockend[$block-1] + 1, $blockstart[$block] - 1;
+		$col = 0;
+	    }
+	    if ($last < $blockstart[$block] && $i >= $blockstart[$block]) {
+		print "\n" if ($col);
+		printf "\n$blockname[$block] (U+%04X-U+%04X):\n\n",
+		$blockstart[$block], $blockend[$block];
+		$col = 0;
+	    }
+	    if ($category{$i} eq 'Mn') {
+		# prefix non-spacing character with U+25CC DOTTED CIRCLE
+		print "\x{25CC}";
+	    } elsif ($category{$i} eq 'Me') {
+		# prefix enclosing non-spacing character with space
+		print " ";
+	    }
+	    print pack("U", $i);
+	    $col += 1 + iswide($i);
+	    if ($col >= 64) {
+		print "\n";
+		$col = 0;
+	    }
+	    $last = $i;
+	}
+	print "\n" if ($col);
+    } elsif (/^collections$/) {
+	$block = 0;
+	$last = -1;
+	for $i (sort({$a <=> $b} keys(%used))) {
+	    next if ($name{$i} eq "<control>");
+	    while ($blockend[$block] < $i && $block < $blocks - 1) {
+		$block++;
+	    }
+	    if ($last < $blockstart[$block] && $i >= $blockstart[$block]) {
+		print $blockname[$block],
+		  " " x (40 - length($blockname[$block]));
+		printf "%04X-%04X\n",
+		  $blockstart[$block], $blockend[$block];
+	    }
+	    $last = $i;
+	}
+    } elsif (/^nr$/) {
+	print "<P>" if $html;
+	print "# " unless $html;
+	print "Number of characters in above table: ";
+	$count = 0;
+	for $i (keys(%used)) {
+	    $count++ unless $name{$i} eq "<control>";
+	}
+	print $count;
+	print "\n";
+    } elsif (/^clean$/) {
+	# remove characters from set that are not in $unicodedata
+	for $i (keys(%used)) {
+	    delete $used{$i} unless is_unicode($i);
+	}
+    } elsif (/^unknown$/) {
+	# remove characters from set that are in $unicodedata
+	for $i (keys(%used)) {
+	    delete $used{$i} if is_unicode($i);
+	}
+    } else {
+	die("Unknown command line command '$_'");
+    };
+}
-- 
2.13.2