22885: fix problems with multibyte tokenized strings

unposted: _todo.sh: more contexts

3 files changed, 47 insertions, 8 deletions

diff --git a/ChangeLog b/ChangeLog
index 0512b2d3b..4ca317af6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2006-10-19  Peter Stephenson  <pws@csr.com>
+
+	* unposted: Completion/Unix/Command/_todo.sh: more places
+	where we complete priorities and contexts.
+
+	* quoted in 22885: Src/utils.c: 22544 introduced problems
+	with multibyte tokenized strings.
+
 2006-10-17  Peter Stephenson  <p.w.stephenson@ntlworld.com>
 
 	* unposted, c.f. 22833: rename Completion/Mandrake and
diff --git a/Completion/Unix/Command/_todo.sh b/Completion/Unix/Command/_todo.sh
index fc984f7d4..2b6b444ee 100644
--- a/Completion/Unix/Command/_todo.sh
+++ b/Completion/Unix/Command/_todo.sh
@@ -3,9 +3,12 @@
 # See http://todotxt.com for todo.sh.
 #
 # Featurettes:
-#  - "replace" will complete the original text for editing.
+#  - "replace" will complete the original text for editing
 #  - completing priorities will cycle through A to Z (even without
-#    menu completion).
+#    menu completion)
+#  - list and listall will complete p:<project> and @<where> from
+#    values in existing entries
+#  - will complete after p: and @ if typed in message text
 
 setopt localoptions braceccl
 
@@ -57,7 +60,11 @@ case $state in
 	nextstate=pri
 	;;
 	(append|prepend)
-	_message $txtmsg
+	if [[ -prefix p: || -prefix @ ]]; then
+	  nextstate=proj
+	else
+	  _message $txtmsg
+	fi
 	;;
 	(replace)
 	compadd -Q -- "${(qq)$(todo.sh list "^0*${words[CURRENT-1]} ")##<-> }"
@@ -67,14 +74,15 @@ case $state in
     ;;
 
     (add)
-    _message $txtmsg
+    if [[ -prefix p: || -prefix @ ]]; then
+      nextstate=proj
+    else
+      _message $txtmsg
+    fi
     ;;
 
     (list|listall)
-    # This completes stuff beginning with p: (projects) or @ (contexts);
-    # these are todo.sh conventions.
-    _wanted search expl 'context or project' \
-      compadd ${${=${${(M)${(f)"$(todo.sh list)"}##<-> *}##<-> }}:#^(p:*|@*)}
+    nextstate=proj
     ;;
 
     (listpri)
@@ -103,4 +111,12 @@ case $nextstate in
     _wanted priority expl 'priority' compadd {A-Z}
   fi
   ;;
+
+  (proj)
+  # This completes stuff beginning with p: (projects) or @ (contexts);
+  # these are todo.sh conventions.
+  _wanted search expl 'context or project' \
+    compadd ${${=${${(M)${(f)"$(todo.sh list)"}##<-> *}##<-> }}:#^(p:*|@*)}
+  ;;
 esac
+
diff --git a/Src/utils.c b/Src/utils.c
index 2e124443f..3658e615f 100644
--- a/Src/utils.c
+++ b/Src/utils.c
@@ -4003,6 +4003,21 @@ mb_metacharlenconv(const char *s, wint_t *wcp)
 	    *wcp = (wint_t)(*s == Meta ? s[1] ^ 32 : *s);
 	return 1 + (*s == Meta);
     }
+    /*
+     * We have to handle tokens here, since we may be looking
+     * through a tokenized input.  Obviously this isn't
+     * a valid multibyte character, so just return WEOF
+     * and let the caller handle it as a single character.
+     *
+     * TODO: I've a sneaking suspicion we could do more here
+     * to prevent the caller always needing to handle invalid
+     * characters specially, but sometimes it may need to know.
+     */
+    if (itok(*s)) {
+	if (wcp)
+	    *wcp = EOF;
+	return 1;
+    }
 
     ret = MB_INVALID;
     for (ptr = s; *ptr; ) {