28510: add (z+c+) and (z+C+) parameter flags

9 files changed, 159 insertions, 17 deletions

diff --git a/ChangeLog b/ChangeLog
index 63cb77eb2..492ee37b4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2010-12-12  Peter Stephenson  <p.w.stephenson@ntlworld.com>
+
+	* 28510: Doc/Zsh/expn.yo, Src/hist.c, Src/lex.c, Src/subst.c,
+	Src/Modules/parameter.c, Src/Zle/zle_hist.c, Src/Zle/zle_misc.c,
+	Test/D04parameter.ztst: add (z+c+) and (z+C+) parameter flags.
+
 2010-12-07  Peter Stephenson  <pws@csr.com>
 
 	* unposted: remove users/15622 which causes problems
@@ -13930,5 +13936,5 @@
 
 *****************************************************
 * This is used by the shell to define $ZSH_PATCHLEVEL
-* $Revision: 1.5146 $
+* $Revision: 1.5147 $
 *****************************************************
diff --git a/Doc/Zsh/expn.yo b/Doc/Zsh/expn.yo
index aadc72dfc..5768b82df 100644
--- a/Doc/Zsh/expn.yo
+++ b/Doc/Zsh/expn.yo
@@ -1006,6 +1006,17 @@ errors are silently ignored.
 item(tt(z))(
 Split the result of the expansion into words using shell parsing to
 find the words, i.e. taking into account any quoting in the value.
+Comments are not treated specially but as ordinary strings, similar
+to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset.
+
+The flag can take option letters between a following pair of
+`tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR()) causes
+comments to be parsed as a string and retained; any field in the
+resulting array beginning with an unquoted comment character is a
+comment.  tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
+and removed.  The rule for comments is standard: anything between a word
+starting with the third charcter of tt($HISTCHARS), default tt(#), up to
+the next newline is a comment.
 
 Note that this is done very late, as for the `tt((s))' flag. So to
 access single words in the result, one has to use nested expansions as 
diff --git a/Src/Modules/parameter.c b/Src/Modules/parameter.c
index 38b462001..793249f32 100644
--- a/Src/Modules/parameter.c
+++ b/Src/Modules/parameter.c
@@ -1044,7 +1044,7 @@ histwgetfn(UNUSED(Param pm))
     int i = addhistnum(curhist, -1, HIST_FOREIGN), iw;
     Histent he = gethistent(i, GETHIST_UPWARD);
 
-    if ((ll = bufferwords(NULL, NULL, NULL)))
+    if ((ll = bufferwords(NULL, NULL, NULL, 0)))
         for (n = firstnode(ll); n; incnode(n))
             pushnode(l, getdata(n));
 
diff --git a/Src/Zle/zle_hist.c b/Src/Zle/zle_hist.c
index 412644827..b5ff05cd1 100644
--- a/Src/Zle/zle_hist.c
+++ b/Src/Zle/zle_hist.c
@@ -677,7 +677,7 @@ insertlastword(char **args)
 	 * a deleted word, because that can only have come
 	 * from a non-empty line.  I think.
 	 */
-	if (!(l = bufferwords(NULL, NULL, NULL))) {
+	if (!(l = bufferwords(NULL, NULL, NULL, 0))) {
 	    unmetafy_line();
 	    return 1;
 	}
diff --git a/Src/Zle/zle_misc.c b/Src/Zle/zle_misc.c
index 0beb43d6d..25404c1bd 100644
--- a/Src/Zle/zle_misc.c
+++ b/Src/Zle/zle_misc.c
@@ -843,7 +843,7 @@ copyprevshellword(UNUSED(char **args))
     if (zmult <= 0)
 	return 1;
 
-    if ((l = bufferwords(NULL, NULL, &i))) {
+    if ((l = bufferwords(NULL, NULL, &i, 0))) {
 	i -= (zmult-1);
 	if (i < 0)
 	    return 1;
diff --git a/Src/hist.c b/Src/hist.c
index e65ddb1b6..89db826b3 100644
--- a/Src/hist.c
+++ b/Src/hist.c
@@ -2345,7 +2345,7 @@ readhistfile(char *fn, int err, int readflags)
 		/*
 		 * Attempt to do this using the lexer.
 		 */
-		LinkList wordlist = bufferwords(NULL, pt, NULL);
+		LinkList wordlist = bufferwords(NULL, pt, NULL, 1);
 		LinkNode wordnode;
 		int nwords_max;
 		nwords_max = 2 * countlinknodes(wordlist);
@@ -2885,11 +2885,27 @@ histfileIsLocked(void)
  * which may not even be valid at this point.
  *
  * However, I'm so confused it could simply be baking Bakewell tarts.
+ *
+ * list may be an existing linked list (off the heap), in which case
+ * it will be appended to; otherwise it will be created.
+ *
+ * If buf is set we will take input from that string, else we will
+ * attempt to use ZLE directly in a way they tell you not to do on all
+ * programming courses.
+ *
+ * If index is non-NULL, and input is from a string in ZLE, *index
+ * is set to the position of the end of the current editor word.
+ *
+ * comments is used if buf is non-NULL (i.e. this is not a string
+ * from ZLE).
+ * If it is 0, comments are not parsed; they are treated as ordinary words.
+ * If it is 1, comments are treated as single strings, one per line.
+ * If it is 2, comments are removed.
  */
 
 /**/
 mod_export LinkList
-bufferwords(LinkList list, char *buf, int *index)
+bufferwords(LinkList list, char *buf, int *index, int comments)
 {
     int num = 0, cur = -1, got = 0, ne = noerrs;
     int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments;
@@ -2906,7 +2922,6 @@ bufferwords(LinkList list, char *buf, int *index)
      * string expression, we just turn the option off for this function.
      */
     opts[RCQUOTES] = 0;
-    zleparse = 1;
     addedx = 0;
     noerrs = 1;
     lexsave();
@@ -2928,11 +2943,18 @@ bufferwords(LinkList list, char *buf, int *index)
 	inpush(p, 0, NULL);
 	zlemetall = strlen(p) ;
 	zlemetacs = zlemetall + 1;
-	nocomments = 1;
+
+	/*
+	 * If comments is non-zero we are handling comments.
+	 * zleparse indicates the mode to the lexer.
+	 */
+	zleparse = 1 + comments;
+	nocomments = !comments;
     } else {
 	int ll, cs;
 	char *linein;
 
+	zleparse = 1;
 	linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs);
 	zlemetall = ll + 1; /* length of line plus space added below */
 	zlemetacs = cs;
diff --git a/Src/lex.c b/Src/lex.c
index fdb4b98ac..44cfa17ca 100644
--- a/Src/lex.c
+++ b/Src/lex.c
@@ -116,7 +116,22 @@ mod_export int wb, we;
 /**/
 mod_export int noaliases;
 
-/* we are parsing a line sent to use by the editor */
+/*
+ * we are parsing a line sent to use by the editor, or some other string
+ * that's not part of standard command input (e.g. eval is part of
+ * normal command input).
+ *
+ * zleparse = 1 is the normal case.
+ * zleparse = 2 is used for word splitting; the difference is we
+ *              preserve comments.
+ * zleparse = 3 is also for word splitting, here handling comments
+ *              but stripping them.
+ *
+ * Note that although it is passed into the lexer as an input, the
+ * lexer can set it to zero after finding the word it's searching for.
+ * This only happens if the line being parsed actually does come from
+ * ZLE.
+ */
 
 /**/
 mod_export int zleparse;
@@ -743,26 +758,50 @@ gettok(void)
 
     /* chars in initial position in word */
 
+    /*
+     * Handle comments.  There are some special cases when this
+     * is not normal command input: zleparse implies we are examining
+     * a line lexically without it being used for normal command input.
+     * If zleparse is 1 we treat comments as normal for interactive
+     * mode.
+     * If zleparse is 2 (which has actually got nothing to do with zle)
+     * we always handle comments and retain them.
+     * If zleparse is 3 we always handle comments and discard them.
+     */
     if (c == hashchar && !nocomments &&
 	(isset(INTERACTIVECOMMENTS) ||
-	 (!zleparse && !expanding &&
+	 ((zleparse != 1) && !expanding &&
 	  (!interact || unset(SHINSTDIN) || strin)))) {
 	/* History is handled here to prevent extra  *
 	 * newlines being inserted into the history. */
 
+	if (zleparse == 2) {
+	    len = 0;
+	    bptr = tokstr = (char *)hcalloc(bsiz = 32);
+	    add(c);
+	}
 	while ((c = ingetc()) != '\n' && !lexstop) {
 	    hwaddc(c);
 	    addtoline(c);
+	    if (zleparse == 2)
+		add(c);
 	}
 
 	if (errflag)
 	    peek = LEXERR;
 	else {
-	    hwend();
-	    hwbegin(0);
-	    hwaddc('\n');
-	    addtoline('\n');
-	    peek = NEWLIN;
+	    if (zleparse == 2) {
+		*bptr = '\0';
+		if (!lexstop)
+		    hungetc(c);
+		peek = STRING;
+	    } else {
+		hwend();
+		hwbegin(0);
+		hwaddc('\n');
+		addtoline('\n');
+		peek = NEWLIN;
+	    }
 	}
 	return peek;
     }
diff --git a/Src/subst.c b/Src/subst.c
index 031a1affd..799682df2 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -1556,6 +1556,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
      * The (z) flag, nothing to do with SH_WORD_SPLIT which is tied
      * spbreak, see above; fairly straighforward in use but c.f.
      * the comment for mods.
+     *
+     * This ultimately becomes zleparse during lexical analysis, via
+     * the comments argument to bufferwords(). It's got nothing
+     * to do with zle.
      */
     int shsplit = 0;
     /*
@@ -1934,6 +1938,27 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 
 		case 'z':
 		    shsplit = 1;
+		    if (s[1] == '+') {
+			s += 2;
+			while (*s && *s != '+' && *s != ')' && *s != Outpar) {
+			    switch (*s++) {
+			    case 'c':
+				/* Parse and keep comments */
+				shsplit = 2;
+				break;
+
+			    case 'C':
+				/* Parse and remove comments */
+				shsplit = 3;
+				break;
+
+			    default:
+				goto flagerr;
+			    }
+			}
+			if (*s != '+')
+			    goto flagerr;
+		    }
 		    break;
 
 		case 'u':
@@ -3207,10 +3232,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 	if (isarr) {
 	    char **ap;
 	    for (ap = aval; *ap; ap++)
-		list = bufferwords(list, *ap, NULL);
+		list = bufferwords(list, *ap, NULL, shsplit-1);
 	    isarr = 0;
 	} else
-	    list = bufferwords(NULL, val, NULL);
+	    list = bufferwords(NULL, val, NULL, shsplit-1);
 
 	if (!list || !firstnode(list))
 	    val = dupstring("");
diff --git a/Test/D04parameter.ztst b/Test/D04parameter.ztst
index 4bd911e5f..e2772afe5 100644
--- a/Test/D04parameter.ztst
+++ b/Test/D04parameter.ztst
@@ -417,6 +417,45 @@
 >5:i++ :
 >6:)):
 
+  line=$'A line with # someone\'s comment\nanother line # (1 more\nanother one'
+  print "*** Normal ***"
+  print -l ${(z)line}
+  print "*** Kept ***"
+  print -l ${(z+c+)line}
+  print "*** Removed ***"
+  print -l ${(z+C+)line}
+0:Comments with (z)
+>*** Normal ***
+>A
+>line
+>with
+>#
+>someone's comment
+>another line # (1 more
+>another one
+>*** Kept ***
+>A
+>line
+>with
+># someone's comment
+>;
+>another
+>line
+># (1 more
+>;
+>another
+>one
+>*** Removed ***
+>A
+>line
+>with
+>;
+>another
+>line
+>;
+>another
+>one
+
   psvar=(dog)
   setopt promptsubst
   foo='It shouldn'\''t $(happen) to a %1v.'