diff options
| -rw-r--r-- | ChangeLog | 6 | ||||
| -rw-r--r-- | Test/B01cd.ztst | 4 | ||||
| -rwxr-xr-x | Test/ztst.zsh | 30 | ||||
| -rw-r--r-- | Util/ztst-syntax.vim | 2 |
4 files changed, 38 insertions, 4 deletions
@@ -1,3 +1,9 @@ +2020-01-01 Daniel Shahaf <danielsh@apache.org> + + * 45169/0001: Test/B01cd.ztst, Test/ztst.zsh, + Util/ztst-syntax.vim: In the test suite, allow test cases to + be marked as expected to fail. + 2019-12-31 Daniel Shahaf <danielsh@apache.org> * users/24582 + users/24583: Test/D04parameter.ztst, diff --git a/Test/B01cd.ztst b/Test/B01cd.ztst index f79e18f4b..977cbdfe5 100644 --- a/Test/B01cd.ztst +++ b/Test/B01cd.ztst @@ -70,11 +70,13 @@ # the expected status returned by the code when run, or - if it is # irrelevant. An optional set of single-letter flags follows the status # or -. The following are understood: -# . d Don't diff stdout against the expected stdout. +# . d Don't diff stdout against the expected stdout. # D Don't diff stderr against the expected stderr. # q All redirection lines given in the test script (not the lines # actually produced by the test) are subject to ordinary quoted shell # expansion (i.e. not globbing). +# f Test is expected to fail. If the test's exit code, stdout, and stderr +# match, report a problem; otherwise, carry on to the next test case. # This can be followed by a `:' and a message describing the # test, which will be printed if the test fails, along with a # description of the failure that occurred. The `:' and message are diff --git a/Test/ztst.zsh b/Test/ztst.zsh index a4c625230..375efd16c 100755 --- a/Test/ztst.zsh +++ b/Test/ztst.zsh @@ -366,6 +366,7 @@ ZTST_test() { local last match mbegin mend found substlines local diff_out diff_err local ZTST_skip + integer expected_to_fail while true; do rm -f $ZTST_in $ZTST_out $ZTST_err @@ -460,8 +461,21 @@ $ZTST_curline" fi fi + if [[ $ZTST_flags = *f* ]]; then + expected_to_fail=1 + ZTST_xfail_diff() { ZTST_diff "$@" > /dev/null } + ZTST_diff=ZTST_xfail_diff + else + expected_to_fail=0 + ZTST_diff=ZTST_diff + fi + # First check we got the right status, if specified. if [[ $ZTST_xstatus != - && $ZTST_xstatus != $ZTST_status ]]; then + if (( expected_to_fail )); then + ZTST_verbose 1 "Test failed, as expected." + continue + fi ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from: $ZTST_code${$(<$ZTST_terr):+ Error output: @@ -480,7 +494,11 @@ $(<$ZTST_terr)" rm -rf $ZTST_out print -r -- "${(e)substlines}" >$ZTST_out fi - if [[ $ZTST_flags != *d* ]] && ! ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then + if [[ $ZTST_flags != *d* ]] && ! $ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then + if (( expected_to_fail )); then + ZTST_verbose 1 "Test failed, as expected." + continue + fi ZTST_testfailed "output differs from expected as shown above for: $ZTST_code${$(<$ZTST_terr):+ Error output: @@ -492,11 +510,19 @@ $(<$ZTST_terr)}" rm -rf $ZTST_err print -r -- "${(e)substlines}" >$ZTST_err fi - if [[ $ZTST_flags != *D* ]] && ! ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then + if [[ $ZTST_flags != *D* ]] && ! $ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then + if (( expected_to_fail )); then + ZTST_verbose 1 "Test failed, as expected." + continue + fi ZTST_testfailed "error output differs from expected as shown above for: $ZTST_code" return 1 fi + if (( expected_to_fail )); then + ZTST_testfailed "test was expected to fail, but passed." + return 1 + fi fi ZTST_verbose 1 "Test successful." [[ -n $last ]] && break diff --git a/Util/ztst-syntax.vim b/Util/ztst-syntax.vim index 8d8becfe7..f0c82d74c 100644 --- a/Util/ztst-syntax.vim +++ b/Util/ztst-syntax.vim @@ -28,7 +28,7 @@ syn include @zsh syntax/zsh.vim syn match ztstPayload /^\s\+\zs.*/ contains=@zsh syn match ztstExitCode /^\d\+\|^-/ nextgroup=ztstFlags -syn match ztstFlags /[.dDq]*:/ contained nextgroup=ztstTestName contains=ztstColon +syn match ztstFlags /[.dDqf]*:/ contained nextgroup=ztstTestName contains=ztstColon syn match ztstColon /:/ contained syn region ztstTestName start=// end=/$/ contained |