Skip to content

Commit 89c3b15

Browse files
mblandmbland
committed
test, kcov: Run coverage only by default on Travis
Now, instead of having Travis run the test, then run them again, and ignore the return status of `kcov`, this change should allow Travis to just perform the coverage run and rely on the `kcov` exit status. As explained in the comment for the `test: run coverage by default on Travis Linux` test case: This test also makes sure the invocation doesn't cause a second recursive call to `run_kcov` thanks to the `_COVERAGE_RUN` variable. Previously, seemingly successful coverage runs (added in commit 4440832) were causing Travis failures, ameliorated in commit cc284d1. These were due to the `run_kcov` getting called recursively and failing because the first call already created the `tests/coverage` directory. Here was the chain of events: - Travis calls `./go test`. - Test suite runs and succeeds. - `"$?" -eq '0' && "$TRAVIS_OS_NAME" == 'linux'` condition met. - `_test_coverage` and `run_kcov` executed. - `run_kcov` creates `tests/coverage` and executes `kcov ./go test`. - Test suite runs and succeeds. - `"$?" -eq '0' && "$TRAVIS_OS_NAME" == 'linux'` condition met. - `_test_coverage` and `run_kcov` executed. - `run_kcov` fails because `tests/coverage` already exists. - `kcov` sends coverage info to Coveralls, but exits with an error. - Travis build reports failure. With the `_COVERAGE_RUN` variable, the recursive call is now short-circuited.
1 parent 5cbdfb3 commit 89c3b15

File tree

3 files changed

+108
-12
lines changed

3 files changed

+108
-12
lines changed

‎scripts/lib/kcov

+2-4
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,15 @@ run_kcov() {
4646
mkdir "$coverage_dir"
4747
printf 'Starting coverage run:\n %s\n' "${kcov_argv[*]}"
4848

49-
# Somehow kcov exits with a failure status on Travis, even though the tests
50-
# all pass and information is successfully posted to Coveralls.
51-
if "${kcov_argv[@]}" 2>/dev/null || [[ -n "$TRAVIS_JOB_ID" ]]; then
49+
if "${kcov_argv[@]}" 2>/dev/null; then
5250
if [[ "$send_to_coveralls" == 'false' ]]; then
5351
@go.printf 'Coverage results located in:\n %s\n' \
5452
"$_GO_ROOTDIR/$coverage_dir"
5553
else
5654
@go.printf 'Coverage results sent to:\n %s\n' "$coveralls_url"
5755
fi
5856
else
59-
@go.printf 'kcov exited with errors'
57+
@go.printf 'kcov exited with errors\n'
6058
return 1
6159
fi
6260
}

‎scripts/test

+6-8
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,9 @@ _test() {
5858
git submodule update --init tests/bats
5959
fi
6060

61-
if [[ "$1" == '--coverage' ]]; then
61+
if [[ "$1" == '--coverage' && "$_COVERAGE_RUN" != 'true' ]]; then
6262
shift
63+
local -x _COVERAGE_RUN='true'
6364
_test_coverage "$@"
6465
elif [[ "$1" == '--list' ]]; then
6566
shift
@@ -68,16 +69,13 @@ _test() {
6869
shift
6970
local tests=($(@go 'glob' "${__GO_TEST_GLOB_ARGS[@]}" "$@"))
7071
@go 'edit' "${tests[@]}"
72+
elif [[ "$TRAVIS_OS_NAME" == 'linux' ]]; then
73+
# Collect coverage by default on Travis. Doesn't seem to slow anything down
74+
# substantially.
75+
_test '--coverage' "$@"
7176
else
7277
local tests=($(@go 'glob' "${__GO_TEST_GLOB_ARGS[@]}" "$@"))
7378
"$BASH" "$bats_path" "${tests[@]}"
74-
local result="$?"
75-
76-
if [[ "$result" -eq 0 && "$TRAVIS_OS_NAME" == 'linux' ]]; then
77-
_test_coverage "$@"
78-
else
79-
return "$result"
80-
fi
8179
fi
8280
}
8381

‎tests/test.bats

+100
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@
44

55
load environment
66
load assertions
7+
load script_helper
8+
9+
teardown() {
10+
remove_test_go_rootdir
11+
}
712

813
@test "$SUITE: tab complete flags" {
914
run ./go test --complete 0 '-'
@@ -64,7 +69,102 @@ _trim_expected() {
6469
assert_success "${expected[*]}"
6570
}
6671

72+
@test "$SUITE: open EDITOR on --edit" {
73+
run env EDITOR='echo' ./go test --edit test aliases 'builtins*'
74+
local expected=(
75+
'tests/test.bats' 'tests/aliases.bats' 'tests/builtins.bats'
76+
tests/builtins/*)
77+
assert_success "${expected[*]}"
78+
}
79+
6780
@test "$SUITE: produce an error if any test pattern fails to match" {
6881
run ./go test --list test 'foo*'
6982
assert_failure '"foo*" does not match any .bats files in tests.'
7083
}
84+
85+
@test "$SUITE: update bats submodule if not present" {
86+
mkdir -p "$TEST_GO_SCRIPTS_DIR/bin"
87+
88+
local git_dummy="$TEST_GO_SCRIPTS_DIR/bin/git"
89+
printf '#! /usr/bin/env bash\necho "GIT ARGV: $*"\n' >"$git_dummy"
90+
chmod 700 "$git_dummy"
91+
92+
cp "$_GO_ROOTDIR/scripts/test" "$TEST_GO_SCRIPTS_DIR"
93+
create_test_go_script "PATH=\"$TEST_GO_SCRIPTS_DIR/bin:\$PATH\"; @go \"\$@\""
94+
95+
# This will fail because we didn't create the tests/ directory, but git should
96+
# have been called correctly.
97+
run "$TEST_GO_SCRIPT" test --list test
98+
local expected_output=(
99+
'GIT ARGV: submodule update --init tests/bats'
100+
'Root directory argument tests is not a directory.')
101+
local IFS=$'\n'
102+
assert_failure "${expected_output[*]}"
103+
}
104+
105+
write_bats_dummy_stub_kcov_lib_and_copy_test_script() {
106+
# Avoid `git submodule update` by writing dummy bats.
107+
local bats_dummy="$TEST_GO_ROOTDIR/tests/bats/libexec/bats"
108+
109+
mkdir -p "${bats_dummy%/*}" "$TEST_GO_SCRIPTS_DIR/lib"
110+
echo '#! /usr/bin/env bats' >>"$bats_dummy"
111+
chmod 700 "$bats_dummy"
112+
113+
# Stub the kcov lib to assert it's called correctly.
114+
create_test_command_script 'lib/kcov' \
115+
"run_kcov() { IFS=\$'\n'; echo \"\$*\"; }"
116+
117+
cp "$_GO_ROOTDIR/scripts/test" "$TEST_GO_SCRIPTS_DIR"
118+
}
119+
120+
@test "$SUITE: coverage run" {
121+
write_bats_dummy_stub_kcov_lib_and_copy_test_script
122+
create_test_go_script '@go "$@"'
123+
124+
local test_cmd_argv=("$TEST_GO_SCRIPT" 'test' '--coverage' 'foo' 'bar/baz')
125+
local expected_kcov_args=(
126+
'tests/kcov'
127+
'tests/coverage'
128+
'go,go-core.bash,lib/,libexec/,scripts/'
129+
'/tmp,tests/bats/'
130+
'https://coveralls.io/github/mbland/go-script-bash'
131+
'foo'
132+
'bar/baz')
133+
134+
run env _COVERAGE_RUN= TRAVIS_OS_NAME= "${test_cmd_argv[@]}"
135+
local IFS=$'\n'
136+
assert_success "${expected_kcov_args[*]}"
137+
}
138+
139+
# This test also makes sure the invocation doesn't cause a second recursive call
140+
# to `run_kcov` thanks to the `_COVERAGE_RUN` variable. Previously, seemingly
141+
# successful coverage runs (added in commit
142+
# 4440832c257c3fa455d7d773ee56fd66c4431a19) were causing Travis failures,
143+
# ameliorated in commit cc284d11e010442392029afdcddc5b1c761ad9a0. These were
144+
# due to the `run_kcov` getting called recursively and failing because the first
145+
# call already created the `tests/coverage` directory.
146+
#
147+
# Here was the chain of events:
148+
#
149+
# - Travis calls `./go test`.
150+
# - Test suite runs and succeeds.
151+
# - `"$?" -eq '0' && "$TRAVIS_OS_NAME" == 'linux'` condition met.
152+
# - `_test_coverage` and `run_kcov` executed.
153+
# - `run_kcov` creates `tests/coverage` and executes `kcov ./go test`.
154+
# - Test suite runs and succeeds.
155+
# - `"$?" -eq '0' && "$TRAVIS_OS_NAME" == 'linux'` condition met.
156+
# - `_test_coverage` and `run_kcov` executed.
157+
# - `run_kcov` fails because `tests/coverage` already exists.
158+
# - `kcov` sends coverage info to Coveralls, but exits with an error.
159+
# - Travis build reports failure.
160+
#
161+
# With the `_COVERAGE_RUN` variable, the recursive call is now
162+
# short-circuited.
163+
@test "$SUITE: run coverage by default on Travis Linux" {
164+
write_bats_dummy_stub_kcov_lib_and_copy_test_script
165+
create_test_go_script '@go "$@"'
166+
167+
run env _COVERAGE_RUN= TRAVIS_OS_NAME='linux' "$TEST_GO_SCRIPT" test
168+
assert_success
169+
assert_line_equals 0 'tests/kcov'
170+
}

0 commit comments

Comments
 (0)