Add coverage checks

This commit adds a makefile target for generating a report for unit test
coverage. It also adds the coverage_check tool to assert that the actual
test coverage meets the specified requirement.

This also improves various aspects of the testing utilities:
* The "test" package has been renamed to "testutil"
* The "Objs" member has been removed from the CmdTest object
* The "Cmd" member has been added to the CmdTest object. This allows
  testing of multiple variants of a root airshipctl command

Finally, this commit includes additional tests for root.go. These are
required in order to meet the required coverage threshold.

Change-Id: Id48343166c0488c543a405ec3143e4a75355ba43
This commit is contained in:
Ian Howell 2019-07-11 08:29:15 -05:00
parent 7c8ee26de4
commit 1c999e2095
9 changed files with 140 additions and 27 deletions

View File

@ -22,6 +22,7 @@ DOCKER_IMAGE ?= $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/$(DOCKER_IMAGE_
# go options # go options
PKG := ./... PKG := ./...
TESTS := . TESTS := .
COVER_PROFILE := cover.out
.PHONY: get-modules .PHONY: get-modules
get-modules: get-modules:
@ -35,13 +36,18 @@ build: get-modules
test: lint test: lint
test: TESTFLAGS += -race -v test: TESTFLAGS += -race -v
test: unit-tests test: unit-tests
test: cover
.PHONY: unit-tests .PHONY: unit-tests
unit-tests: build unit-tests: build
@echo "Performing unit test step..." @echo "Performing unit test step..."
@GO111MODULE=on go test -run $(TESTS) $(PKG) $(TESTFLAGS) @GO111MODULE=on go test -run $(TESTS) $(PKG) $(TESTFLAGS) -covermode=atomic -coverprofile=$(COVER_PROFILE)
@echo "All unit tests passed" @echo "All unit tests passed"
.PHONY: cover
cover: unit-tests
@./tools/coverage_check $(COVER_PROFILE)
.PHONY: lint .PHONY: lint
lint: lint:
@echo "Performing linting step..." @echo "Performing linting step..."
@ -57,7 +63,7 @@ print-docker-image-tag:
@echo "$(DOCKER_IMAGE)" @echo "$(DOCKER_IMAGE)"
.PHONY: docker-image-unit-tests .PHONY: docker-image-unit-tests
docker-image-unit-tests: DOCKER_MAKE_TARGET = unit-tests docker-image-unit-tests: DOCKER_MAKE_TARGET = cover
docker-image-unit-tests: docker-image docker-image-unit-tests: docker-image
.PHONY: docker-image-lint .PHONY: docker-image-lint
@ -67,6 +73,7 @@ docker-image-lint: docker-image
.PHONY: clean .PHONY: clean
clean: clean:
@rm -fr $(BINDIR) @rm -fr $(BINDIR)
@rm -fr $(COVER_PROFILE)
.PHONY: docs .PHONY: docs
docs: docs:

View File

@ -3,22 +3,56 @@ package cmd_test
import ( import (
"testing" "testing"
"github.com/spf13/cobra"
"opendev.org/airship/airshipctl/cmd" "opendev.org/airship/airshipctl/cmd"
"opendev.org/airship/airshipctl/test" "opendev.org/airship/airshipctl/cmd/bootstrap"
"opendev.org/airship/airshipctl/pkg/environment"
"opendev.org/airship/airshipctl/testutil"
) )
func TestRoot(t *testing.T) { func TestRoot(t *testing.T) {
cmdTests := []*test.CmdTest{ tests := []*testutil.CmdTest{
{ {
Name: "default", Name: "rootCmd-with-no-defaults",
CmdLine: "", CmdLine: "",
Cmd: getVanillaRootCmd(t),
},
{
Name: "rootCmd-with-defaults",
CmdLine: "",
Cmd: getDefaultRootCmd(t),
},
{
Name: "specialized-rootCmd-with-bootstrap",
CmdLine: "",
Cmd: getSpecializedRootCmd(t),
}, },
} }
for _, tt := range tests {
testutil.RunTest(t, tt)
}
}
func getVanillaRootCmd(t *testing.T) *cobra.Command {
rootCmd, _, err := cmd.NewRootCmd(nil) rootCmd, _, err := cmd.NewRootCmd(nil)
if err != nil { if err != nil {
t.Fatalf("Could not create root command: %s", err.Error()) t.Fatalf("Could not create root command: %s", err.Error())
} }
for _, tt := range cmdTests { return rootCmd
test.RunTest(t, tt, rootCmd)
} }
func getDefaultRootCmd(t *testing.T) *cobra.Command {
rootCmd, _, err := cmd.NewAirshipCTLCommand(nil)
if err != nil {
t.Fatalf("Could not create root command: %s", err.Error())
}
return rootCmd
}
func getSpecializedRootCmd(t *testing.T) *cobra.Command {
rootCmd := getVanillaRootCmd(t)
rootCmd.AddCommand(bootstrap.NewBootstrapCommand(&environment.AirshipCTLSettings{}))
return rootCmd
} }

View File

@ -0,0 +1,20 @@
airshipctl is a unified entrypoint to various airship components
Usage:
airshipctl [command]
Available Commands:
argo argo is the command line interface to Argo
bootstrap bootstraps airshipctl
completion Generate autocompletions script for the specified shell (bash or zsh)
help Help about any command
kubeadm kubeadm: easily bootstrap a secure Kubernetes cluster
kubectl kubectl controls the Kubernetes cluster manager
kustomize Build a kustomization target from a directory or a remote url.
version Show the version number of airshipctl
Flags:
--debug enable verbose output
-h, --help help for airshipctl
Use "airshipctl [command] --help" for more information about a command.

View File

@ -0,0 +1,15 @@
airshipctl is a unified entrypoint to various airship components
Usage:
airshipctl [command]
Available Commands:
bootstrap bootstraps airshipctl
help Help about any command
version Show the version number of airshipctl
Flags:
--debug enable verbose output
-h, --help help for airshipctl
Use "airshipctl [command] --help" for more information about a command.

View File

@ -4,22 +4,25 @@ import (
"testing" "testing"
"opendev.org/airship/airshipctl/cmd" "opendev.org/airship/airshipctl/cmd"
"opendev.org/airship/airshipctl/test" "opendev.org/airship/airshipctl/testutil"
) )
func TestVersion(t *testing.T) { func TestVersion(t *testing.T) {
cmdTests := []*test.CmdTest{
{
Name: "version",
CmdLine: "version",
},
}
rootCmd, _, err := cmd.NewRootCmd(nil) rootCmd, _, err := cmd.NewRootCmd(nil)
if err != nil { if err != nil {
t.Fatalf("Could not create root command: %s", err.Error()) t.Fatalf("Could not create root command: %s", err.Error())
} }
rootCmd.AddCommand(cmd.NewVersionCommand()) rootCmd.AddCommand(cmd.NewVersionCommand())
cmdTests := []*testutil.CmdTest{
{
Name: "version",
CmdLine: "version",
Cmd: rootCmd,
},
}
for _, tt := range cmdTests { for _, tt := range cmdTests {
test.RunTest(t, tt, rootCmd) testutil.RunTest(t, tt)
} }
} }

2
go.mod
View File

@ -95,7 +95,7 @@ require (
gotest.tools v2.2.0+incompatible // indirect gotest.tools v2.2.0+incompatible // indirect
k8s.io/api v0.0.0-20190516230258-a675ac48af67 // indirect k8s.io/api v0.0.0-20190516230258-a675ac48af67 // indirect
k8s.io/apiextensions-apiserver v0.0.0-20190516231611-bf6753f2aa24 // indirect k8s.io/apiextensions-apiserver v0.0.0-20190516231611-bf6753f2aa24 // indirect
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d // indirect
k8s.io/apiserver v0.0.0-20190516230822-f89599b3f645 // indirect k8s.io/apiserver v0.0.0-20190516230822-f89599b3f645 // indirect
k8s.io/cli-runtime v0.0.0-20190516231937-17bc0b7fcef5 // indirect k8s.io/cli-runtime v0.0.0-20190516231937-17bc0b7fcef5 // indirect
k8s.io/client-go v11.0.1-0.20190516230509-ae8359b20417+incompatible k8s.io/client-go v11.0.1-0.20190516230509-ae8359b20417+incompatible

View File

@ -1,4 +1,4 @@
package test package testutil
import ( import (
"bytes" "bytes"
@ -10,7 +10,6 @@ import (
"testing" "testing"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime"
) )
// UpdateGolden writes out the golden files with the latest values, rather than failing the test. // UpdateGolden writes out the golden files with the latest values, rather than failing the test.
@ -24,22 +23,34 @@ const (
// CmdTest is a command to be run on the command line as a test // CmdTest is a command to be run on the command line as a test
type CmdTest struct { type CmdTest struct {
// The name of the test. This will be used when generating golden
// files
Name string Name string
// The values that would be inputted to airshipctl as commands, flags,
// and arguments. The initial "airshipctl" is implied
CmdLine string CmdLine string
Objs []runtime.Object
// The instatiated version of the root airshipctl command to test
Cmd *cobra.Command
} }
// RunTest either asserts that a specific command's output matches the expected // RunTest either asserts that a specific command's output matches the expected
// output from its golden file, or generates golden files if the -update flag // output from its golden file, or generates golden files if the -update flag
// is passed // is passed
func RunTest(t *testing.T, test *CmdTest, cmd *cobra.Command) { func RunTest(t *testing.T, test *CmdTest) {
cmd := test.Cmd
actual := &bytes.Buffer{} actual := &bytes.Buffer{}
cmd.SetOutput(actual) cmd.SetOutput(actual)
args := strings.Fields(test.CmdLine) args := strings.Fields(test.CmdLine)
cmd.SetArgs(args) cmd.SetArgs(args)
if err := cmd.Execute(); err != nil { if err := cmd.Execute(); err != nil {
t.Fatalf("Unexpected error: %s", err.Error()) t.Fatalf("Unexpected error: %s", err.Error())
} }
if *shouldUpdateGolden { if *shouldUpdateGolden {
updateGolden(t, test, actual.Bytes()) updateGolden(t, test, actual.Bytes())
} else { } else {
@ -50,13 +61,13 @@ func RunTest(t *testing.T, test *CmdTest, cmd *cobra.Command) {
func updateGolden(t *testing.T, test *CmdTest, actual []byte) { func updateGolden(t *testing.T, test *CmdTest, actual []byte) {
goldenDir := filepath.Join(testdataDir, t.Name()+goldenDirSuffix) goldenDir := filepath.Join(testdataDir, t.Name()+goldenDirSuffix)
if err := os.MkdirAll(goldenDir, 0775); err != nil { if err := os.MkdirAll(goldenDir, 0775); err != nil {
t.Fatalf("failed to create golden directory %s: %s", goldenDir, err) t.Fatalf("Failed to create golden directory %s: %s", goldenDir, err)
} }
t.Logf("Created %s", goldenDir) t.Logf("Created %s", goldenDir)
goldenFilePath := filepath.Join(goldenDir, test.Name+goldenFileSuffix) goldenFilePath := filepath.Join(goldenDir, test.Name+goldenFileSuffix)
t.Logf("updating golden file: %s", goldenFilePath) t.Logf("Updating golden file: %s", goldenFilePath)
if err := ioutil.WriteFile(goldenFilePath, normalize(actual), 0666); err != nil { if err := ioutil.WriteFile(goldenFilePath, normalize(actual), 0666); err != nil {
t.Fatalf("failed to update golden file: %s", err) t.Fatalf("Failed to update golden file: %s", err)
} }
} }
@ -65,10 +76,10 @@ func assertEqualGolden(t *testing.T, test *CmdTest, actual []byte) {
goldenFilePath := filepath.Join(goldenDir, test.Name+goldenFileSuffix) goldenFilePath := filepath.Join(goldenDir, test.Name+goldenFileSuffix)
golden, err := ioutil.ReadFile(goldenFilePath) golden, err := ioutil.ReadFile(goldenFilePath)
if err != nil { if err != nil {
t.Fatalf("failed while reading golden file: %s", err) t.Fatalf("Failed while reading golden file: %s", err)
} }
if !bytes.Equal(actual, golden) { if !bytes.Equal(actual, golden) {
errFmt := "output does not match golden file: %s\nEXPECTED:\n%s\nGOT:\n%s" errFmt := "Output does not match golden file: %s\nEXPECTED:\n%s\nGOT:\n%s"
t.Errorf(errFmt, goldenFilePath, string(golden), string(actual)) t.Errorf(errFmt, goldenFilePath, string(golden), string(actual))
} }
} }

23
tools/coverage_check Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -ex
if [[ $# -ne 1 ]]; then
printf "Usage: %s <coverfile>\n" "$0"
exit 0
fi
cover_file=$1
min_coverage=80
coverage_report=$(go tool cover -func="$cover_file")
printf "%s\n" "$coverage_report"
coverage_float=$(awk "/^total:/ { print \$3 }" <<< "$coverage_report")
coverage_int=${coverage_float%.*}
if (( "$coverage_int" < "$min_coverage" )) ; then
printf "FAIL: Test coverage is at %s, which does not meet the required coverage (%s%%)\n" "$coverage_float" "$min_coverage"
exit 1
else
printf "SUCCESS: Test coverage is at %s, which meets the required coverage (%s%%)\n" "$coverage_float" "$min_coverage"
fi