Add coverage checks

This commit adds a makefile target for generating a report for unit test
coverage. It also adds the coverage_check tool to assert that the actual
test coverage meets the specified requirement.

This also improves various aspects of the testing utilities:
* The "test" package has been renamed to "testutil"
* The "Objs" member has been removed from the CmdTest object
* The "Cmd" member has been added to the CmdTest object. This allows
  testing of multiple variants of a root airshipctl command

Finally, this commit includes additional tests for root.go. These are
required in order to meet the required coverage threshold.

Change-Id: Id48343166c0488c543a405ec3143e4a75355ba43

Makefile

@@ -22,6 +22,7 @@ DOCKER_IMAGE        ?= $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/$(DOCKER_IMAGE_
 # go options
 PKG                 := ./...
 TESTS               := .
+COVER_PROFILE       := cover.out
 
 .PHONY: get-modules
 get-modules:
@@ -35,13 +36,18 @@ build: get-modules
 test: lint
 test: TESTFLAGS += -race -v
 test: unit-tests
+test: cover
 
 .PHONY: unit-tests
 unit-tests: build
 	@echo "Performing unit test step..."
-	@GO111MODULE=on go test -run $(TESTS) $(PKG) $(TESTFLAGS)
+	@GO111MODULE=on go test -run $(TESTS) $(PKG) $(TESTFLAGS) -covermode=atomic -coverprofile=$(COVER_PROFILE)
 	@echo "All unit tests passed"
 
+.PHONY: cover
+cover: unit-tests
+	@./tools/coverage_check $(COVER_PROFILE)
+
 .PHONY: lint
 lint:
 	@echo "Performing linting step..."
@@ -57,7 +63,7 @@ print-docker-image-tag:
 	@echo "$(DOCKER_IMAGE)"
 
 .PHONY: docker-image-unit-tests
-docker-image-unit-tests: DOCKER_MAKE_TARGET = unit-tests
+docker-image-unit-tests: DOCKER_MAKE_TARGET = cover
 docker-image-unit-tests: docker-image
 
 .PHONY: docker-image-lint
@@ -67,6 +73,7 @@ docker-image-lint: docker-image
 .PHONY: clean
 clean:
 	@rm -fr $(BINDIR)
+	@rm -fr $(COVER_PROFILE)
 
 .PHONY: docs
 docs:

cmd/root_test.go

@@ -3,22 +3,56 @@ package cmd_test
 import (
 	"testing"
 
+	"github.com/spf13/cobra"
+
 	"opendev.org/airship/airshipctl/cmd"
-	"opendev.org/airship/airshipctl/test"
+	"opendev.org/airship/airshipctl/cmd/bootstrap"
+	"opendev.org/airship/airshipctl/pkg/environment"
+	"opendev.org/airship/airshipctl/testutil"
 )
 
 func TestRoot(t *testing.T) {
-	cmdTests := []*test.CmdTest{
+	tests := []*testutil.CmdTest{
 		{
-			Name:    "default",
+			Name:    "rootCmd-with-no-defaults",
 			CmdLine: "",
+			Cmd:     getVanillaRootCmd(t),
+		},
+		{
+			Name:    "rootCmd-with-defaults",
+			CmdLine: "",
+			Cmd:     getDefaultRootCmd(t),
+		},
+		{
+			Name:    "specialized-rootCmd-with-bootstrap",
+			CmdLine: "",
+			Cmd:     getSpecializedRootCmd(t),
 		},
 	}
+
+	for _, tt := range tests {
+		testutil.RunTest(t, tt)
+	}
+}
+
+func getVanillaRootCmd(t *testing.T) *cobra.Command {
 	rootCmd, _, err := cmd.NewRootCmd(nil)
 	if err != nil {
 		t.Fatalf("Could not create root command: %s", err.Error())
 	}
-	for _, tt := range cmdTests {
+	return rootCmd
-		test.RunTest(t, tt, rootCmd)
 }
+
+func getDefaultRootCmd(t *testing.T) *cobra.Command {
+	rootCmd, _, err := cmd.NewAirshipCTLCommand(nil)
+	if err != nil {
+		t.Fatalf("Could not create root command: %s", err.Error())
+	}
+	return rootCmd
+}
+
+func getSpecializedRootCmd(t *testing.T) *cobra.Command {
+	rootCmd := getVanillaRootCmd(t)
+	rootCmd.AddCommand(bootstrap.NewBootstrapCommand(&environment.AirshipCTLSettings{}))
+	return rootCmd
 }

cmd/testdata/TestRootGoldenOutput/rootCmd-with-defaults.golden

@@ -0,0 +1,20 @@
+airshipctl is a unified entrypoint to various airship components
+
+Usage:
+  airshipctl [command]
+
+Available Commands:
+  argo        argo is the command line interface to Argo
+  bootstrap   bootstraps airshipctl
+  completion  Generate autocompletions script for the specified shell (bash or zsh)
+  help        Help about any command
+  kubeadm     kubeadm: easily bootstrap a secure Kubernetes cluster
+  kubectl     kubectl controls the Kubernetes cluster manager
+  kustomize   Build a kustomization target from a directory or a remote url.
+  version     Show the version number of airshipctl
+
+Flags:
+      --debug   enable verbose output
+  -h, --help    help for airshipctl
+
+Use "airshipctl [command] --help" for more information about a command.

cmd/testdata/TestRootGoldenOutput/specialized-rootCmd-with-bootstrap.golden

@@ -0,0 +1,15 @@
+airshipctl is a unified entrypoint to various airship components
+
+Usage:
+  airshipctl [command]
+
+Available Commands:
+  bootstrap   bootstraps airshipctl
+  help        Help about any command
+  version     Show the version number of airshipctl
+
+Flags:
+      --debug   enable verbose output
+  -h, --help    help for airshipctl
+
+Use "airshipctl [command] --help" for more information about a command.

cmd/version_test.go

@@ -4,22 +4,25 @@ import (
 	"testing"
 
 	"opendev.org/airship/airshipctl/cmd"
-	"opendev.org/airship/airshipctl/test"
+	"opendev.org/airship/airshipctl/testutil"
 )
 
 func TestVersion(t *testing.T) {
-	cmdTests := []*test.CmdTest{
-		{
-			Name:    "version",
-			CmdLine: "version",
-		},
-	}
 	rootCmd, _, err := cmd.NewRootCmd(nil)
 	if err != nil {
 		t.Fatalf("Could not create root command: %s", err.Error())
 	}
 	rootCmd.AddCommand(cmd.NewVersionCommand())
+
+	cmdTests := []*testutil.CmdTest{
+		{
+			Name:    "version",
+			CmdLine: "version",
+			Cmd:     rootCmd,
+		},
+	}
+
 	for _, tt := range cmdTests {
-		test.RunTest(t, tt, rootCmd)
+		testutil.RunTest(t, tt)
 	}
 }

go.mod

@@ -95,7 +95,7 @@ require (
 	gotest.tools v2.2.0+incompatible // indirect
 	k8s.io/api v0.0.0-20190516230258-a675ac48af67 // indirect
 	k8s.io/apiextensions-apiserver v0.0.0-20190516231611-bf6753f2aa24 // indirect
-	k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
+	k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d // indirect
 	k8s.io/apiserver v0.0.0-20190516230822-f89599b3f645 // indirect
 	k8s.io/cli-runtime v0.0.0-20190516231937-17bc0b7fcef5 // indirect
 	k8s.io/client-go v11.0.1-0.20190516230509-ae8359b20417+incompatible

testutil/utilities.go

@@ -1,4 +1,4 @@
-package test
+package testutil
 
 import (
 	"bytes"
@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/spf13/cobra"
-	"k8s.io/apimachinery/pkg/runtime"
 )
 
 // UpdateGolden writes out the golden files with the latest values, rather than failing the test.
@@ -24,22 +23,34 @@ const (
 
 // CmdTest is a command to be run on the command line as a test
 type CmdTest struct {
+	// The name of the test. This will be used when generating golden
+	// files
 	Name string
+
+	// The values that would be inputted to airshipctl as commands, flags,
+	// and arguments. The initial "airshipctl" is implied
 	CmdLine string
-	Objs    []runtime.Object
+
+	// The instatiated version of the root airshipctl command to test
+	Cmd *cobra.Command
 }
 
 // RunTest either asserts that a specific command's output matches the expected
 // output from its golden file, or generates golden files if the -update flag
 // is passed
-func RunTest(t *testing.T, test *CmdTest, cmd *cobra.Command) {
+func RunTest(t *testing.T, test *CmdTest) {
+	cmd := test.Cmd
+
 	actual := &bytes.Buffer{}
 	cmd.SetOutput(actual)
+
 	args := strings.Fields(test.CmdLine)
 	cmd.SetArgs(args)
+
 	if err := cmd.Execute(); err != nil {
 		t.Fatalf("Unexpected error: %s", err.Error())
 	}
+
 	if *shouldUpdateGolden {
 		updateGolden(t, test, actual.Bytes())
 	} else {
@@ -50,13 +61,13 @@ func RunTest(t *testing.T, test *CmdTest, cmd *cobra.Command) {
 func updateGolden(t *testing.T, test *CmdTest, actual []byte) {
 	goldenDir := filepath.Join(testdataDir, t.Name()+goldenDirSuffix)
 	if err := os.MkdirAll(goldenDir, 0775); err != nil {
-		t.Fatalf("failed to create golden directory %s: %s", goldenDir, err)
+		t.Fatalf("Failed to create golden directory %s: %s", goldenDir, err)
 	}
 	t.Logf("Created %s", goldenDir)
 	goldenFilePath := filepath.Join(goldenDir, test.Name+goldenFileSuffix)
-	t.Logf("updating golden file: %s", goldenFilePath)
+	t.Logf("Updating golden file: %s", goldenFilePath)
 	if err := ioutil.WriteFile(goldenFilePath, normalize(actual), 0666); err != nil {
-		t.Fatalf("failed to update golden file: %s", err)
+		t.Fatalf("Failed to update golden file: %s", err)
 	}
 }
 
@@ -65,10 +76,10 @@ func assertEqualGolden(t *testing.T, test *CmdTest, actual []byte) {
 	goldenFilePath := filepath.Join(goldenDir, test.Name+goldenFileSuffix)
 	golden, err := ioutil.ReadFile(goldenFilePath)
 	if err != nil {
-		t.Fatalf("failed while reading golden file: %s", err)
+		t.Fatalf("Failed while reading golden file: %s", err)
 	}
 	if !bytes.Equal(actual, golden) {
-		errFmt := "output does not match golden file: %s\nEXPECTED:\n%s\nGOT:\n%s"
+		errFmt := "Output does not match golden file: %s\nEXPECTED:\n%s\nGOT:\n%s"
 		t.Errorf(errFmt, goldenFilePath, string(golden), string(actual))
 	}
 }

tools/coverage_check

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -ex
+
+if [[ $# -ne 1 ]]; then
+  printf "Usage: %s <coverfile>\n" "$0"
+  exit 0
+fi
+
+cover_file=$1
+min_coverage=80
+
+coverage_report=$(go tool cover -func="$cover_file")
+printf "%s\n" "$coverage_report"
+
+coverage_float=$(awk "/^total:/ { print \$3 }" <<< "$coverage_report")
+coverage_int=${coverage_float%.*}
+
+if (( "$coverage_int" < "$min_coverage" )) ; then
+  printf "FAIL: Test coverage is at %s, which does not meet the required coverage (%s%%)\n" "$coverage_float" "$min_coverage"
+  exit 1
+else
+  printf "SUCCESS: Test coverage is at %s, which meets the required coverage (%s%%)\n" "$coverage_float" "$min_coverage"
+fi