validation: use t.Fail when checking for main test errors

So far `util.Fatal()`, which calls `os.Exit(1)`, has been widely used to
exit immediately when a critical error happened. Its downside is though
that no further function can be called after that moment, even defer
functions cannot be called. That's fine if there's nothing to clean
up or the test is simple enough. Though it could be an issue, for
example, when a test should print out TAP output by calling
`t.AutoPlan()`, or when it should clean up something.

So let's try to use `util.Fatal()` only for errors from critical cases
like errors from `util.GetDefaultGenerator()` or `util.PrepareBundle()`.
In case of main test errors, use instead `t.Fail(err.Error())` to print
out errors in TAP outputs.

Partly addresses #582

/cc @liangchenye

Signed-off-by: Dongsu Park <dongsu@kinvolk.io>

Author: Dongsu Park

validation/config_updates_without_affect/config_updates_without_affect.go

@@ -82,7 +82,7 @@ func main() {
 		err = r.Delete()
 	}
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	t.AutoPlan()

validation/delete/delete.go

@@ -85,7 +85,7 @@ func main() {
 			util.WaitingForStatus(testRuntime, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
 			err = testRuntime.Delete()
 			if err != nil {
-				util.Fatal(err)
+				t.Fail(err.Error())
 			}
 		}
 	}

validation/hooks_stdin/hooks_stdin.go

@@ -140,7 +140,7 @@ func main() {
 
 	err = util.RuntimeLifecycleValidate(config)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	expectedState := rspecs.State{

validation/hostname/hostname.go

@@ -40,7 +40,7 @@ func main() {
 
 	for _, h := range hostnames {
 		if err := testHostname(t, h); err != nil {
-			util.Fatal(err)
+			t.Fail(err.Error())
 		}
 	}
 }

validation/kill_no_effect/kill_no_effect.go

@@ -57,7 +57,7 @@ func main() {
 	}
 	err = util.RuntimeLifecycleValidate(config)
 	if err != nil && err != targetErr {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	} else {
 		util.SpecErrorOK(t, err == nil, targetErr, nil)
 	}

validation/killsig/killsig.go

@@ -31,6 +31,7 @@ func main() {
 	containerID := uuid.NewV4().String()
 	sigConfig, err := util.GetDefaultGenerator()
 	if err != nil {
+		os.RemoveAll(bundleDir)
 		util.Fatal(err)
 	}
 	rootDir := filepath.Join(bundleDir, sigConfig.Spec().Root.Path)

validation/linux_ns_itype/linux_ns_itype.go

@@ -124,7 +124,7 @@ func main() {
 
 	err := testNamespaceInheritType(t)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	t.AutoPlan()

validation/linux_ns_nopath/linux_ns_nopath.go

@@ -125,7 +125,7 @@ func main() {
 
 	err := testNamespaceNoPath(t)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	t.AutoPlan()

validation/start/start.go

@@ -17,6 +17,7 @@ import (
 func main() {
 	t := tap.New()
 	t.Header(0)
+	defer t.AutoPlan()
 
 	bundleDir, err := util.PrepareBundle()
 	if err != nil {
@@ -54,7 +55,8 @@ func main() {
 
 	err = r.Create()
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
+		return
 	}
 	_, err = os.Stat(output)
 	// check the existence of the output file
@@ -67,7 +69,8 @@ func main() {
 	} else {
 		err = util.WaitingForStatus(r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
 		if err != nil {
-			util.Fatal(err)
+			t.Fail(err.Error())
+			return
 		}
 		outputData, outputErr := ioutil.ReadFile(output)
 		// check the output
@@ -81,7 +84,8 @@ func main() {
 
 	err = util.WaitingForStatus(r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
+		return
 	}
 
 	outputData, outputErr := ioutil.ReadFile(output)
@@ -90,7 +94,8 @@ func main() {
 
 	err = r.Delete()
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
+		return
 	}
 
 	g.Spec().Process = nil
@@ -100,7 +105,8 @@ func main() {
 	}
 	err = r.Create()
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
+		return
 	}
 
 	err = r.Start()
@@ -110,8 +116,6 @@ func main() {
 		err = r.Delete()
 	}
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
-
-	t.AutoPlan()
 }