fix: return non-zero exit code on test failures

Author: radim

cmd/test.go

@@ -9,13 +9,14 @@ import (
 )
 
 var (
-	testCwd          string
-	testRunFilter    string
-	testFormat       string
-	testOutputPath   string
-	testCommit       bool
-	testNoRestore    bool
-	testForceRestore bool
+	testCwd           string
+	testRunFilter     string
+	testFormat        string
+	testOutputPath    string
+	testCommit        bool
+	testNoRestore     bool
+	testForceRestore  bool
+	testFailOnSkipped bool
 
 	testCmd = &cobra.Command{
 		Use:   "test [flags]",
@@ -25,7 +26,7 @@ var (
 				fmt.Print(err.Error())
 				os.Exit(1)
 			}
-			regresql.Test(testCwd, testRunFilter, testFormat, testOutputPath, testCommit, testNoRestore, testForceRestore)
+			regresql.Test(testCwd, testRunFilter, testFormat, testOutputPath, testCommit, testNoRestore, testForceRestore, testFailOnSkipped)
 		},
 	}
 )
@@ -40,4 +41,5 @@ func init() {
 	testCmd.Flags().BoolVar(&testCommit, "commit", false, "Commit transactions instead of rollback (use with caution)")
 	testCmd.Flags().BoolVar(&testNoRestore, "no-restore", false, "Skip snapshot restore before test")
 	testCmd.Flags().BoolVar(&testForceRestore, "force-restore", false, "Force snapshot restore even if unchanged")
+	testCmd.Flags().BoolVar(&testFailOnSkipped, "fail-on-skipped", false, "Exit with code 2 if skipped tests exist")
 }

regresql/regresql.go

@@ -256,7 +256,16 @@ func validateServerSettings(cfg config, root string) error {
 
 // Test runs regression tests for all queries.
 // Each query runs in its own transaction that rolls back (unless commit is true).
-func Test(root, runFilter, formatName, outputPath string, commit, noRestore, forceRestore bool) {
+//
+// Exit codes:
+//
+//	0  - all tests passed
+//	1  - test failures
+//	2  - skipped tests (if failOnSkipped)
+//	3  - config error
+//	13 - query execution error
+//	14 - invalid formatter
+func Test(root, runFilter, formatName, outputPath string, commit, noRestore, forceRestore, failOnSkipped bool) {
 	config, err := ReadConfig(root)
 	ignorePatterns := []string{}
 	if err == nil {
@@ -314,10 +323,17 @@ func Test(root, runFilter, formatName, outputPath string, commit, noRestore, for
 		os.Exit(14)
 	}
 
-	if err := suite.testQueries(config.PgUri, formatter, outputPath, commit); err != nil {
+	summary, err := suite.testQueries(config.PgUri, formatter, outputPath, commit)
+	if err != nil {
 		fmt.Print(err.Error())
 		os.Exit(13)
 	}
+	if summary.Failed > 0 {
+		os.Exit(1)
+	}
+	if failOnSkipped && summary.Skipped > 0 {
+		os.Exit(2)
+	}
 }
 
 // List walks a repository, builds a Suite instance and pretty prints it.

regresql/suite.go

@@ -289,22 +289,23 @@ func (s *Suite) createExpectedResults(pguri string, commit bool) error {
 // necessary. It then compares the actual output to the expected output and
 // reports results using the specified formatter.
 // Each query runs in its own transaction that rolls back (unless commit is true).
-func (s *Suite) testQueries(pguri string, formatter OutputFormatter, outputPath string, commit bool) error {
+// Returns the test summary for exit code determination.
+func (s *Suite) testQueries(pguri string, formatter OutputFormatter, outputPath string, commit bool) (*TestSummary, error) {
 	db, err := sql.Open("pgx", pguri)
 	if err != nil {
-		return fmt.Errorf("Failed to connect to '%s': %s\n", pguri, err)
+		return nil, fmt.Errorf("Failed to connect to '%s': %s\n", pguri, err)
 	}
 	defer db.Close()
 
 	w, close, err := getWriter(outputPath)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer close()
 
 	summary := NewTestSummary()
 	if err := formatter.Start(w); err != nil {
-		return err
+		return nil, err
 	}
 
 	for _, folder := range s.Dirs {
@@ -319,7 +320,7 @@ func (s *Suite) testQueries(pguri string, formatter OutputFormatter, outputPath
 
 			queries, err := parseQueryFile(qfile)
 			if err != nil {
-				return err
+				return nil, err
 			}
 
 			for _, q := range queries {
@@ -334,7 +335,7 @@ func (s *Suite) testQueries(pguri string, formatter OutputFormatter, outputPath
 
 				p, err := q.GetPlan(rdir)
 				if err != nil {
-					return err
+					return nil, err
 				}
 
 				if err := s.runInTransaction(db, commit, func(tx *sql.Tx) error {
@@ -362,13 +363,16 @@ func (s *Suite) testQueries(pguri string, formatter OutputFormatter, outputPath
 					}
 					return nil
 				}); err != nil {
-					return err
+					return nil, err
 				}
 			}
 		}
 	}
 
-	return formatter.Finish(summary, w)
+	if err := formatter.Finish(summary, w); err != nil {
+		return nil, err
+	}
+	return summary, nil
 }
 
 // runInTransaction executes fn within a transaction, rolling back on error or if commit is false