feat: differentiated exit codes (0/1/2/3/4) for the Kosli CLI

TODO.md

@@ -3,6 +3,13 @@
 <!-- Each feature gets its own ## section below. -->
 <!-- Only edit YOUR feature's section. Delete it after merging to main. -->
 
+## Structured exit codes (issue #4988)
+
+- [x] Slice 1: Typed errors + `os.Exit` dispatch in `main.go` + compliance exit code 1
+- [x] Slice 2: HTTP error classification in `requests.go` (5xx→2, 401/403→3, network→2)
+- [x] Slice 3: Usage errors (unknown flag, missing required flag → exit 4)
+- [x] Slice 4: Remaining assert commands + docs
+
 ## kosli evaluate trail
 
 - [x] Slice 1: Skeleton `evaluate` parent + `evaluate trail` fetches trail JSON

cmd/kosli/assertApproval.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/kosli-dev/cli/internal/requests"
 	"github.com/spf13/cobra"
 )
@@ -116,7 +117,7 @@ func (o *assertApprovalOptions) run(args []string) error {
 		return err
 	}
 	if len(approvals) == 0 {
-		return fmt.Errorf("artifact with fingerprint %s has no approvals created", o.fingerprint)
+		return kosliErrors.NewErrCompliance(fmt.Sprintf("artifact with fingerprint %s has no approvals created", o.fingerprint))
 	}
 
 	state, ok := approvals[len(approvals)-1]["state"].(string)
@@ -125,6 +126,6 @@ func (o *assertApprovalOptions) run(args []string) error {
 		logger.Info("artifact with fingerprint %s is approved (approval no. [%v])", o.fingerprint, approvalNumber)
 		return nil
 	} else {
-		return fmt.Errorf("artifact with fingerprint %s is not approved", o.fingerprint)
+		return kosliErrors.NewErrCompliance(fmt.Sprintf("artifact with fingerprint %s is not approved", o.fingerprint))
 	}
 }

cmd/kosli/assertApproval_test.go

@@ -49,10 +49,11 @@ func (suite *AssertApprovalCommandTestSuite) SetupTest() {
 func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
 	tests := []cmdTestCase{
 		{
-			wantError: true,
-			name:      "1 missing --org fails",
-			cmd:       fmt.Sprintf(`assert approval --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c  --flow %s --api-token secret`, suite.flowName),
-			golden:    "Error: --org is not set\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+			wantError:    true,
+			wantExitCode: 4,
+			name:         "1 missing --org fails",
+			cmd:          fmt.Sprintf(`assert approval --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c  --flow %s --api-token secret`, suite.flowName),
+			golden:       "Error: --org is not set\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
 		},
 		{
 			wantError: true,
@@ -61,21 +62,24 @@ func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
 			golden:    "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-approval' belonging to organization 'docs-cmd-test-user'\n",
 		},
 		{
-			wantError: true,
-			name:      "3 asserting an existing artifact that does not have an approval (using --fingerprint) works and exits with non-zero code",
-			cmd:       fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
-			golden:    "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "3 asserting an existing artifact that does not have an approval (using --fingerprint) works and exits with non-zero code",
+			cmd:          fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
+			golden:       "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
 		},
 		{
-			wantError: true,
-			name:      "4 asserting approval of an existing artifact that does not have an approval (using --artifact-type) works and exits with non-zero code",
-			cmd:       fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
-			golden:    "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "4 asserting approval of an existing artifact that does not have an approval (using --artifact-type) works and exits with non-zero code",
+			cmd:          fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
+			golden:       "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
 		},
 		{
-			name:   "5 asserting approval of an existing artifact that has an approval (using --artifact-type) works and exits with zero code",
-			cmd:    fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
-			golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
+			name:         "5 asserting approval of an existing artifact that has an approval (using --artifact-type) works and exits with zero code",
+			wantExitCode: 0,
+			cmd:          fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
+			golden:       "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
 			additionalConfig: assertApprovalTestConfig{
 				createApproval: true,
 				isRequest:      false,
@@ -88,39 +92,44 @@ func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
 			golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
 		},
 		{
-			wantError: true,
-			name:      "7 not providing --fingerprint nor --artifact-type fails",
-			cmd:       fmt.Sprintf(`assert approval --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
-			golden:    "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+			wantError:    true,
+			wantExitCode: 4,
+			name:         "7 not providing --fingerprint nor --artifact-type fails",
+			cmd:          fmt.Sprintf(`assert approval --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
+			golden:       "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
 		},
 		{
-			wantError: true,
-			name:      "8 providing both --fingerprint and --artifact-type fails",
-			cmd:       fmt.Sprintf(`assert approval --artifact-type file --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
-			golden:    "Error: only one of --fingerprint, --artifact-type is allowed\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "8 providing both --fingerprint and --artifact-type fails",
+			cmd:          fmt.Sprintf(`assert approval --artifact-type file --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
+			golden:       "Error: only one of --fingerprint, --artifact-type is allowed\n",
 		},
 		{
-			wantError: true,
-			name:      "9 missing --flow fails",
-			cmd:       fmt.Sprintf(`assert approval --fingerprint %s  %s`, suite.fingerprint, suite.defaultKosliArguments),
-			golden:    "Error: required flag(s) \"flow\" not set\n",
+			wantError:    true,
+			wantExitCode: 4,
+			name:         "9 missing --flow fails",
+			cmd:          fmt.Sprintf(`assert approval --fingerprint %s  %s`, suite.fingerprint, suite.defaultKosliArguments),
+			golden:       "Error: required flag(s) \"flow\" not set\n",
 		},
 		{
-			wantError: true,
-			name:      "10 asserting approval of an unapproved existing artifact (using --artifact-type) works and exits with non-zero code",
-			cmd:       fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
-			golden:    "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "10 asserting approval of an unapproved existing artifact (using --artifact-type) works and exits with non-zero code",
+			cmd:          fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
+			golden:       "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
 			additionalConfig: assertApprovalTestConfig{
 				createApproval: true,
 				isRequest:      true,
 			},
 		},
 		// The approval request created in test 9 is valid here too
 		{
-			wantError: true,
-			name:      "11 asserting approval of an unapproved existing artifact (using --fingerprint) works and exits with non-zero code",
-			cmd:       fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
-			golden:    "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "11 asserting approval of an unapproved existing artifact (using --fingerprint) works and exits with non-zero code",
+			cmd:          fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
+			golden:       "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
 		},
 	}
 

cmd/kosli/assertArtifact.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/kosli-dev/cli/internal/output"
 	"github.com/kosli-dev/cli/internal/requests"
 	"github.com/spf13/cobra"
@@ -169,7 +170,7 @@ func (o *assertArtifactOptions) run(out io.Writer, args []string) error {
 
 	isCompliant := evaluationResult["compliant"].(bool)
 	if !isCompliant {
-		return fmt.Errorf("Artifact is not compliant")
+		return kosliErrors.NewErrCompliance("Artifact is not compliant")
 	}
 	return nil
 }

cmd/kosli/assertArtifact_test.go

@@ -83,10 +83,11 @@ func (suite *AssertArtifactCommandTestSuite) SetupTest() {
 func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
 	tests := []cmdTestCase{
 		{
-			wantError: true,
-			name:      "01 missing --org fails",
-			cmd:       fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c  --flow %s --api-token secret`, suite.flowName1),
-			golden:    "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+			wantError:    true,
+			wantExitCode: 4,
+			name:         "01 missing --org fails",
+			cmd:          fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c  --flow %s --api-token secret`, suite.flowName1),
+			golden:       "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
 		},
 		{
 			wantError: true,
@@ -95,9 +96,10 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
 			golden:    "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact-one' belonging to organization 'docs-cmd-test-user'\n",
 		},
 		{
-			name:        "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
-			cmd:         fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
-			goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
+			name:         "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
+			wantExitCode: 0,
+			cmd:          fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
+			goldenRegex:  "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
 		},
 		{
 			name: "04 json output of asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
@@ -176,28 +178,32 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
 			},
 		},
 		{
-			wantError: true,
-			name:      "14 not providing --fingerprint nor --artifact-type fails",
-			cmd:       fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
-			golden:    "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+			wantError:    true,
+			wantExitCode: 4,
+			name:         "14 not providing --fingerprint nor --artifact-type fails",
+			cmd:          fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
+			golden:       "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
 		},
 		{
-			wantError: true,
-			name:      "15 providing both --environment and --polices fails",
-			cmd:       fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
-			golden:    "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "15 providing both --environment and --polices fails",
+			cmd:          fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
+			golden:       "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
 		},
 		{
-			wantError:   true,
-			name:        "16 asserting a single existing non-compliant artifact (using --fingerprint) results in non-zero exit",
-			cmd:         fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint3, suite.defaultKosliArguments),
-			goldenRegex: "^Error: NON-COMPLIANT\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "16 asserting a single existing non-compliant artifact (using --fingerprint) results in non-zero exit",
+			cmd:          fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint3, suite.defaultKosliArguments),
+			goldenRegex:  "^Error: NON-COMPLIANT\n",
 		},
 		{
-			wantError:   true,
-			name:        "17 asserting a single existing non-compliant artifact (using --artifact-type) results in non-zero exit",
-			cmd:         fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact3Path, suite.defaultKosliArguments),
-			goldenRegex: "^Error: NON-COMPLIANT\n",
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "17 asserting a single existing non-compliant artifact (using --artifact-type) results in non-zero exit",
+			cmd:          fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact3Path, suite.defaultKosliArguments),
+			goldenRegex:  "^Error: NON-COMPLIANT\n",
 		},
 	}
 

cmd/kosli/assertPRAzure.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	azUtils "github.com/kosli-dev/cli/internal/azure"
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -64,7 +65,7 @@ func (o *assertPullRequestAzureOptions) run(args []string) error {
 		return err
 	}
 	if len(pullRequestsEvidence) == 0 {
-		return fmt.Errorf("assert failed: found no pull request(s) in Azure DevOps for commit: %s", o.commit)
+		return kosliErrors.NewErrCompliance(fmt.Sprintf("assert failed: found no pull request(s) in Azure DevOps for commit: %s", o.commit))
 	}
 	logger.Info("found [%d] pull request(s) in Azure DevOps for commit: %s", len(pullRequestsEvidence), o.commit)
 	return nil

cmd/kosli/assertPRAzure_test.go

@@ -30,22 +30,25 @@ func (suite *AssertPRAzureCommandTestSuite) SetupTest() {
 func (suite *AssertPRAzureCommandTestSuite) TestAssertPRAzureCmd() {
 	tests := []cmdTestCase{
 		{
-			name: "assert Azure PR evidence passes when commit has a PR in Azure",
-			cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli 
+			name:         "assert Azure PR evidence passes when commit has a PR in Azure",
+			wantExitCode: 0,
+			cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
 			--commit e6b38318747f1c225e6d2cdba1e88aa00fbcae29` + suite.defaultKosliArguments,
 			golden: "found [1] pull request(s) in Azure DevOps for commit: e6b38318747f1c225e6d2cdba1e88aa00fbcae29\n",
 		},
 		{
-			wantError: true,
-			name:      "assert Azure PR evidence fails when commit has no PRs in Azure",
-			cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli 
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "assert Azure PR evidence fails when commit has no PRs in Azure",
+			cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
 			--commit 58d8aad96e0dcd11ada3dc6650d23909eed336ed` + suite.defaultKosliArguments,
 			golden: "Error: assert failed: found no pull request(s) in Azure DevOps for commit: 58d8aad96e0dcd11ada3dc6650d23909eed336ed\n",
 		},
 		{
-			wantError: true,
-			name:      "assert Azure PR evidence fails when commit does not exist",
-			cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli 
+			wantError:    true,
+			wantExitCode: 1,
+			name:         "assert Azure PR evidence fails when commit does not exist",
+			cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
 			--commit c4fa4c2ce6bef984abc93be9258a85f9137ff1c9` + suite.defaultKosliArguments,
 			golden: "Error: assert failed: found no pull request(s) in Azure DevOps for commit: c4fa4c2ce6bef984abc93be9258a85f9137ff1c9\n",
 		},

cmd/kosli/assertPRBitbucket.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	bbUtils "github.com/kosli-dev/cli/internal/bitbucket"
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -86,7 +87,7 @@ func (o *assertPullRequestBitbucketOptions) run(args []string) error {
 		return err
 	}
 	if len(pullRequestsEvidence) == 0 {
-		return fmt.Errorf("assert failed: found no pull request(s) in Bitbucket for commit: %s", o.commit)
+		return kosliErrors.NewErrCompliance(fmt.Sprintf("assert failed: found no pull request(s) in Bitbucket for commit: %s", o.commit))
 	}
 	logger.Info("found [%d] pull request(s) in Bitbucket for commit: %s", len(pullRequestsEvidence), o.commit)
 	return nil