feat(service check): add fail early option

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- Add a "Fail early" option to the service status check. When enabled (the default, matching the previous behavior), the "All the time" mode fails as soon as a deviating status is observed. When disabled, the check keeps collecting events for the whole duration and only fails at the end of the step (with a past-tense message, since the status may have recovered by then). Only affects the "All the time" mode.
+
 ## v1.0.28
 
 - chore(deps): bump github.com/steadybit/action-kit/go/action_kit_sdk

extservice/service_check.go

@@ -40,6 +40,10 @@
 	ExpectedStatus     string
 	StatusCheckMode    string
 	StatusCheckSuccess bool
+	FailEarly          bool
+	// DeviationTitle remembers the first observed deviation in 'All the time' + fail-at-end mode
+	// (FailEarly = false) so it can be reported once the step ends.
+	DeviationTitle string
 }
 
 type GetSnapshotApi interface {
@@ -131,6 +135,16 @@
 				Required: new(true),
 				Order:    new(4),
 			},
+			{
+				Name:         "failEarly",
+				Label:        "Fail early",
+				Description:  new("If enabled, the check fails as soon as a deviating status is observed. If disabled, the check keeps collecting events for the whole duration and only fails at the end of the step. Only affects the 'All the time' mode; 'At least once' can only be evaluated at the end of the step."),
+				Type:         action_kit_api.ActionParameterTypeBoolean,
+				DefaultValue: new("true"),
+				Advanced:     new(true),
+				Required:     new(false),
+				Order:        new(5),
+			},
 		},
 		Widgets: new([]action_kit_api.Widget{
 			action_kit_api.StateOverTimeWidget{
@@ -187,6 +201,11 @@
 	state.ExpectedStatus = expectedStatus
 	state.StatusCheckMode = statusCheckMode
 	state.StatusCheckSuccess = state.StatusCheckMode == statusCheckModeAllTheTime
+	// Default to failing early to preserve the previous behavior for experiments that don't set this parameter.
+	state.FailEarly = true
+	if request.Config["failEarly"] != nil {
+		state.FailEarly = extutil.ToBool(request.Config["failEarly"])
+	}
 
 	return nil, nil
 }
@@ -199,7 +218,7 @@
 	return MonitorStatusCheckStatus(ctx, state, Client)
 }
 
 func MonitorStatusCheckStatus(ctx context.Context, state *ServiceStatusCheckState, api GetSnapshotApi) (*action_kit_api.StatusResult, error) {
 	now := time.Now()
 	component, err := loadServiceComponent(ctx, state, api)
 	if err != nil {
@@ -210,15 +229,33 @@
 	var checkError *action_kit_api.ActionKitError
 	if len(state.ExpectedStatus) > 0 {
 		componentHealthState := component.State.HealthState
-		if state.StatusCheckMode == statusCheckModeAllTheTime && componentHealthState != state.ExpectedStatus {
-			checkError = new(action_kit_api.ActionKitError{
-				Title: fmt.Sprintf("Service '%s' (id %s) has status '%s' whereas '%s' is expected.",
-					component.Name,
-					state.ServiceId,
-					componentHealthState,
-					state.ExpectedStatus),
-				Status: extutil.Ptr(action_kit_api.Failed),
-			})
+		if state.StatusCheckMode == statusCheckModeAllTheTime {
+			if componentHealthState != state.ExpectedStatus {
+				if state.FailEarly {
+					// Fail as soon as a deviating status is observed (present tense - it is deviating now).
+					checkError = new(action_kit_api.ActionKitError{
+						Title: fmt.Sprintf("Service '%s' (id %s) has status '%s' whereas '%s' is expected.",
+							component.Name,
+							state.ServiceId,
+							componentHealthState,
+							state.ExpectedStatus),
+						Status: extutil.Ptr(action_kit_api.Failed),
+					})
+				} else if state.DeviationTitle == "" {
+					// Remember the first deviation to report at the end (past tense - it may have recovered).
+					state.DeviationTitle = fmt.Sprintf("Service '%s' (id %s) had status '%s' whereas '%s' is expected.",
+						component.Name,
+						state.ServiceId,
+						componentHealthState,
+						state.ExpectedStatus)
+				}
+			}
+			if !state.FailEarly && completed && state.DeviationTitle != "" {
+				checkError = new(action_kit_api.ActionKitError{
+					Title:  state.DeviationTitle,
+					Status: extutil.Ptr(action_kit_api.Failed),
+				})
+			}
 		} else if state.StatusCheckMode == statusCheckModeAtLeastOnce {
 			if componentHealthState == state.ExpectedStatus {
 				state.StatusCheckSuccess = true

extservice/service_check_test.go

@@ -155,6 +155,28 @@ func TestServiceCheck(t *testing.T) {
 		require.Equal(t, (*status.Metrics)[0].Metric["state"], "warn")
 	})
 
+	t.Run("status allTheTime fail at end", func(t *testing.T) {
+		state := serviceCheckState(statusCheckModeAllTheTime)
+		state.FailEarly = false
+		mockedApi := new(getSnapshotApiMock)
+		mockedApi.On("GetServiceSnapshot", mock.Anything, mock.Anything).Return(apiResponseWithStatus(200), serviceResponseWithState("DEVIATING"), nil)
+
+		// Deviation observed but time not up: must not fail early, deviation is remembered.
+		status, err := MonitorStatusCheckStatus(context.TODO(), &state, mockedApi)
+		require.NoError(t, err)
+		require.False(t, status.Completed)
+		require.Nil(t, status.Error)
+		require.NotEmpty(t, state.DeviationTitle)
+
+		// Time is up: the remembered deviation is reported with the past-tense message.
+		state.End = time.Now().Add(-1 * time.Hour)
+		status, err = MonitorStatusCheckStatus(context.TODO(), &state, mockedApi)
+		require.NoError(t, err)
+		require.True(t, status.Completed)
+		require.NotNil(t, status.Error)
+		require.Contains(t, status.Error.Title, "had status")
+	})
+
 	t.Run("status atLeastOnce success", func(t *testing.T) {
 		state := serviceCheckState(statusCheckModeAtLeastOnce)
 		response := apiResponseWithStatus(200)
@@ -247,6 +269,7 @@ func serviceCheckState(mode string) ServiceStatusCheckState {
 	state.ExpectedStatus = "CLEAR"
 	state.StatusCheckMode = mode
 	state.StatusCheckSuccess = mode == statusCheckModeAllTheTime
+	state.FailEarly = true // matches the production default; 'All the time' fails fast
 	state.End = time.Now().Add(1 * time.Hour)
 	return state
 }