Update datachannel retry strategy to not retry for a specific error scenario

cr https://code.amazon.com/reviews/CR-54790048

agent/session/config/config.go

@@ -65,6 +65,9 @@ const (
 	DataChannelRetryInitialDelayMillis = 100
 	DataChannelRetryMaxIntervalMillis  = 5000
 
+	// MGS Errors
+	SessionAlreadyTerminatedError = "Session is already terminated"
+
 	IpcFileName        = "ipcTempFile"
 	ExecOutputFileName = "output"
 	LogFileExtension   = ".log"

agent/session/controlchannel/controlchannel.go

@@ -123,6 +123,7 @@ func (controlChannel *ControlChannel) SetWebSocket(context context.T,
 			InitialDelayInMilli: rand.Intn(mgsConfig.ControlChannelRetryInitialDelayMillis) + mgsConfig.ControlChannelRetryInitialDelayMillis,
 			MaxDelayInMilli:     mgsConfig.ControlChannelRetryMaxIntervalMillis,
 			MaxAttempts:         mgsConfig.ControlChannelNumMaxRetries,
+			NonRetryableErrors:  getNonRetryableControlChannelErrors(),
 		}
 
 		// add a jitter to the first control-channel call
@@ -364,3 +365,8 @@ func delayWithJitter(maxDelayMillis int64) {
 	jitter := rand.Int63n(maxDelayMillis)
 	time.Sleep(time.Duration(jitter) * time.Millisecond)
 }
+
+// getNonRetryableControlChannelErrors returns list of non retryable errors for control channel retry strategy
+func getNonRetryableControlChannelErrors() []string {
+	return []string{}
+}

agent/session/datachannel/datachannel.go

@@ -299,6 +299,7 @@ func (dataChannel *DataChannel) SetWebSocket(context context.T,
 			InitialDelayInMilli: rand.Intn(mgsConfig.DataChannelRetryInitialDelayMillis) + mgsConfig.DataChannelRetryInitialDelayMillis,
 			MaxDelayInMilli:     mgsConfig.DataChannelRetryMaxIntervalMillis,
 			MaxAttempts:         mgsConfig.DataChannelNumMaxAttempts,
+			NonRetryableErrors:  getNonRetryableDataChannelErrors(),
 		}
 		if _, err := retryer.Call(); err != nil {
 			log.Error(err)
@@ -1142,3 +1143,8 @@ func getMgsEndpoint(context context.T, region string) (string, error) {
 	endpointBuilder.WriteString(hostName)
 	return endpointBuilder.String(), nil
 }
+
+// getNonRetryableDataChannelErrors returns list of non retryable errors for data channel retry strategy
+func getNonRetryableDataChannelErrors() []string {
+	return []string{mgsConfig.SessionAlreadyTerminatedError}
+}

agent/session/retry/retryer.go

@@ -17,6 +17,7 @@ package retry
 import (
 	"math"
 	"math/rand"
+	"strings"
 	"time"
 )
 
@@ -34,6 +35,7 @@ type ExponentialRetryer struct {
 	InitialDelayInMilli int
 	MaxDelayInMilli     int
 	MaxAttempts         int
+	NonRetryableErrors  []string
 }
 
 // Init initializes the retryer
@@ -63,7 +65,7 @@ func (retryer *ExponentialRetryer) Call() (channel interface{}, err error) {
 	failedAttemptsSoFar := 0
 	for {
 		channel, err := retryer.CallableFunc()
-		if err == nil || failedAttemptsSoFar == retryer.MaxAttempts {
+		if err == nil || failedAttemptsSoFar == retryer.MaxAttempts || retryer.isNonRetryableError(err) {
 			return channel, err
 		}
 		sleep, exceedMaxDelay := retryer.NextSleepTime(attempt)
@@ -74,3 +76,13 @@ func (retryer *ExponentialRetryer) Call() (channel interface{}, err error) {
 		failedAttemptsSoFar++
 	}
 }
+
+// isNonRetryableError returns true if passed error is in the list of NonRetryableErrors
+func (retryer *ExponentialRetryer) isNonRetryableError(err error) bool {
+	for _, nonRetryableError := range retryer.NonRetryableErrors {
+		if strings.Contains(err.Error(), nonRetryableError) {
+			return true
+		}
+	}
+	return false
+}

agent/session/retry/retryer_test.go

@@ -37,6 +37,8 @@ var (
 		totalAttempts = totalAttempts + 1
 		return RetryCounter{TotalAttempts: totalAttempts}, errors.New("error occured in callable function")
 	}
+	nonRetryableError = "non retryable error"
+	retryableError    = "retryable error"
 )
 
 func TestRepeatableExponentialRetryerRetriesForGivenNumberOfMaxAttempts(t *testing.T) {
@@ -47,6 +49,7 @@ func TestRepeatableExponentialRetryerRetriesForGivenNumberOfMaxAttempts(t *testi
 		initialDelayInMilli,
 		maxDelayInMilli,
 		maxAttempts,
+		[]string{},
 	}
 
 	retryCounterInterface, err := retryer.Call()
@@ -65,9 +68,82 @@ func TestExponentialRetryerWithJitter(t *testing.T) {
 		initialDelayInMilli,
 		maxDelayInMilli,
 		1,
+		[]string{},
 	}
 	minDelay := int64(initialDelayInMilli) * time.Millisecond.Nanoseconds()
 	maxDelay := int64(float64(minDelay) * (1.0 + jitterRatio))
 	sleep, _ := retryerWithJitter.NextSleepTime(0)
 	assert.True(t, sleep.Nanoseconds() >= minDelay && sleep.Nanoseconds() < maxDelay)
 }
+
+func TestRepeatableExponentialRetryerDoesNotRetryInCaseOfNoError(t *testing.T) {
+	totalAttempts := 0
+	callableFunc := func() (interface{}, error) {
+		totalAttempts = totalAttempts + 1
+		return RetryCounter{TotalAttempts: totalAttempts}, nil
+	}
+
+	retryer := ExponentialRetryer{
+		callableFunc,
+		retryGeometricRatio,
+		jitterRatio,
+		initialDelayInMilli,
+		maxDelayInMilli,
+		maxAttempts,
+		[]string{nonRetryableError},
+	}
+
+	retryCounterInterface, err := retryer.Call()
+
+	retryCounter := retryCounterInterface.(RetryCounter)
+	assert.Nil(t, err)
+	assert.Equal(t, retryCounter.TotalAttempts, 1)
+}
+
+func TestRepeatableExponentialRetryerDoesNotRetryInCaseOfNonRetryableError(t *testing.T) {
+	totalAttempts := 0
+	callableFunc := func() (interface{}, error) {
+		totalAttempts = totalAttempts + 1
+		return RetryCounter{TotalAttempts: totalAttempts}, errors.New(nonRetryableError)
+	}
+
+	retryer := ExponentialRetryer{
+		callableFunc,
+		retryGeometricRatio,
+		jitterRatio,
+		initialDelayInMilli,
+		maxDelayInMilli,
+		maxAttempts,
+		[]string{nonRetryableError},
+	}
+
+	retryCounterInterface, err := retryer.Call()
+
+	retryCounter := retryCounterInterface.(RetryCounter)
+	assert.NotNil(t, err)
+	assert.Equal(t, retryCounter.TotalAttempts, 1)
+}
+
+func TestRepeatableExponentialRetryerRetriesInCaseOfRetryableError(t *testing.T) {
+	totalAttempts := 0
+	callableFunc := func() (interface{}, error) {
+		totalAttempts = totalAttempts + 1
+		return RetryCounter{TotalAttempts: totalAttempts}, errors.New(retryableError)
+	}
+
+	retryer := ExponentialRetryer{
+		callableFunc,
+		retryGeometricRatio,
+		jitterRatio,
+		initialDelayInMilli,
+		maxDelayInMilli,
+		maxAttempts,
+		[]string{nonRetryableError},
+	}
+
+	retryCounterInterface, err := retryer.Call()
+
+	retryCounter := retryCounterInterface.(RetryCounter)
+	assert.NotNil(t, err)
+	assert.Equal(t, retryCounter.TotalAttempts, maxAttempts+1)
+}