Commit 09ab4ddd authored by nitikag's avatar nitikag Committed by Daniel Robinson
Browse files

Update datachannel retry strategy to not retry for a specific error scenario

cr https://code.amazon.com/reviews/CR-54790048
parent 5742704c
......@@ -65,6 +65,9 @@ const (
DataChannelRetryInitialDelayMillis = 100
DataChannelRetryMaxIntervalMillis = 5000
// MGS Errors
SessionAlreadyTerminatedError = "Session is already terminated"
IpcFileName = "ipcTempFile"
ExecOutputFileName = "output"
LogFileExtension = ".log"
......
......@@ -123,6 +123,7 @@ func (controlChannel *ControlChannel) SetWebSocket(context context.T,
InitialDelayInMilli: rand.Intn(mgsConfig.ControlChannelRetryInitialDelayMillis) + mgsConfig.ControlChannelRetryInitialDelayMillis,
MaxDelayInMilli: mgsConfig.ControlChannelRetryMaxIntervalMillis,
MaxAttempts: mgsConfig.ControlChannelNumMaxRetries,
NonRetryableErrors: getNonRetryableControlChannelErrors(),
}
// add a jitter to the first control-channel call
......@@ -364,3 +365,8 @@ func delayWithJitter(maxDelayMillis int64) {
jitter := rand.Int63n(maxDelayMillis)
time.Sleep(time.Duration(jitter) * time.Millisecond)
}
// getNonRetryableControlChannelErrors returns list of non retryable errors for control channel retry strategy
func getNonRetryableControlChannelErrors() []string {
return []string{}
}
......@@ -299,6 +299,7 @@ func (dataChannel *DataChannel) SetWebSocket(context context.T,
InitialDelayInMilli: rand.Intn(mgsConfig.DataChannelRetryInitialDelayMillis) + mgsConfig.DataChannelRetryInitialDelayMillis,
MaxDelayInMilli: mgsConfig.DataChannelRetryMaxIntervalMillis,
MaxAttempts: mgsConfig.DataChannelNumMaxAttempts,
NonRetryableErrors: getNonRetryableDataChannelErrors(),
}
if _, err := retryer.Call(); err != nil {
log.Error(err)
......@@ -1142,3 +1143,8 @@ func getMgsEndpoint(context context.T, region string) (string, error) {
endpointBuilder.WriteString(hostName)
return endpointBuilder.String(), nil
}
// getNonRetryableDataChannelErrors returns list of non retryable errors for data channel retry strategy
func getNonRetryableDataChannelErrors() []string {
return []string{mgsConfig.SessionAlreadyTerminatedError}
}
......@@ -17,6 +17,7 @@ package retry
import (
"math"
"math/rand"
"strings"
"time"
)
......@@ -34,6 +35,7 @@ type ExponentialRetryer struct {
InitialDelayInMilli int
MaxDelayInMilli int
MaxAttempts int
NonRetryableErrors []string
}
// Init initializes the retryer
......@@ -63,7 +65,7 @@ func (retryer *ExponentialRetryer) Call() (channel interface{}, err error) {
failedAttemptsSoFar := 0
for {
channel, err := retryer.CallableFunc()
if err == nil || failedAttemptsSoFar == retryer.MaxAttempts {
if err == nil || failedAttemptsSoFar == retryer.MaxAttempts || retryer.isNonRetryableError(err) {
return channel, err
}
sleep, exceedMaxDelay := retryer.NextSleepTime(attempt)
......@@ -74,3 +76,13 @@ func (retryer *ExponentialRetryer) Call() (channel interface{}, err error) {
failedAttemptsSoFar++
}
}
// isNonRetryableError returns true if passed error is in the list of NonRetryableErrors
func (retryer *ExponentialRetryer) isNonRetryableError(err error) bool {
for _, nonRetryableError := range retryer.NonRetryableErrors {
if strings.Contains(err.Error(), nonRetryableError) {
return true
}
}
return false
}
......@@ -37,6 +37,8 @@ var (
totalAttempts = totalAttempts + 1
return RetryCounter{TotalAttempts: totalAttempts}, errors.New("error occured in callable function")
}
nonRetryableError = "non retryable error"
retryableError = "retryable error"
)
func TestRepeatableExponentialRetryerRetriesForGivenNumberOfMaxAttempts(t *testing.T) {
......@@ -47,6 +49,7 @@ func TestRepeatableExponentialRetryerRetriesForGivenNumberOfMaxAttempts(t *testi
initialDelayInMilli,
maxDelayInMilli,
maxAttempts,
[]string{},
}
retryCounterInterface, err := retryer.Call()
......@@ -65,9 +68,82 @@ func TestExponentialRetryerWithJitter(t *testing.T) {
initialDelayInMilli,
maxDelayInMilli,
1,
[]string{},
}
minDelay := int64(initialDelayInMilli) * time.Millisecond.Nanoseconds()
maxDelay := int64(float64(minDelay) * (1.0 + jitterRatio))
sleep, _ := retryerWithJitter.NextSleepTime(0)
assert.True(t, sleep.Nanoseconds() >= minDelay && sleep.Nanoseconds() < maxDelay)
}
func TestRepeatableExponentialRetryerDoesNotRetryInCaseOfNoError(t *testing.T) {
totalAttempts := 0
callableFunc := func() (interface{}, error) {
totalAttempts = totalAttempts + 1
return RetryCounter{TotalAttempts: totalAttempts}, nil
}
retryer := ExponentialRetryer{
callableFunc,
retryGeometricRatio,
jitterRatio,
initialDelayInMilli,
maxDelayInMilli,
maxAttempts,
[]string{nonRetryableError},
}
retryCounterInterface, err := retryer.Call()
retryCounter := retryCounterInterface.(RetryCounter)
assert.Nil(t, err)
assert.Equal(t, retryCounter.TotalAttempts, 1)
}
func TestRepeatableExponentialRetryerDoesNotRetryInCaseOfNonRetryableError(t *testing.T) {
totalAttempts := 0
callableFunc := func() (interface{}, error) {
totalAttempts = totalAttempts + 1
return RetryCounter{TotalAttempts: totalAttempts}, errors.New(nonRetryableError)
}
retryer := ExponentialRetryer{
callableFunc,
retryGeometricRatio,
jitterRatio,
initialDelayInMilli,
maxDelayInMilli,
maxAttempts,
[]string{nonRetryableError},
}
retryCounterInterface, err := retryer.Call()
retryCounter := retryCounterInterface.(RetryCounter)
assert.NotNil(t, err)
assert.Equal(t, retryCounter.TotalAttempts, 1)
}
func TestRepeatableExponentialRetryerRetriesInCaseOfRetryableError(t *testing.T) {
totalAttempts := 0
callableFunc := func() (interface{}, error) {
totalAttempts = totalAttempts + 1
return RetryCounter{TotalAttempts: totalAttempts}, errors.New(retryableError)
}
retryer := ExponentialRetryer{
callableFunc,
retryGeometricRatio,
jitterRatio,
initialDelayInMilli,
maxDelayInMilli,
maxAttempts,
[]string{nonRetryableError},
}
retryCounterInterface, err := retryer.Call()
retryCounter := retryCounterInterface.(RetryCounter)
assert.NotNil(t, err)
assert.Equal(t, retryCounter.TotalAttempts, maxAttempts+1)
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment