Commit 46cd37e8 authored by Darrell Kienzle's avatar Darrell Kienzle Committed by Daniel Robinson
Browse files

add logic to limit orchestration directory cleanup to at most once/hour

cr: https://code.amazon.com/reviews/CR-54860524
parent 40dc63b9
...@@ -20,6 +20,7 @@ import ( ...@@ -20,6 +20,7 @@ import (
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime/debug" "runtime/debug"
"sync"
"time" "time"
"github.com/aws/amazon-ssm-agent/agent/appconfig" "github.com/aws/amazon-ssm-agent/agent/appconfig"
...@@ -301,6 +302,36 @@ func isLegacyAssociationDirectory(log log.T, commandOrchestrationPath string) (b ...@@ -301,6 +302,36 @@ func isLegacyAssociationDirectory(log log.T, commandOrchestrationPath string) (b
return false, nil return false, nil
} }
// Global variables to throttle the impact of constantly rechecking the stale orchestation files
var cleanupLock sync.Mutex
var inCleanup = make(map[string]bool)
var nextCleanup = make(map[string]time.Time) // okay that these will default to start of epoch
func getLock(name string) bool {
cleanupLock.Lock()
defer cleanupLock.Unlock()
if inCleanup[name] {
return false
}
if !nextCleanup[name].IsZero() && time.Now().Before(nextCleanup[name]) {
return false
}
inCleanup[name] = true
return true
}
func releaseLock(name string) {
cleanupLock.Lock()
defer cleanupLock.Unlock()
inCleanup[name] = false
}
func updateTime(name string) {
cleanupLock.Lock()
defer cleanupLock.Unlock()
nextCleanup[name] = time.Now().Add(time.Minute * 60)
}
// DeleteOldOrchestrationDirectories deletes expired orchestration directories based on retentionDurationHours and associationRetentionDurationHours. // DeleteOldOrchestrationDirectories deletes expired orchestration directories based on retentionDurationHours and associationRetentionDurationHours.
func DeleteOldOrchestrationDirectories(log log.T, instanceID, orchestrationRootDirName string, retentionDurationHours int, associationRetentionDurationHours int) { func DeleteOldOrchestrationDirectories(log log.T, instanceID, orchestrationRootDirName string, retentionDurationHours int, associationRetentionDurationHours int) {
defer func() { defer func() {
...@@ -309,6 +340,14 @@ func DeleteOldOrchestrationDirectories(log log.T, instanceID, orchestrationRootD ...@@ -309,6 +340,14 @@ func DeleteOldOrchestrationDirectories(log log.T, instanceID, orchestrationRootD
log.Errorf("Stacktrace:\n%s", debug.Stack()) log.Errorf("Stacktrace:\n%s", debug.Stack())
} }
}() }()
// if somebody else is cleaning up this directory, or if it's too soon to try again, bail.
if !getLock(orchestrationRootDirName) {
return
}
// make certain that we release our hold if something goes sideways
defer releaseLock(orchestrationRootDirName)
orchestrationRootDir, dirNames, err := getOrchestrationDirectoryNames(log, instanceID, orchestrationRootDirName, appconfig.DefaultDocumentRootDirName) orchestrationRootDir, dirNames, err := getOrchestrationDirectoryNames(log, instanceID, orchestrationRootDirName, appconfig.DefaultDocumentRootDirName)
if err != nil { if err != nil {
log.Debugf("Failed to get orchestration directories under %v", err) log.Debugf("Failed to get orchestration directories under %v", err)
...@@ -350,7 +389,9 @@ func DeleteOldOrchestrationDirectories(log log.T, instanceID, orchestrationRootD ...@@ -350,7 +389,9 @@ func DeleteOldOrchestrationDirectories(log log.T, instanceID, orchestrationRootD
} }
updateTime(orchestrationRootDir)
log.Debugf("Completed orchestration directory clean up") log.Debugf("Completed orchestration directory clean up")
} }
// DeleteSessionOrchestrationDirectories deletes expired orchestration directories based on session retentionDurationHours. // DeleteSessionOrchestrationDirectories deletes expired orchestration directories based on session retentionDurationHours.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment