add max job runtime

This commit is contained in:
AJ ONeal 2021-03-17 13:10:38 -06:00
parent d5dec5f3b3
commit 0fcea9342f
5 changed files with 48 additions and 29 deletions

View File

@ -33,6 +33,7 @@ func init() {
LogDir: "./test-logs/api", LogDir: "./test-logs/api",
TmpDir: tmpDir, TmpDir: tmpDir,
DebounceDelay: 25 * time.Millisecond, DebounceDelay: 25 * time.Millisecond,
DefaultMaxJobTime: 5 * time.Second, // very short
StaleJobAge: 5 * time.Minute, StaleJobAge: 5 * time.Minute,
StaleLogAge: 5 * time.Minute, StaleLogAge: 5 * time.Minute,
ExpiredLogAge: 10 * time.Minute, ExpiredLogAge: 10 * time.Minute,

View File

@ -461,12 +461,25 @@ func run(curHook *webhooks.Ref, runOpts *options.ServerConfig) {
Actives.Store(pendingID, j) Actives.Store(pendingID, j)
go func() { go func() {
// TODO make configurable
timer := time.AfterFunc(runOpts.DefaultMaxJobTime, func() {
if nil == cmd.Process {
log.Printf("[SANITY] [%s] never exited, but does not exist", pendingID)
}
if err := cmd.Process.Kill(); nil != err {
log.Printf("[%s] failed to kill process: %v", pendingID, err)
}
//deathRow <- pendingID
})
//log.Printf("[%s] job started", pendingID) //log.Printf("[%s] job started", pendingID)
if err := cmd.Wait(); nil != err { if err := cmd.Wait(); nil != err {
log.Printf("[%s] exited with error: %v", pendingID, err) log.Printf("[%s] exited with error: %v", pendingID, err)
} else { } else {
log.Printf("[%s] exited successfully", pendingID) log.Printf("[%s] exited successfully", pendingID)
} }
_ = timer.Stop()
if nil != txtFile { if nil != txtFile {
_ = txtFile.Close() _ = txtFile.Close()
} }

View File

@ -27,6 +27,7 @@ func init() {
LogDir: "./test-logs/debounce", LogDir: "./test-logs/debounce",
TmpDir: tmpDir, TmpDir: tmpDir,
DebounceDelay: 25 * time.Millisecond, DebounceDelay: 25 * time.Millisecond,
DefaultMaxJobTime: 5 * time.Second, // very short
StaleJobAge: 5 * time.Minute, StaleJobAge: 5 * time.Minute,
StaleLogAge: 5 * time.Minute, StaleLogAge: 5 * time.Minute,
ExpiredLogAge: 10 * time.Minute, ExpiredLogAge: 10 * time.Minute,
@ -239,6 +240,7 @@ func TestRecents(t *testing.T) {
urlRevID := webhooks.URLSafeGitID( urlRevID := webhooks.URLSafeGitID(
base64.RawURLEncoding.EncodeToString([]byte(hook.GetRevID())), base64.RawURLEncoding.EncodeToString([]byte(hook.GetRevID())),
) )
//lint:ignore SA4006 The linter is wrong, j is used
j, err = LoadLogs(runOpts, urlRevID) j, err = LoadLogs(runOpts, urlRevID)
if nil != err { if nil != err {
t.Errorf("error loading logs: %v", err) t.Errorf("error loading logs: %v", err)

View File

@ -20,6 +20,7 @@ type ServerConfig struct {
LogDir string // where the job logs should go LogDir string // where the job logs should go
TmpDir string // where the backlog files go TmpDir string // where the backlog files go
DebounceDelay time.Duration DebounceDelay time.Duration
DefaultMaxJobTime time.Duration
StaleJobAge time.Duration // how old a dead job is before it's stale StaleJobAge time.Duration // how old a dead job is before it's stale
StaleLogAge time.Duration StaleLogAge time.Duration
ExpiredLogAge time.Duration ExpiredLogAge time.Duration

View File

@ -48,7 +48,6 @@ func ver() string {
var runOpts *options.ServerConfig var runOpts *options.ServerConfig
var runFlags *flag.FlagSet var runFlags *flag.FlagSet
var initFlags *flag.FlagSet var initFlags *flag.FlagSet
var promotions []string
var promotionList string var promotionList string
var defaultPromotionList = "production,staging,master" var defaultPromotionList = "production,staging,master"
var oldScripts string var oldScripts string
@ -173,6 +172,9 @@ func main() {
} }
log.Printf("TEMP_DIR=%s", runOpts.TmpDir) log.Printf("TEMP_DIR=%s", runOpts.TmpDir)
} }
if 0 == runOpts.DefaultMaxJobTime {
runOpts.DefaultMaxJobTime = 10 * time.Minute
}
if 0 == runOpts.DebounceDelay { if 0 == runOpts.DebounceDelay {
runOpts.DebounceDelay = 5 * time.Second runOpts.DebounceDelay = 5 * time.Second
} }