From 0fcea9342f249521c534427ce9c38ce852ec93c2 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 17 Mar 2021 13:10:38 -0600 Subject: [PATCH] add max job runtime --- internal/api/api_test.go | 15 ++++++++------- internal/jobs/job.go | 13 +++++++++++++ internal/jobs/job_test.go | 18 ++++++++++-------- internal/options/options.go | 27 ++++++++++++++------------- main.go | 4 +++- 5 files changed, 48 insertions(+), 29 deletions(-) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 0a198bf..d9de279 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -29,13 +29,14 @@ func init() { tmpDir, _ := ioutil.TempDir("", "gitdeploy-*") runOpts = &options.ServerConfig{ //Addr: "localhost:4483", - ScriptsPath: "./testdata", - LogDir: "./test-logs/api", - TmpDir: tmpDir, - DebounceDelay: 25 * time.Millisecond, - StaleJobAge: 5 * time.Minute, - StaleLogAge: 5 * time.Minute, - ExpiredLogAge: 10 * time.Minute, + ScriptsPath: "./testdata", + LogDir: "./test-logs/api", + TmpDir: tmpDir, + DebounceDelay: 25 * time.Millisecond, + DefaultMaxJobTime: 5 * time.Second, // very short + StaleJobAge: 5 * time.Minute, + StaleLogAge: 5 * time.Minute, + ExpiredLogAge: 10 * time.Minute, } logDir, _ = filepath.Abs(runOpts.LogDir) diff --git a/internal/jobs/job.go b/internal/jobs/job.go index 9afa22c..bed0423 100644 --- a/internal/jobs/job.go +++ b/internal/jobs/job.go @@ -461,12 +461,25 @@ func run(curHook *webhooks.Ref, runOpts *options.ServerConfig) { Actives.Store(pendingID, j) go func() { + // TODO make configurable + timer := time.AfterFunc(runOpts.DefaultMaxJobTime, func() { + if nil == cmd.Process { + log.Printf("[SANITY] [%s] never exited, but does not exist", pendingID) + } + if err := cmd.Process.Kill(); nil != err { + log.Printf("[%s] failed to kill process: %v", pendingID, err) + } + //deathRow <- pendingID + }) + //log.Printf("[%s] job started", pendingID) if err := cmd.Wait(); nil != err { log.Printf("[%s] exited with error: %v", pendingID, err) } else { log.Printf("[%s] exited successfully", pendingID) } + _ = timer.Stop() + if nil != txtFile { _ = txtFile.Close() } diff --git a/internal/jobs/job_test.go b/internal/jobs/job_test.go index 5404758..f8a1faa 100644 --- a/internal/jobs/job_test.go +++ b/internal/jobs/job_test.go @@ -22,14 +22,15 @@ var t0 = time.Now().UTC() func init() { tmpDir, _ := ioutil.TempDir("", "gitdeploy-*") runOpts = &options.ServerConfig{ - Addr: "localhost:4483", - ScriptsPath: "./testdata", - LogDir: "./test-logs/debounce", - TmpDir: tmpDir, - DebounceDelay: 25 * time.Millisecond, - StaleJobAge: 5 * time.Minute, - StaleLogAge: 5 * time.Minute, - ExpiredLogAge: 10 * time.Minute, + Addr: "localhost:4483", + ScriptsPath: "./testdata", + LogDir: "./test-logs/debounce", + TmpDir: tmpDir, + DebounceDelay: 25 * time.Millisecond, + DefaultMaxJobTime: 5 * time.Second, // very short + StaleJobAge: 5 * time.Minute, + StaleLogAge: 5 * time.Minute, + ExpiredLogAge: 10 * time.Minute, } logDir, _ = filepath.Abs(runOpts.LogDir) @@ -239,6 +240,7 @@ func TestRecents(t *testing.T) { urlRevID := webhooks.URLSafeGitID( base64.RawURLEncoding.EncodeToString([]byte(hook.GetRevID())), ) + //lint:ignore SA4006 The linter is wrong, j is used j, err = LoadLogs(runOpts, urlRevID) if nil != err { t.Errorf("error loading logs: %v", err) diff --git a/internal/options/options.go b/internal/options/options.go index ee44c48..d3e0379 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -10,19 +10,20 @@ var Server *ServerConfig // ServerConfig is an options struct type ServerConfig struct { - Addr string - TrustProxy bool - RepoList string - Compress bool - ServePath string - ScriptsPath string - Promotions []string - LogDir string // where the job logs should go - TmpDir string // where the backlog files go - DebounceDelay time.Duration - StaleJobAge time.Duration // how old a dead job is before it's stale - StaleLogAge time.Duration - ExpiredLogAge time.Duration + Addr string + TrustProxy bool + RepoList string + Compress bool + ServePath string + ScriptsPath string + Promotions []string + LogDir string // where the job logs should go + TmpDir string // where the backlog files go + DebounceDelay time.Duration + DefaultMaxJobTime time.Duration + StaleJobAge time.Duration // how old a dead job is before it's stale + StaleLogAge time.Duration + ExpiredLogAge time.Duration // TODO use BacklogDir instead? } diff --git a/main.go b/main.go index 5b483be..19156ee 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,6 @@ func ver() string { var runOpts *options.ServerConfig var runFlags *flag.FlagSet var initFlags *flag.FlagSet -var promotions []string var promotionList string var defaultPromotionList = "production,staging,master" var oldScripts string @@ -173,6 +172,9 @@ func main() { } log.Printf("TEMP_DIR=%s", runOpts.TmpDir) } + if 0 == runOpts.DefaultMaxJobTime { + runOpts.DefaultMaxJobTime = 10 * time.Minute + } if 0 == runOpts.DebounceDelay { runOpts.DebounceDelay = 5 * time.Second }