add max job runtime

internal/api/api_test.go

@@ -33,6 +33,7 @@ func init() {
 		LogDir:            "./test-logs/api",
 		TmpDir:            tmpDir,
 		DebounceDelay:     25 * time.Millisecond,
+		DefaultMaxJobTime: 5 * time.Second, // very short
 		StaleJobAge:       5 * time.Minute,
 		StaleLogAge:       5 * time.Minute,
 		ExpiredLogAge:     10 * time.Minute,

internal/jobs/job.go

@@ -461,12 +461,25 @@ func run(curHook *webhooks.Ref, runOpts *options.ServerConfig) {
 	Actives.Store(pendingID, j)
 
 	go func() {
+		// TODO make configurable
+		timer := time.AfterFunc(runOpts.DefaultMaxJobTime, func() {
+			if nil == cmd.Process {
+				log.Printf("[SANITY] [%s] never exited, but does not exist", pendingID)
+			}
+			if err := cmd.Process.Kill(); nil != err {
+				log.Printf("[%s] failed to kill process: %v", pendingID, err)
+			}
+			//deathRow <- pendingID
+		})
+
 		//log.Printf("[%s] job started", pendingID)
 		if err := cmd.Wait(); nil != err {
 			log.Printf("[%s] exited with error: %v", pendingID, err)
 		} else {
 			log.Printf("[%s] exited successfully", pendingID)
 		}
+		_ = timer.Stop()
+
 		if nil != txtFile {
 			_ = txtFile.Close()
 		}

internal/jobs/job_test.go

@@ -27,6 +27,7 @@ func init() {
 		LogDir:            "./test-logs/debounce",
 		TmpDir:            tmpDir,
 		DebounceDelay:     25 * time.Millisecond,
+		DefaultMaxJobTime: 5 * time.Second, // very short
 		StaleJobAge:       5 * time.Minute,
 		StaleLogAge:       5 * time.Minute,
 		ExpiredLogAge:     10 * time.Minute,
@@ -239,6 +240,7 @@ func TestRecents(t *testing.T) {
 		urlRevID := webhooks.URLSafeGitID(
 			base64.RawURLEncoding.EncodeToString([]byte(hook.GetRevID())),
 		)
+		//lint:ignore SA4006 The linter is wrong, j is used
 		j, err = LoadLogs(runOpts, urlRevID)
 		if nil != err {
 			t.Errorf("error loading logs: %v", err)

internal/options/options.go

@@ -20,6 +20,7 @@ type ServerConfig struct {
 	LogDir            string // where the job logs should go
 	TmpDir            string // where the backlog files go
 	DebounceDelay     time.Duration
+	DefaultMaxJobTime time.Duration
 	StaleJobAge       time.Duration // how old a dead job is before it's stale
 	StaleLogAge       time.Duration
 	ExpiredLogAge     time.Duration

main.go

@@ -48,7 +48,6 @@ func ver() string {
 var runOpts *options.ServerConfig
 var runFlags *flag.FlagSet
 var initFlags *flag.FlagSet
-var promotions []string
 var promotionList string
 var defaultPromotionList = "production,staging,master"
 var oldScripts string
@@ -173,6 +172,9 @@ func main() {
 			}
 			log.Printf("TEMP_DIR=%s", runOpts.TmpDir)
 		}
+		if 0 == runOpts.DefaultMaxJobTime {
+			runOpts.DefaultMaxJobTime = 10 * time.Minute
+		}
 		if 0 == runOpts.DebounceDelay {
 			runOpts.DebounceDelay = 5 * time.Second
 		}