add max job runtime

internal/api/api_test.go

@@ -29,13 +29,14 @@ func init() {
 	tmpDir, _ := ioutil.TempDir("", "gitdeploy-*")
 	runOpts = &options.ServerConfig{
 		//Addr:          "localhost:4483",
-		ScriptsPath:   "./testdata",
+		ScriptsPath:       "./testdata",
-		LogDir:        "./test-logs/api",
+		LogDir:            "./test-logs/api",
-		TmpDir:        tmpDir,
+		TmpDir:            tmpDir,
-		DebounceDelay: 25 * time.Millisecond,
+		DebounceDelay:     25 * time.Millisecond,
-		StaleJobAge:   5 * time.Minute,
+		DefaultMaxJobTime: 5 * time.Second, // very short
-		StaleLogAge:   5 * time.Minute,
+		StaleJobAge:       5 * time.Minute,
-		ExpiredLogAge: 10 * time.Minute,
+		StaleLogAge:       5 * time.Minute,
+		ExpiredLogAge:     10 * time.Minute,
 	}
 	logDir, _ = filepath.Abs(runOpts.LogDir)
 

internal/jobs/job.go

@@ -461,12 +461,25 @@ func run(curHook *webhooks.Ref, runOpts *options.ServerConfig) {
 	Actives.Store(pendingID, j)
 
 	go func() {
+		// TODO make configurable
+		timer := time.AfterFunc(runOpts.DefaultMaxJobTime, func() {
+			if nil == cmd.Process {
+				log.Printf("[SANITY] [%s] never exited, but does not exist", pendingID)
+			}
+			if err := cmd.Process.Kill(); nil != err {
+				log.Printf("[%s] failed to kill process: %v", pendingID, err)
+			}
+			//deathRow <- pendingID
+		})
+
 		//log.Printf("[%s] job started", pendingID)
 		if err := cmd.Wait(); nil != err {
 			log.Printf("[%s] exited with error: %v", pendingID, err)
 		} else {
 			log.Printf("[%s] exited successfully", pendingID)
 		}
+		_ = timer.Stop()
+
 		if nil != txtFile {
 			_ = txtFile.Close()
 		}

internal/jobs/job_test.go

@@ -22,14 +22,15 @@ var t0 = time.Now().UTC()
 func init() {
 	tmpDir, _ := ioutil.TempDir("", "gitdeploy-*")
 	runOpts = &options.ServerConfig{
-		Addr:          "localhost:4483",
+		Addr:              "localhost:4483",
-		ScriptsPath:   "./testdata",
+		ScriptsPath:       "./testdata",
-		LogDir:        "./test-logs/debounce",
+		LogDir:            "./test-logs/debounce",
-		TmpDir:        tmpDir,
+		TmpDir:            tmpDir,
-		DebounceDelay: 25 * time.Millisecond,
+		DebounceDelay:     25 * time.Millisecond,
-		StaleJobAge:   5 * time.Minute,
+		DefaultMaxJobTime: 5 * time.Second, // very short
-		StaleLogAge:   5 * time.Minute,
+		StaleJobAge:       5 * time.Minute,
-		ExpiredLogAge: 10 * time.Minute,
+		StaleLogAge:       5 * time.Minute,
+		ExpiredLogAge:     10 * time.Minute,
 	}
 	logDir, _ = filepath.Abs(runOpts.LogDir)
 
@@ -239,6 +240,7 @@ func TestRecents(t *testing.T) {
 		urlRevID := webhooks.URLSafeGitID(
 			base64.RawURLEncoding.EncodeToString([]byte(hook.GetRevID())),
 		)
+		//lint:ignore SA4006 The linter is wrong, j is used
 		j, err = LoadLogs(runOpts, urlRevID)
 		if nil != err {
 			t.Errorf("error loading logs: %v", err)

internal/options/options.go

@@ -10,19 +10,20 @@ var Server *ServerConfig
 
 // ServerConfig is an options struct
 type ServerConfig struct {
-	Addr          string
+	Addr              string
-	TrustProxy    bool
+	TrustProxy        bool
-	RepoList      string
+	RepoList          string
-	Compress      bool
+	Compress          bool
-	ServePath     string
+	ServePath         string
-	ScriptsPath   string
+	ScriptsPath       string
-	Promotions    []string
+	Promotions        []string
-	LogDir        string // where the job logs should go
+	LogDir            string // where the job logs should go
-	TmpDir        string // where the backlog files go
+	TmpDir            string // where the backlog files go
-	DebounceDelay time.Duration
+	DebounceDelay     time.Duration
-	StaleJobAge   time.Duration // how old a dead job is before it's stale
+	DefaultMaxJobTime time.Duration
-	StaleLogAge   time.Duration
+	StaleJobAge       time.Duration // how old a dead job is before it's stale
-	ExpiredLogAge time.Duration
+	StaleLogAge       time.Duration
+	ExpiredLogAge     time.Duration
 	// TODO use BacklogDir instead?
 }
 

main.go

@@ -48,7 +48,6 @@ func ver() string {
 var runOpts *options.ServerConfig
 var runFlags *flag.FlagSet
 var initFlags *flag.FlagSet
-var promotions []string
 var promotionList string
 var defaultPromotionList = "production,staging,master"
 var oldScripts string
@@ -173,6 +172,9 @@ func main() {
 			}
 			log.Printf("TEMP_DIR=%s", runOpts.TmpDir)
 		}
+		if 0 == runOpts.DefaultMaxJobTime {
+			runOpts.DefaultMaxJobTime = 10 * time.Minute
+		}
 		if 0 == runOpts.DebounceDelay {
 			runOpts.DebounceDelay = 5 * time.Second
 		}