gitdeploy/internal/api/api.go

package api

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"git.rootprojects.org/root/gitdeploy/internal/log"
	"git.rootprojects.org/root/gitdeploy/internal/options"
	"git.rootprojects.org/root/gitdeploy/internal/webhooks"

	"github.com/go-chi/chi"
)

type job struct {
	ID        string // {HTTPSURL}#{BRANCH}
	Cmd       *exec.Cmd
	GitRef    webhooks.Ref
	CreatedAt time.Time
}

var jobs = make(map[string]*job)
var killers = make(chan string)
var tmpDir string

// Job is the JSON we send back through the API about jobs
type Job struct {
	JobID     string       `json:"job_id"`
	CreatedAt time.Time    `json:"created_at"`
	GitRef    webhooks.Ref `json:"ref"`
	Promote   bool         `json:"promote,omitempty"`
}

// KillMsg describes which job to kill
type KillMsg struct {
	JobID string `json:"job_id"`
	Kill  bool   `json:"kill"`
}

func init() {
	var err error
	tmpDir, err = ioutil.TempDir("", "gitdeploy-*")
	if nil != err {
		fmt.Fprintf(os.Stderr, "could not create temporary directory")
		os.Exit(1)
		return
	}
	log.Printf("TEMP_DIR=%s", tmpDir)
}

// ReposResponse is the successful response to /api/repos
type ReposResponse struct {
	Success bool   `json:"success"`
	Repos   []Repo `json:"repos"`
}

// Repo is one of the elements of /api/repos
type Repo struct {
	ID         string   `json:"id"`
	CloneURL   string   `json:"clone_url"`
	Promotions []string `json:"_promotions"`
}

// Route will set up the API and such
func Route(r chi.Router, runOpts *options.ServerConfig) {

	go func() {
		// TODO read from backlog
		for {
			//hook := webhooks.Accept()
			select {
			case hook := <-webhooks.Hooks:
				runHook(hook, runOpts)
			case jobID := <-killers:
				remove(jobID, false)
			}
		}
	}()

	webhooks.RouteHandlers(r)

	r.Route("/api/admin", func(r chi.Router) {

		r.Use(func(next http.Handler) http.Handler {
			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				// r.Body is always .Close()ed by Go's http server
				r.Body = http.MaxBytesReader(w, r.Body, options.DefaultMaxBodySize)
				// TODO admin auth middleware
				log.Printf("TODO: handle authentication")
				next.ServeHTTP(w, r)
			})
		})

		r.Get("/repos", func(w http.ResponseWriter, r *http.Request) {
			repos := []Repo{}

			for _, id := range strings.Fields(runOpts.RepoList) {
				repos = append(repos, Repo{
					ID:         id,
					CloneURL:   fmt.Sprintf("https://%s.git", id),
					Promotions: runOpts.Promotions,
				})
			}
			err := filepath.Walk(runOpts.ScriptsPath, func(path string, info os.FileInfo, err error) error {
				if nil != err {
					fmt.Printf("error walking %q: %v\n", path, err)
					return nil
				}
				// "scripts/github.com/org/repo"
				parts := strings.Split(filepath.ToSlash(path), "/")
				if len(parts) < 3 {
					return nil
				}
				path = strings.Join(parts[1:], "/")
				if info.Mode().IsRegular() && "deploy.sh" == info.Name() && runOpts.ScriptsPath != path {
					id := filepath.Dir(path)
					repos = append(repos, Repo{
						ID:         id,
						CloneURL:   fmt.Sprintf("https://%s.git", id),
						Promotions: runOpts.Promotions,
					})
				}
				return nil
			})
			if nil != err {
				http.Error(w, "the scripts directory disappeared", http.StatusInternalServerError)
				return
			}
			b, _ := json.MarshalIndent(ReposResponse{
				Success: true,
				Repos:   repos,
			}, "", "  ")
			w.Header().Set("Content-Type", "application/json")
			w.Write(append(b, '\n'))
		})

		r.Get("/jobs", func(w http.ResponseWriter, r *http.Request) {
			// again, possible race condition, but not one that much matters
			jjobs := []Job{}
			for jobID, job := range jobs {
				jjobs = append(jjobs, Job{
					JobID:     jobID,
					GitRef:    job.GitRef,
					CreatedAt: job.CreatedAt,
				})
			}
			b, _ := json.Marshal(struct {
				Success bool  `json:"success"`
				Jobs    []Job `json:"jobs"`
			}{
				Success: true,
				Jobs:    jjobs,
			})
			w.Write(append(b, '\n'))
		})

		r.Post("/jobs", func(w http.ResponseWriter, r *http.Request) {
			defer func() {
				_ = r.Body.Close()
			}()

			decoder := json.NewDecoder(r.Body)
			msg := &KillMsg{}
			if err := decoder.Decode(msg); nil != err {
				log.Printf("kill job invalid json:\n%v", err)
				http.Error(w, "invalid json body", http.StatusBadRequest)
				return
			}

			w.Header().Set("Content-Type", "application/json")
			// possible race condition, but not the kind that should matter
			if _, exists := jobs[msg.JobID]; !exists {
				w.Write([]byte(
					`{ "success": false, "error": "job does not exist" }` + "\n",
				))
				return
			}

			// killing a job *should* always succeed ...right?
			killers <- msg.JobID
			w.Write([]byte(
				`{ "success": true }` + "\n",
			))
		})

		r.Post("/promote", func(w http.ResponseWriter, r *http.Request) {
			decoder := json.NewDecoder(r.Body)
			msg := &webhooks.Ref{}
			if err := decoder.Decode(msg); nil != err {
				log.Printf("promotion job invalid json:\n%v", err)
				http.Error(w, "invalid json body", http.StatusBadRequest)
				return
			}
			if "" == msg.HTTPSURL || "" == msg.RefName {
				log.Printf("promotion job incomplete json %s", msg)
				http.Error(w, "incomplete json body", http.StatusBadRequest)
				return
			}

			n := -2
			for i := range runOpts.Promotions {
				if runOpts.Promotions[i] == msg.RefName {
					n = i - 1
					break
				}
			}
			if n < 0 {
				log.Printf("promotion job invalid: cannot promote: %d", n)
				http.Error(w, "invalid promotion", http.StatusBadRequest)
				return
			}

			promoteTo := runOpts.Promotions[n]
			runPromote(*msg, promoteTo, runOpts)

			b, _ := json.Marshal(struct {
				Success   bool   `json:"success"`
				PromoteTo string `json:"promote_to"`
			}{
				Success:   true,
				PromoteTo: promoteTo,
			})
			w.Write(append(b, '\n'))

		})
	})

}

func runHook(hook webhooks.Ref, runOpts *options.ServerConfig) {
	fmt.Printf("%#v\n", hook)

	jobID := base64.RawURLEncoding.EncodeToString([]byte(
		fmt.Sprintf("%s#%s", hook.HTTPSURL, hook.RefName),
	))
	repoID := getRepoID(hook.HTTPSURL)
	jobName := fmt.Sprintf("%s#%s", strings.ReplaceAll(repoID, "/", "-"), hook.RefName)

	env := os.Environ()
	envs := getEnvs(jobID, runOpts.RepoList, hook)
	envs = append(envs, "GIT_DEPLOY_JOB_ID="+jobID)

	args := []string{
		runOpts.ScriptsPath + "/deploy.sh",
		jobID,
		hook.RefName,
		hook.RefType,
		hook.Owner,
		hook.Repo,
		hook.HTTPSURL,
	}
	cmd := exec.Command("bash", args...)
	cmd.Env = append(env, envs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if _, exists := jobs[jobID]; exists {
		saveBacklog(hook, jobName, jobID)
		log.Printf("[runHook] gitdeploy job already started for %s#%s\n", hook.HTTPSURL, hook.RefName)
		return
	}

	if err := cmd.Start(); nil != err {
		log.Printf("gitdeploy exec error: %s\n", err)
		return
	}

	jobs[jobID] = &job{
		ID:        jobID,
		Cmd:       cmd,
		GitRef:    hook,
		CreatedAt: time.Now(),
	}

	go func() {
		log.Printf("gitdeploy job for %s#%s started\n", hook.HTTPSURL, hook.RefName)
		if err := cmd.Wait(); nil != err {
			log.Printf("gitdeploy job for %s#%s exited with error: %v", hook.HTTPSURL, hook.RefName, err)
		} else {
			log.Printf("gitdeploy job for %s#%s finished\n", hook.HTTPSURL, hook.RefName)
		}
		remove(jobID, true)
		restoreBacklog(jobName, jobID)
	}()
}

func remove(jobID string, nokill bool) {
	job, exists := jobs[jobID]
	if !exists {
		return
	}
	delete(jobs, jobID)

	if nil != job.Cmd.ProcessState {
		// is not yet finished
		if nil != job.Cmd.Process {
			// but definitely was started
			err := job.Cmd.Process.Kill()
			log.Printf("error killing job:\n%v", err)
		}
	}
}

func runPromote(hook webhooks.Ref, promoteTo string, runOpts *options.ServerConfig) {
	// TODO create an origin-branch tag with a timestamp?
	jobID1 := base64.RawURLEncoding.EncodeToString([]byte(
		fmt.Sprintf("%s#%s", hook.HTTPSURL, hook.RefName),
	))
	jobID2 := base64.RawURLEncoding.EncodeToString([]byte(
		fmt.Sprintf("%s#%s", hook.HTTPSURL, promoteTo),
	))

	args := []string{
		runOpts.ScriptsPath + "/promote.sh",
		jobID1,
		promoteTo,
		hook.RefName,
		hook.RefType,
		hook.Owner,
		hook.Repo,
		hook.HTTPSURL,
	}
	cmd := exec.Command("bash", args...)

	env := os.Environ()
	envs := getEnvs(jobID1, runOpts.RepoList, hook)
	envs = append(envs, "GIT_DEPLOY_PROMOTE_TO="+promoteTo)
	cmd.Env = append(env, envs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if _, exists := jobs[jobID1]; exists {
		// TODO put promote in backlog
		log.Printf("[promote] gitdeploy job already started for %s#%s\n", hook.HTTPSURL, hook.RefName)
		return
	}
	if _, exists := jobs[jobID2]; exists {
		// TODO put promote in backlog
		log.Printf("[promote] gitdeploy job already started for %s#%s\n", hook.HTTPSURL, promoteTo)
		return
	}

	if err := cmd.Start(); nil != err {
		log.Printf("gitdeploy exec error: %s\n", err)
		return
	}

	jobs[jobID1] = &job{
		ID:        jobID2,
		Cmd:       cmd,
		GitRef:    hook,
		CreatedAt: time.Now(),
	}
	jobs[jobID2] = &job{
		ID:        jobID2,
		Cmd:       cmd,
		GitRef:    hook,
		CreatedAt: time.Now(),
	}

	go func() {
		log.Printf("gitdeploy promote for %s#%s started\n", hook.HTTPSURL, hook.RefName)
		_ = cmd.Wait()
		killers <- jobID1
		killers <- jobID2
		log.Printf("gitdeploy promote for %s#%s finished\n", hook.HTTPSURL, hook.RefName)
		// TODO check for backlog
	}()
}

func saveBacklog(hook webhooks.Ref, jobName, jobID string) {
	b, _ := json.MarshalIndent(hook, "", "  ")
	f, err := ioutil.TempFile(tmpDir, "tmp-*")
	if nil != err {
		log.Printf("[warn] could not create backlog file for %s:\n%v", jobID, err)
		return
	}
	if _, err := f.Write(b); nil != err {
		log.Printf("[warn] could not write backlog file for %s:\n%v", jobID, err)
		return
	}

	jobFile := filepath.Join(tmpDir, jobName)
	_ = os.Remove(jobFile)
	if err := os.Rename(f.Name(), jobFile); nil != err {
		log.Printf("[warn] could not rename file %s => %s:\n%v", f.Name(), jobFile, err)
		return
	}
	log.Printf("[BACKLOG] new backlog job for %s", jobName)
}

func restoreBacklog(jobName, jobID string) {
	jobFile := filepath.Join(tmpDir, jobName)
	_ = os.Remove(jobFile + ".cur")
	_ = os.Rename(jobFile, jobFile+".cur")

	b, err := ioutil.ReadFile(jobFile + ".cur")
	if nil != err {
		if !os.IsNotExist(err) {
			log.Printf("[warn] could not create backlog file for %s:\n%v", jobID, err)
		}
		// doesn't exist => no backlog
		log.Printf("[NO BACKLOG] no backlog items for %s", jobName)
		return
	}

	ref := webhooks.Ref{}
	if err := json.Unmarshal(b, &ref); nil != err {
		log.Printf("[warn] could not parse backlog file for %s:\n%v", jobID, err)
		return
	}
	log.Printf("[BACKLOG] pop backlog for %s", jobName)
	webhooks.Hook(ref)
}

// https://git.example.com/example/project.git
//      => git.example.com/example/project
func getRepoID(httpsURL string) string {
	repoID := strings.TrimPrefix(httpsURL, "https://")
	repoID = strings.TrimPrefix(repoID, "https://")
	repoID = strings.TrimSuffix(repoID, ".git")
	return repoID
}

func getEnvs(jobID string, repoList string, hook webhooks.Ref) []string {
	repoID := getRepoID(hook.HTTPSURL)

	envs := []string{
		"GIT_DEPLOY_JOB_ID=" + jobID,
		"GIT_REF_NAME=" + hook.RefName,
		"GIT_REF_TYPE=" + hook.RefType,
		"GIT_REPO_ID=" + repoID,
		"GIT_REPO_OWNER=" + hook.Owner,
		"GIT_REPO_NAME=" + hook.Repo,
		"GIT_CLONE_URL=" + hook.HTTPSURL, // deprecated
		"GIT_HTTPS_URL=" + hook.HTTPSURL,
		"GIT_SSH_URL=" + hook.SSHURL,
	}

	// GIT_REPO_TRUSTED
	// Set GIT_REPO_TRUSTED=TRUE if the repo matches exactly, or by pattern
	repoID = strings.ToLower(repoID)
	for _, repo := range strings.Fields(repoList) {
		last := len(repo) - 1
		if len(repo) < 0 {
			continue
		}
		repo = strings.ToLower(repo)
		if '*' == repo[last] {
			// Wildcard match a prefix, for example:
			// github.com/whatever/*					MATCHES github.com/whatever/foo
			// github.com/whatever/ProjectX-* MATCHES github.com/whatever/ProjectX-Foo
			if strings.HasPrefix(repoID, repo[:last]) {
				envs = append(envs, "GIT_REPO_TRUSTED=true")
				break
			}
		} else if repo == repoID {
			envs = append(envs, "GIT_REPO_TRUSTED=true")
			break
		}
	}

	return envs
}