gitdeploy/internal/api/api.go

472 lines
12 KiB
Go
Raw Normal View History

2020-11-21 12:41:01 +00:00
package api
import (
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
2020-12-01 00:48:09 +00:00
"git.rootprojects.org/root/gitdeploy/internal/log"
2020-11-21 12:41:01 +00:00
"git.rootprojects.org/root/gitdeploy/internal/options"
"git.rootprojects.org/root/gitdeploy/internal/webhooks"
"github.com/go-chi/chi"
)
type job struct {
ID string // {HTTPSURL}#{BRANCH}
Cmd *exec.Cmd
GitRef webhooks.Ref
CreatedAt time.Time
}
var jobs = make(map[string]*job)
var killers = make(chan string)
var tmpDir string
// Job is the JSON we send back through the API about jobs
type Job struct {
JobID string `json:"job_id"`
CreatedAt time.Time `json:"created_at"`
GitRef webhooks.Ref `json:"ref"`
Promote bool `json:"promote,omitempty"`
}
// KillMsg describes which job to kill
type KillMsg struct {
JobID string `json:"job_id"`
Kill bool `json:"kill"`
}
func init() {
var err error
tmpDir, err = ioutil.TempDir("", "gitdeploy-*")
if nil != err {
fmt.Fprintf(os.Stderr, "could not create temporary directory")
os.Exit(1)
return
}
log.Printf("TEMP_DIR=%s", tmpDir)
}
// ReposResponse is the successful response to /api/repos
type ReposResponse struct {
Success bool `json:"success"`
Repos []Repo `json:"repos"`
}
// Repo is one of the elements of /api/repos
type Repo struct {
ID string `json:"id"`
CloneURL string `json:"clone_url"`
Promotions []string `json:"_promotions"`
}
2020-11-21 12:41:01 +00:00
// Route will set up the API and such
func Route(r chi.Router, runOpts *options.ServerConfig) {
go func() {
// TODO read from backlog
for {
//hook := webhooks.Accept()
select {
case hook := <-webhooks.Hooks:
runHook(hook, runOpts)
case jobID := <-killers:
remove(jobID, false)
}
}
}()
webhooks.RouteHandlers(r)
r.Route("/api/admin", func(r chi.Router) {
2020-11-21 12:41:01 +00:00
r.Use(func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// r.Body is always .Close()ed by Go's http server
r.Body = http.MaxBytesReader(w, r.Body, options.DefaultMaxBodySize)
// TODO admin auth middleware
2020-12-01 00:48:09 +00:00
log.Printf("TODO: handle authentication")
2020-11-21 12:41:01 +00:00
next.ServeHTTP(w, r)
})
})
r.Get("/repos", func(w http.ResponseWriter, r *http.Request) {
repos := []Repo{}
for _, id := range strings.Fields(runOpts.RepoList) {
repos = append(repos, Repo{
ID: id,
CloneURL: fmt.Sprintf("https://%s.git", id),
Promotions: runOpts.Promotions,
})
}
err := filepath.Walk(runOpts.ScriptsPath, func(path string, info os.FileInfo, err error) error {
if nil != err {
fmt.Printf("error walking %q: %v\n", path, err)
return nil
}
// "scripts/github.com/org/repo"
parts := strings.Split(filepath.ToSlash(path), "/")
if len(parts) < 3 {
return nil
}
path = strings.Join(parts[1:], "/")
if info.Mode().IsRegular() && "deploy.sh" == info.Name() && runOpts.ScriptsPath != path {
id := filepath.Dir(path)
repos = append(repos, Repo{
ID: id,
CloneURL: fmt.Sprintf("https://%s.git", id),
Promotions: runOpts.Promotions,
})
}
return nil
})
if nil != err {
http.Error(w, "the scripts directory disappeared", http.StatusInternalServerError)
return
}
b, _ := json.MarshalIndent(ReposResponse{
Success: true,
Repos: repos,
}, "", " ")
w.Header().Set("Content-Type", "application/json")
w.Write(append(b, '\n'))
})
2020-11-21 12:41:01 +00:00
r.Get("/jobs", func(w http.ResponseWriter, r *http.Request) {
// again, possible race condition, but not one that much matters
jjobs := []Job{}
for jobID, job := range jobs {
jjobs = append(jjobs, Job{
JobID: jobID,
GitRef: job.GitRef,
CreatedAt: job.CreatedAt,
})
}
b, _ := json.Marshal(struct {
Success bool `json:"success"`
Jobs []Job `json:"jobs"`
}{
Success: true,
Jobs: jjobs,
})
w.Write(append(b, '\n'))
})
2020-11-21 12:41:01 +00:00
r.Post("/jobs", func(w http.ResponseWriter, r *http.Request) {
defer func() {
_ = r.Body.Close()
}()
decoder := json.NewDecoder(r.Body)
msg := &KillMsg{}
if err := decoder.Decode(msg); nil != err {
2020-12-01 00:48:09 +00:00
log.Printf("kill job invalid json:\n%v", err)
2020-11-21 12:41:01 +00:00
http.Error(w, "invalid json body", http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
// possible race condition, but not the kind that should matter
if _, exists := jobs[msg.JobID]; !exists {
w.Write([]byte(
`{ "success": false, "error": "job does not exist" }` + "\n",
))
return
}
// killing a job *should* always succeed ...right?
killers <- msg.JobID
w.Write([]byte(
`{ "success": true }` + "\n",
))
})
r.Post("/promote", func(w http.ResponseWriter, r *http.Request) {
decoder := json.NewDecoder(r.Body)
msg := &webhooks.Ref{}
if err := decoder.Decode(msg); nil != err {
2020-12-01 00:48:09 +00:00
log.Printf("promotion job invalid json:\n%v", err)
2020-11-21 12:41:01 +00:00
http.Error(w, "invalid json body", http.StatusBadRequest)
return
}
if "" == msg.HTTPSURL || "" == msg.RefName {
2020-12-01 00:48:09 +00:00
log.Printf("promotion job incomplete json %s", msg)
2020-11-21 12:41:01 +00:00
http.Error(w, "incomplete json body", http.StatusBadRequest)
return
}
n := -2
for i := range runOpts.Promotions {
if runOpts.Promotions[i] == msg.RefName {
n = i - 1
break
}
}
if n < 0 {
2020-12-01 00:48:09 +00:00
log.Printf("promotion job invalid: cannot promote: %d", n)
2020-11-21 12:41:01 +00:00
http.Error(w, "invalid promotion", http.StatusBadRequest)
return
}
promoteTo := runOpts.Promotions[n]
runPromote(*msg, promoteTo, runOpts)
b, _ := json.Marshal(struct {
Success bool `json:"success"`
PromoteTo string `json:"promote_to"`
}{
Success: true,
PromoteTo: promoteTo,
})
w.Write(append(b, '\n'))
})
})
}
func runHook(hook webhooks.Ref, runOpts *options.ServerConfig) {
fmt.Printf("%#v\n", hook)
2020-11-21 12:41:01 +00:00
jobID := base64.RawURLEncoding.EncodeToString([]byte(
fmt.Sprintf("%s#%s", hook.HTTPSURL, hook.RefName),
))
repoID := getRepoID(hook.HTTPSURL)
jobName := fmt.Sprintf("%s#%s", strings.ReplaceAll(repoID, "/", "-"), hook.RefName)
env := os.Environ()
envs := getEnvs(jobID, runOpts.RepoList, hook)
envs = append(envs, "GIT_DEPLOY_JOB_ID="+jobID)
2020-11-21 12:41:01 +00:00
args := []string{
runOpts.ScriptsPath + "/deploy.sh",
jobID,
hook.RefName,
hook.RefType,
hook.Owner,
hook.Repo,
hook.HTTPSURL,
}
cmd := exec.Command("bash", args...)
cmd.Env = append(env, envs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if _, exists := jobs[jobID]; exists {
saveBacklog(hook, jobName, jobID)
log.Printf("[runHook] gitdeploy job already started for %s#%s\n", hook.HTTPSURL, hook.RefName)
return
}
if err := cmd.Start(); nil != err {
log.Printf("gitdeploy exec error: %s\n", err)
return
}
jobs[jobID] = &job{
ID: jobID,
Cmd: cmd,
GitRef: hook,
CreatedAt: time.Now(),
}
go func() {
log.Printf("gitdeploy job for %s#%s started\n", hook.HTTPSURL, hook.RefName)
if err := cmd.Wait(); nil != err {
log.Printf("gitdeploy job for %s#%s exited with error: %v", hook.HTTPSURL, hook.RefName, err)
} else {
log.Printf("gitdeploy job for %s#%s finished\n", hook.HTTPSURL, hook.RefName)
}
remove(jobID, true)
restoreBacklog(jobName, jobID)
}()
}
func remove(jobID string, nokill bool) {
job, exists := jobs[jobID]
if !exists {
return
}
delete(jobs, jobID)
if nil != job.Cmd.ProcessState {
// is not yet finished
if nil != job.Cmd.Process {
// but definitely was started
err := job.Cmd.Process.Kill()
2020-12-01 00:48:09 +00:00
log.Printf("error killing job:\n%v", err)
2020-11-21 12:41:01 +00:00
}
}
}
func runPromote(hook webhooks.Ref, promoteTo string, runOpts *options.ServerConfig) {
// TODO create an origin-branch tag with a timestamp?
jobID1 := base64.RawURLEncoding.EncodeToString([]byte(
fmt.Sprintf("%s#%s", hook.HTTPSURL, hook.RefName),
))
jobID2 := base64.RawURLEncoding.EncodeToString([]byte(
fmt.Sprintf("%s#%s", hook.HTTPSURL, promoteTo),
))
args := []string{
runOpts.ScriptsPath + "/promote.sh",
jobID1,
promoteTo,
hook.RefName,
hook.RefType,
hook.Owner,
hook.Repo,
hook.HTTPSURL,
}
cmd := exec.Command("bash", args...)
env := os.Environ()
envs := getEnvs(jobID1, runOpts.RepoList, hook)
envs = append(envs, "GIT_DEPLOY_PROMOTE_TO="+promoteTo)
2020-11-21 12:41:01 +00:00
cmd.Env = append(env, envs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if _, exists := jobs[jobID1]; exists {
// TODO put promote in backlog
log.Printf("[promote] gitdeploy job already started for %s#%s\n", hook.HTTPSURL, hook.RefName)
return
}
if _, exists := jobs[jobID2]; exists {
// TODO put promote in backlog
log.Printf("[promote] gitdeploy job already started for %s#%s\n", hook.HTTPSURL, promoteTo)
return
}
if err := cmd.Start(); nil != err {
log.Printf("gitdeploy exec error: %s\n", err)
return
}
jobs[jobID1] = &job{
ID: jobID2,
Cmd: cmd,
GitRef: hook,
CreatedAt: time.Now(),
}
jobs[jobID2] = &job{
ID: jobID2,
Cmd: cmd,
GitRef: hook,
CreatedAt: time.Now(),
}
go func() {
log.Printf("gitdeploy promote for %s#%s started\n", hook.HTTPSURL, hook.RefName)
_ = cmd.Wait()
killers <- jobID1
killers <- jobID2
log.Printf("gitdeploy promote for %s#%s finished\n", hook.HTTPSURL, hook.RefName)
// TODO check for backlog
}()
}
func saveBacklog(hook webhooks.Ref, jobName, jobID string) {
b, _ := json.MarshalIndent(hook, "", " ")
f, err := ioutil.TempFile(tmpDir, "tmp-*")
if nil != err {
log.Printf("[warn] could not create backlog file for %s:\n%v", jobID, err)
return
}
if _, err := f.Write(b); nil != err {
log.Printf("[warn] could not write backlog file for %s:\n%v", jobID, err)
return
}
jobFile := filepath.Join(tmpDir, jobName)
_ = os.Remove(jobFile)
if err := os.Rename(f.Name(), jobFile); nil != err {
log.Printf("[warn] could not rename file %s => %s:\n%v", f.Name(), jobFile, err)
return
}
log.Printf("[BACKLOG] new backlog job for %s", jobName)
}
func restoreBacklog(jobName, jobID string) {
jobFile := filepath.Join(tmpDir, jobName)
_ = os.Remove(jobFile + ".cur")
_ = os.Rename(jobFile, jobFile+".cur")
b, err := ioutil.ReadFile(jobFile + ".cur")
if nil != err {
if !os.IsNotExist(err) {
log.Printf("[warn] could not create backlog file for %s:\n%v", jobID, err)
}
// doesn't exist => no backlog
log.Printf("[NO BACKLOG] no backlog items for %s", jobName)
return
}
ref := webhooks.Ref{}
if err := json.Unmarshal(b, &ref); nil != err {
log.Printf("[warn] could not parse backlog file for %s:\n%v", jobID, err)
return
}
log.Printf("[BACKLOG] pop backlog for %s", jobName)
webhooks.Hook(ref)
}
// https://git.example.com/example/project.git
// => git.example.com/example/project
func getRepoID(httpsURL string) string {
repoID := strings.TrimPrefix(httpsURL, "https://")
repoID = strings.TrimPrefix(repoID, "https://")
repoID = strings.TrimSuffix(repoID, ".git")
return repoID
2020-11-21 12:41:01 +00:00
}
func getEnvs(jobID string, repoList string, hook webhooks.Ref) []string {
repoID := getRepoID(hook.HTTPSURL)
envs := []string{
"GIT_DEPLOY_JOB_ID=" + jobID,
"GIT_REF_NAME=" + hook.RefName,
"GIT_REF_TYPE=" + hook.RefType,
"GIT_REPO_ID=" + repoID,
"GIT_REPO_OWNER=" + hook.Owner,
"GIT_REPO_NAME=" + hook.Repo,
"GIT_CLONE_URL=" + hook.HTTPSURL, // deprecated
"GIT_HTTPS_URL=" + hook.HTTPSURL,
"GIT_SSH_URL=" + hook.SSHURL,
}
// GIT_REPO_TRUSTED
// Set GIT_REPO_TRUSTED=TRUE if the repo matches exactly, or by pattern
repoID = strings.ToLower(repoID)
for _, repo := range strings.Fields(repoList) {
last := len(repo) - 1
if len(repo) < 0 {
continue
}
repo = strings.ToLower(repo)
if '*' == repo[last] {
// Wildcard match a prefix, for example:
// github.com/whatever/* MATCHES github.com/whatever/foo
// github.com/whatever/ProjectX-* MATCHES github.com/whatever/ProjectX-Foo
if strings.HasPrefix(repoID, repo[:last]) {
envs = append(envs, "GIT_REPO_TRUSTED=true")
break
}
} else if repo == repoID {
envs = append(envs, "GIT_REPO_TRUSTED=true")
break
}
}
return envs
2020-11-21 12:41:01 +00:00
}