gitdeploy/main.go

626 lines
16 KiB
Go
Raw Normal View History

package main
import (
"compress/flate"
2020-09-29 03:15:30 +00:00
"encoding/base64"
2020-09-29 10:40:09 +00:00
"encoding/json"
"flag"
"fmt"
2020-09-29 03:15:30 +00:00
"log"
"net/http"
"os"
2020-09-29 03:15:30 +00:00
"os/exec"
"path/filepath"
"strings"
"time"
"git.rootprojects.org/root/vfscopy"
"git.rootprojects.org/root/gitdeploy/assets/public"
"git.rootprojects.org/root/gitdeploy/assets/examples"
2020-10-09 09:22:06 +00:00
"git.rootprojects.org/root/gitdeploy/internal/options"
"git.rootprojects.org/root/gitdeploy/internal/webhooks"
2020-09-28 09:56:51 +00:00
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
2020-09-28 09:56:51 +00:00
_ "github.com/joho/godotenv/autoload"
)
var (
name = "gitdeploy"
version = "0.0.0"
date = "0001-01-01T00:00:00Z"
commit = "0000000"
)
func usage() {
2020-10-20 23:05:01 +00:00
fmt.Println(ver())
fmt.Println("")
2020-10-20 23:05:01 +00:00
fmt.Printf("Use '%s help <command>'\n", name)
fmt.Println(" init")
fmt.Println(" run")
}
2020-10-20 23:05:01 +00:00
func ver() string {
return fmt.Sprintf("%s v%s (%s) %s", name, version, commit[:7], date)
}
2020-09-29 03:15:30 +00:00
type job struct {
2020-09-29 10:40:09 +00:00
ID string // {HTTPSURL}#{BRANCH}
Cmd *exec.Cmd
Ref webhooks.Ref
CreatedAt time.Time
2020-09-29 03:15:30 +00:00
}
var jobs = make(map[string]*job)
2020-09-29 10:40:09 +00:00
var killers = make(chan string)
2020-09-29 03:15:30 +00:00
2020-09-28 23:39:05 +00:00
var runOpts *options.ServerConfig
var runFlags *flag.FlagSet
var initFlags *flag.FlagSet
var promotions []string
var promotionList string
var defaultPromotionList = "production,staging,master"
var oldScripts string
func init() {
2020-09-28 23:39:05 +00:00
runOpts = options.Server
2020-09-28 23:39:05 +00:00
initFlags = options.InitFlags
_ = initFlags.Bool("TODO", false, "init will eventually copy default assets into a local directory")
runFlags = options.ServerFlags
2020-10-09 08:15:39 +00:00
runFlags.StringVar(&runOpts.Addr, "listen", "", "the address and port on which to listen (default :4483)")
2020-09-28 23:39:05 +00:00
runFlags.BoolVar(&runOpts.TrustProxy, "trust-proxy", false, "trust X-Forwarded-For header")
runFlags.StringVar(&runOpts.RepoList, "trust-repos", "",
"list of repos (ex: 'github.com/org/repo', or '*' for all) for which to run '.gitdeploy/deploy.sh'")
2020-09-28 23:39:05 +00:00
runFlags.BoolVar(&runOpts.Compress, "compress", true, "enable compression for text,html,js,css,etc")
2020-09-29 03:15:30 +00:00
runFlags.StringVar(
&runOpts.ServePath, "serve-path", "",
"path to serve, falls back to built-in web app")
runFlags.StringVar(
&oldScripts, "exec", "",
"old alias for --scripts")
runFlags.StringVar(
&runOpts.ScriptsPath, "scripts", "",
2020-09-29 10:40:09 +00:00
"path to ./scripts/{deploy.sh,promote.sh,etc}")
//"path to bash script to run with git info as arguments")
runFlags.StringVar(&promotionList, "promotions", "",
"a list of promotable branches in descending order (default '"+defaultPromotionList+"')")
}
func main() {
2020-10-20 23:05:01 +00:00
// Support [--]version and -V
if len(os.Args) > 1 {
if "version" == strings.TrimLeft(os.Args[1], "-") || "-V" == os.Args[1] {
fmt.Println(ver())
os.Exit(0)
return
}
}
args := os.Args[:]
if 1 == len(args) {
// "run" should be the default
args = append(args, "run")
}
2020-10-20 23:05:01 +00:00
if "help" == strings.TrimLeft(args[1], "-") {
// top-level help
if 2 == len(args) {
usage()
os.Exit(0)
return
}
// move help to subcommand argument
self := args[0]
args = append([]string{self}, args[2:]...)
args = append(args, "--help")
}
switch args[1] {
case "version":
2020-10-20 23:05:01 +00:00
fmt.Println(ver())
os.Exit(0)
return
case "init":
2020-09-29 08:08:35 +00:00
_ = initFlags.Parse(args[2:])
vfs := vfscopy.NewVFS(examples.Assets)
_, err := os.Open("scripts")
if nil == err {
fmt.Fprintf(os.Stderr, "./scripts already exists\n")
os.Exit(1)
return
}
fmt.Println("Copying ...")
if err := vfscopy.CopyAll(vfs, ".", "./scripts", vfscopy.Options {
Skip: func (path string) (bool, error) {
f, _ := vfs.Open(path)
fi, _ := f.Stat()
if !fi.IsDir() {
fmt.Println(" scripts/" + path)
}
return false, nil
},
}); nil != err {
fmt.Fprintf(os.Stderr, "error initializing ./scripts directory\n")
os.Exit(1)
return
}
fmt.Println("Done.")
os.Exit(0)
return
case "run":
2020-09-29 08:08:35 +00:00
_ = runFlags.Parse(args[2:])
if "" == runOpts.ScriptsPath {
if "" != oldScripts {
fmt.Fprintf(os.Stderr, "--exec is deprecated and will be removed. Please use --scripts instead.\n")
runOpts.ScriptsPath = oldScripts
}
}
if "" == runOpts.ScriptsPath {
runOpts.ScriptsPath = "./scripts"
pathname, _ := filepath.Abs("./scripts")
if info, _ := os.Stat("./scripts/deploy.sh"); nil == info || !info.Mode().IsRegular() {
fmt.Printf(
"%q not found.\nPlease provide --scripts ./scripts/ as a path where \"deploy.sh\" can be found\n",
pathname,
)
os.Exit(1)
}
2020-09-29 03:15:30 +00:00
}
2020-10-09 08:15:39 +00:00
if 0 == len(runOpts.Addr) {
runOpts.Addr = os.Getenv("LISTEN")
}
if 0 == len(runOpts.Addr) {
runOpts.Addr = "localhost:4483"
2020-10-09 08:15:39 +00:00
}
if 0 == len(runOpts.Addr) {
fmt.Printf("--listen <[addr]:port> is a required flag")
os.Exit(1)
return
}
if 0 == len(runOpts.RepoList) {
runOpts.RepoList = os.Getenv("TRUST_REPOS")
}
if len(runOpts.RepoList) > 0 {
runOpts.RepoList = strings.ReplaceAll(runOpts.RepoList, ",", " ")
runOpts.RepoList = strings.ReplaceAll(runOpts.RepoList, " ", " ")
}
if 0 == len(promotionList) {
promotionList = os.Getenv("PROMOTIONS")
}
if 0 == len(promotionList) {
promotionList = defaultPromotionList
}
promotions = strings.Fields(
strings.ReplaceAll(promotionList, ",", " "),
)
2020-09-28 23:39:05 +00:00
webhooks.MustRegisterAll()
serve()
default:
usage()
os.Exit(1)
return
}
}
2020-09-29 10:40:09 +00:00
// Job is the JSON we send back through the API about jobs
type Job struct {
JobID string `json:"job_id"`
CreatedAt time.Time `json:"created_at"`
Ref webhooks.Ref `json:"ref"`
Promote bool `json:"promote,omitempty"`
}
// KillMsg describes which job to kill
type KillMsg struct {
JobID string `json:"job_id"`
Kill bool `json:"kill"`
}
func serve() {
r := chi.NewRouter()
// A good base middleware stack
2020-09-28 23:39:05 +00:00
if runOpts.TrustProxy {
r.Use(middleware.RealIP)
}
2020-09-28 23:39:05 +00:00
if runOpts.Compress {
r.Use(middleware.Compress(flate.DefaultCompression))
}
r.Use(middleware.Logger)
r.Use(middleware.Recoverer)
2020-09-28 09:43:30 +00:00
r.Use(middleware.Recoverer)
var staticHandler http.HandlerFunc
pub := http.FileServer(public.Assets)
2020-09-28 23:39:05 +00:00
if len(runOpts.ServePath) > 0 {
// try the user-provided directory first, then fallback to the built-in
2020-09-28 23:39:05 +00:00
devFS := http.Dir(runOpts.ServePath)
dev := http.FileServer(devFS)
staticHandler = func(w http.ResponseWriter, r *http.Request) {
if _, err := devFS.Open(r.URL.Path); nil != err {
pub.ServeHTTP(w, r)
return
}
dev.ServeHTTP(w, r)
}
} else {
staticHandler = func(w http.ResponseWriter, r *http.Request) {
pub.ServeHTTP(w, r)
}
}
2020-09-28 23:39:05 +00:00
webhooks.RouteHandlers(r)
2020-09-28 09:43:30 +00:00
2020-10-20 23:05:01 +00:00
r.Get("/version", func(w http.ResponseWriter, r *http.Request) {
w.Write(append([]byte(ver()), '\n'))
})
r.Get("/api/version", func(w http.ResponseWriter, r *http.Request) {
b, _ := json.MarshalIndent(struct {
Name string `json:"name"`
Version string `json:"version"`
Date string `json:"date"`
Commit string `json:"commit"`
}{
Name: name,
Version: version,
Date: date,
Commit: commit,
}, "", " ")
w.Write(append(b, '\n'))
})
2020-09-29 10:40:09 +00:00
r.Route("/api/admin", func(r chi.Router) {
r.Use(func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// r.Body is always .Close()ed by Go's http server
r.Body = http.MaxBytesReader(w, r.Body, options.DefaultMaxBodySize)
// TODO admin auth middleware
log.Println("TODO: handle authentication")
next.ServeHTTP(w, r)
})
})
r.Get("/repos", func(w http.ResponseWriter, r *http.Request) {
repos := []Repo{}
for _, id := range strings.Fields(
strings.ReplaceAll(runOpts.RepoList, ",", " "),
) {
repos = append(repos, Repo{
ID: id,
CloneURL: fmt.Sprintf("https://%s.git", id),
Promotions: promotions,
})
}
err := filepath.Walk(runOpts.ScriptsPath, func(path string, info os.FileInfo, err error) error {
if nil != err {
fmt.Printf("error walking %q: %v\n", path, err)
return nil
}
2020-10-21 04:49:28 +00:00
// "scripts/github.com/org/repo"
parts := strings.Split(filepath.ToSlash(path), "/")
if len(parts) < 3 {
return nil
}
path = strings.Join(parts[1:], "/")
if info.Mode().IsRegular() && "deploy.sh" == info.Name() && runOpts.ScriptsPath != path {
id := filepath.Dir(path)
repos = append(repos, Repo{
ID: id,
CloneURL: fmt.Sprintf("https://%s.git", id),
Promotions: promotions,
})
}
return nil
})
if nil != err {
http.Error(w, "the scripts directory disappeared", http.StatusInternalServerError)
return
}
b, _ := json.MarshalIndent(ReposResponse{
Success: true,
Repos: repos,
}, "", " ")
w.Header().Set("Content-Type", "application/json")
w.Write(append(b, '\n'))
})
2020-09-29 10:40:09 +00:00
r.Get("/jobs", func(w http.ResponseWriter, r *http.Request) {
// again, possible race condition, but not one that much matters
jjobs := []Job{}
for jobID, job := range jobs {
jjobs = append(jjobs, Job{
JobID: jobID,
Ref: job.Ref,
CreatedAt: job.CreatedAt,
})
}
b, _ := json.Marshal(struct {
Success bool `json:"success"`
Jobs []Job `json:"jobs"`
}{
Success: true,
Jobs: jjobs,
})
w.Write(append(b, '\n'))
})
r.Post("/jobs", func(w http.ResponseWriter, r *http.Request) {
defer func() {
_ = r.Body.Close()
}()
decoder := json.NewDecoder(r.Body)
msg := &KillMsg{}
if err := decoder.Decode(msg); nil != err {
log.Println("kill job invalid json:", err)
http.Error(w, "invalid json body", http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
// possible race condition, but not the kind that should matter
if _, exists := jobs[msg.JobID]; !exists {
w.Write([]byte(
`{ "success": false, "error": "job does not exist" }` + "\n",
))
return
}
// killing a job *should* always succeed ...right?
killers <- msg.JobID
w.Write([]byte(
`{ "success": true }` + "\n",
))
})
r.Post("/promote", func(w http.ResponseWriter, r *http.Request) {
decoder := json.NewDecoder(r.Body)
msg := &webhooks.Ref{}
if err := decoder.Decode(msg); nil != err {
log.Println("promotion job invalid json:", err)
http.Error(w, "invalid json body", http.StatusBadRequest)
return
}
if "" == msg.HTTPSURL || "" == msg.RefName {
log.Println("promotion job incomplete json", msg)
http.Error(w, "incomplete json body", http.StatusBadRequest)
return
}
n := -2
for i := range promotions {
if promotions[i] == msg.RefName {
n = i - 1
break
}
}
if n < 0 {
log.Println("promotion job invalid: cannot promote:", n)
http.Error(w, "invalid promotion", http.StatusBadRequest)
return
}
promoteTo := promotions[n]
runPromote(*msg, promoteTo)
b, _ := json.Marshal(struct {
Success bool `json:"success"`
PromoteTo string `json:"promote_to"`
}{
Success: true,
PromoteTo: promoteTo,
})
w.Write(append(b, '\n'))
2020-09-29 10:40:09 +00:00
})
})
r.Get("/*", staticHandler)
2020-09-28 23:39:05 +00:00
fmt.Println("Listening for http (with reasonable timeouts) on", runOpts.Addr)
srv := &http.Server{
2020-09-28 23:39:05 +00:00
Addr: runOpts.Addr,
Handler: r,
ReadHeaderTimeout: 2 * time.Second,
ReadTimeout: 10 * time.Second,
WriteTimeout: 20 * time.Second,
MaxHeaderBytes: 1024 * 1024, // 1MiB
}
2020-09-28 09:43:30 +00:00
go func() {
2020-09-29 10:40:09 +00:00
// TODO read from backlog
2020-09-28 09:43:30 +00:00
for {
2020-09-29 10:40:09 +00:00
//hook := webhooks.Accept()
select {
case hook := <-webhooks.Hooks:
runHook(hook)
case jobID := <-killers:
2020-10-09 09:30:48 +00:00
remove(jobID, false)
2020-09-29 03:15:30 +00:00
}
2020-09-28 09:43:30 +00:00
}
}()
if err := srv.ListenAndServe(); nil != err {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
return
}
}
2020-09-29 10:40:09 +00:00
func runHook(hook webhooks.Ref) {
fmt.Printf("%#v\n", hook)
jobID := base64.RawURLEncoding.EncodeToString([]byte(
fmt.Sprintf("%s#%s", hook.HTTPSURL, hook.RefName),
))
args := []string{
runOpts.ScriptsPath + "/deploy.sh",
2020-09-29 10:40:09 +00:00
jobID,
hook.RefName,
hook.RefType,
hook.Owner,
hook.Repo,
hook.HTTPSURL,
}
cmd := exec.Command("bash", args...)
// https://git.example.com/example/project.git
// => git.example.com/example/project
repoID := strings.TrimPrefix(hook.HTTPSURL, "https://")
repoID = strings.TrimPrefix(repoID, "https://")
repoID = strings.TrimSuffix(repoID, ".git")
2020-09-29 10:40:09 +00:00
env := os.Environ()
envs := []string{
"GIT_DEPLOY_JOB_ID=" + jobID,
"GIT_REF_NAME=" + hook.RefName,
"GIT_REF_TYPE=" + hook.RefType,
"GIT_REPO_ID=" + repoID,
2020-09-29 10:40:09 +00:00
"GIT_REPO_OWNER=" + hook.Owner,
"GIT_REPO_NAME=" + hook.Repo,
"GIT_CLONE_URL=" + hook.HTTPSURL,
}
for _, repo := range strings.Fields(runOpts.RepoList) {
if "*" == repo || repo == repoID {
envs = append(envs, "GIT_REPO_TRUSTED=true")
break
}
}
2020-09-29 10:40:09 +00:00
cmd.Env = append(env, envs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if _, exists := jobs[jobID]; exists {
// TODO put job in backlog
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy job already started for %s#%s\n", hook.HTTPSURL, hook.RefName)
2020-09-29 10:40:09 +00:00
return
}
if err := cmd.Start(); nil != err {
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy exec error: %s\n", err)
2020-09-29 10:40:09 +00:00
return
}
jobs[jobID] = &job{
ID: jobID,
Cmd: cmd,
Ref: hook,
CreatedAt: time.Now(),
}
go func() {
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy job for %s#%s started\n", hook.HTTPSURL, hook.RefName)
if err := cmd.Wait(); nil != err {
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy job for %s#%s exited with error: %v", hook.HTTPSURL, hook.RefName, err)
2020-10-09 09:30:48 +00:00
} else {
log.Printf("gitdeploy job for %s#%s finished\n", hook.HTTPSURL, hook.RefName)
}
2020-10-09 09:30:48 +00:00
remove(jobID, true)
2020-09-29 10:40:09 +00:00
// TODO check for backlog
}()
}
2020-10-09 09:30:48 +00:00
func remove(jobID string, nokill bool) {
2020-09-29 10:40:09 +00:00
job, exists := jobs[jobID]
if !exists {
return
}
delete(jobs, jobID)
if nil != job.Cmd.ProcessState {
// is not yet finished
if nil != job.Cmd.Process {
// but definitely was started
err := job.Cmd.Process.Kill()
log.Println("error killing job:", err)
}
}
}
func runPromote(hook webhooks.Ref, promoteTo string) {
// TODO create an origin-branch tag with a timestamp?
jobID1 := base64.RawURLEncoding.EncodeToString([]byte(
fmt.Sprintf("%s#%s", hook.HTTPSURL, hook.RefName),
))
jobID2 := base64.RawURLEncoding.EncodeToString([]byte(
fmt.Sprintf("%s#%s", hook.HTTPSURL, promoteTo),
))
args := []string{
runOpts.ScriptsPath + "/promote.sh",
2020-09-29 10:40:09 +00:00
jobID1,
promoteTo,
hook.RefName,
hook.RefType,
hook.Owner,
hook.Repo,
hook.HTTPSURL,
}
cmd := exec.Command("bash", args...)
env := os.Environ()
envs := []string{
"GIT_DEPLOY_JOB_ID=" + jobID1,
"GIT_DEPLOY_PROMOTE_TO=" + promoteTo,
"GIT_REF_NAME=" + hook.RefName,
"GIT_REF_TYPE=" + hook.RefType,
"GIT_REPO_OWNER=" + hook.Owner,
"GIT_REPO_NAME=" + hook.Repo,
"GIT_CLONE_URL=" + hook.HTTPSURL,
}
cmd.Env = append(env, envs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if _, exists := jobs[jobID1]; exists {
// TODO put promote in backlog
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy job already started for %s#%s\n", hook.HTTPSURL, hook.RefName)
2020-09-29 10:40:09 +00:00
return
}
if _, exists := jobs[jobID2]; exists {
// TODO put promote in backlog
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy job already started for %s#%s\n", hook.HTTPSURL, promoteTo)
2020-09-29 10:40:09 +00:00
return
}
if err := cmd.Start(); nil != err {
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy exec error: %s\n", err)
2020-09-29 10:40:09 +00:00
return
}
jobs[jobID1] = &job{
ID: jobID2,
Cmd: cmd,
Ref: hook,
CreatedAt: time.Now(),
}
jobs[jobID2] = &job{
ID: jobID2,
Cmd: cmd,
Ref: hook,
CreatedAt: time.Now(),
}
go func() {
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy promote for %s#%s started\n", hook.HTTPSURL, hook.RefName)
2020-09-29 10:40:09 +00:00
_ = cmd.Wait()
killers <- jobID1
killers <- jobID2
2020-10-09 09:22:06 +00:00
log.Printf("gitdeploy promote for %s#%s finished\n", hook.HTTPSURL, hook.RefName)
2020-09-29 10:40:09 +00:00
// TODO check for backlog
}()
}
// ReposResponse is the successful response to /api/repos
type ReposResponse struct {
Success bool `json:"success"`
Repos []Repo `json:"repos"`
}
// Repo is one of the elements of /api/repos
type Repo struct {
ID string `json:"id"`
CloneURL string `json:"clone_url"`
Promotions []string `json:"_promotions"`
}