From e7b1ceaf1432428ef2c4c7edc4bab393d90aa371 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 25 Feb 2021 02:49:57 -0700 Subject: [PATCH] limit jobs and logs with '?since=0.0' --- README.md | 4 +- internal/api/api.go | 93 ++++++++++++++++++++++++++++++++++----- internal/api/api_test.go | 22 +++++++-- internal/jobs/job.go | 14 +++++- internal/jobs/job_test.go | 4 +- 5 files changed, 119 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c17de1f..99c6eec 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ GIT_REPO_TRUSTED=true ## API ```txt -GET /api/admin/jobs +GET /api/admin/jobs?since=1577881845.999 { "success": true, @@ -197,7 +197,7 @@ POST /api/admin/jobs { "success": true } -GET /api/admin/logs/{job_id} +GET /api/admin/logs/{job_id}?since=1577881845.999 { "success": true, diff --git a/internal/api/api.go b/internal/api/api.go index 4ed1055..73f5cb7 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -3,10 +3,13 @@ package api import ( "encoding/json" "fmt" + "math" "net/http" "os" "path/filepath" + "strconv" "strings" + "time" "git.rootprojects.org/root/gitdeploy/internal/jobs" "git.rootprojects.org/root/gitdeploy/internal/log" @@ -49,6 +52,8 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { r.Route("/admin", func(r chi.Router) { r.Get("/repos", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + repos := []Repo{} for _, id := range strings.Fields(runOpts.RepoList) { @@ -87,33 +92,48 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { Success: true, Repos: repos, }, "", " ") - w.Header().Set("Content-Type", "application/json") w.Write(append(b, '\n')) }) r.Get("/logs/{oldID}", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - // TODO admin auth middleware - log.Printf("[TODO] handle AUTH (logs could be sensitive)") + since, httpErr := ParseSince(r.URL.Query().Get("since")) + if nil != httpErr { + w.WriteHeader(http.StatusBadRequest) + writeError(w, httpErr) + return + } oldID := webhooks.URLSafeGitID(chi.URLParam(r, "oldID")) // TODO add `since` j, err := jobs.LoadLogs(runOpts, oldID) if nil != err { - w.WriteHeader(404) + w.WriteHeader(http.StatusNotFound) w.Write([]byte( `{ "success": false, "error": "job log does not exist" }` + "\n", )) return } + jobCopy := *j + logs := []jobs.Log{} + for _, log := range j.Logs { + if log.Timestamp.Sub(since) > 0 { + logs = append(logs, log) + } + } + jobCopy.Logs = logs + + // TODO admin auth middleware + log.Printf("[TODO] handle AUTH (logs could be sensitive)") + b, _ := json.MarshalIndent(struct { Success bool `json:"success"` jobs.Job }{ Success: true, - Job: *j, + Job: jobCopy, }, "", " ") w.Write(append(b, '\n')) }) @@ -136,8 +156,16 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { */ r.Get("/jobs", func(w http.ResponseWriter, r *http.Request) { - all := jobs.All() + w.Header().Set("Content-Type", "application/json") + since, httpErr := ParseSince(r.URL.Query().Get("since")) + if nil != httpErr { + w.WriteHeader(http.StatusBadRequest) + writeError(w, httpErr) + return + } + + all := jobs.All(since) b, _ := json.Marshal(struct { Success bool `json:"success"` Jobs []*jobs.Job `json:"jobs"` @@ -149,6 +177,8 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { }) r.Post("/jobs", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + decoder := json.NewDecoder(r.Body) msg := &KillMsg{} if err := decoder.Decode(msg); nil != err { @@ -157,7 +187,6 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { return } - w.Header().Set("Content-Type", "application/json") if _, ok := jobs.Actives.Load(webhooks.URLSafeRefID(msg.JobID)); !ok { if _, ok := jobs.Pending.Load(webhooks.URLSafeRefID(msg.JobID)); !ok { w.Write([]byte( @@ -225,7 +254,7 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { report := &Report{} if err := decoder.Decode(report); nil != err { w.WriteHeader(http.StatusBadRequest) - writeError(w, HTTPError{ + writeError(w, &HTTPError{ Code: "E_PARSE", Message: "could not parse request body", Detail: err.Error(), @@ -236,7 +265,7 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { jobID := webhooks.URLSafeRefID(chi.URLParam(r, "jobID")) if err := jobs.SetReport(jobID, report.Report); nil != err { w.WriteHeader(http.StatusInternalServerError) - writeError(w, HTTPError{ + writeError(w, &HTTPError{ Code: "E_SERVER", Message: "could not update report", Detail: err.Error(), @@ -254,7 +283,51 @@ func RouteStopped(r chi.Router, runOpts *options.ServerConfig) { } -func writeError(w http.ResponseWriter, err HTTPError) { +func ParseSince(sinceStr string) (time.Time, *HTTPError) { + if 0 == len(sinceStr) { + return time.Time{}, &HTTPError{ + Code: "E_QUERY", + Message: "missing query parameter '?since=' (Unix epoch seconds as float64)", + } + } + + t, err := ParseUnixTime(sinceStr) + if nil != err { + return time.Time{}, &HTTPError{ + Code: "E_QUERY", + Message: "invalid query parameter '?since=' (Unix epoch seconds as float64)", + Detail: err.Error(), + } + } + + return t, nil +} + +func ParseUnixTime(seconds string) (time.Time, error) { + secs, nano, err := ParseSeconds(seconds) + if nil != err { + return time.Time{}, err + } + + return time.Unix(secs, nano), nil +} + +func ParseSeconds(s string) (int64, int64, error) { + seconds, err := strconv.ParseFloat(s, 64) + if nil != err { + return 0.0, 0.0, err + } + secs, nanos := SecondsToInts(seconds) + return secs, nanos, nil +} + +func SecondsToInts(seconds float64) (int64, int64) { + secs := math.Floor(seconds) + nanos := math.Round((seconds - secs) * 1000000000) + return int64(secs), int64(nanos) +} + +func writeError(w http.ResponseWriter, err *HTTPError) { enc := json.NewEncoder(w) enc.SetIndent("", " ") _ = enc.Encode(err) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 766304d..364a01a 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math" "net/http" "net/http/httptest" "os" @@ -52,6 +53,19 @@ func init() { //server.Close() } +func ToUnixSeconds(t time.Time) float64 { + // 1614236182.651912345 + secs := float64(t.Unix()) // 1614236182 + nanos := float64(t.Nanosecond()) / 1_000_000_000.0 // 0.651912345 + //nanos := (float64(t.UnixNano()) - secs) / 1_000_000_000.0 // 0.651912345 + + // in my case I want to truncate the precision to milliseconds + nanos = math.Round((10000 * nanos) / 10000) // 0.6519 + + s := secs + nanos // 1614236182.651912345 + return s +} + func TestCallback(t *testing.T) { // TODO use full API request with local webhook t7 := time.Now().Add(-40 * time.Second) @@ -82,10 +96,12 @@ func TestCallback(t *testing.T) { // TODO test that the API gives this back to us urlRevID := hook.GetURLSafeRevID() + s := ToUnixSeconds(t7.Add(-1 * time.Second)) + // TODO needs auth - reqURL := fmt.Sprintf("http://%s/api/admin/logs/%s", - runOpts.Addr, - string(urlRevID), + reqURL := fmt.Sprintf( + "http://%s/api/admin/logs/%s?since=%f", + runOpts.Addr, string(urlRevID), s, ) resp, err := http.Get(reqURL) if nil != err { diff --git a/internal/jobs/job.go b/internal/jobs/job.go index 59ff8bf..04a7fa3 100644 --- a/internal/jobs/job.go +++ b/internal/jobs/job.go @@ -127,7 +127,7 @@ func Stop() { } // All returns all jobs, including active, recent, and (TODO) historical -func All() []*Job { +func All(then time.Time) []*Job { jobsTimersMux.Lock() defer jobsTimersMux.Unlock() @@ -135,6 +135,10 @@ func All() []*Job { Pending.Range(func(key, value interface{}) bool { hook := value.(*webhooks.Ref) + if hook.Timestamp.Sub(then) <= 0 { + return true + } + jobCopy := &Job{ //StartedAt: job.StartedAt, ID: string(hook.GetURLSafeRefID()), @@ -148,6 +152,10 @@ func All() []*Job { Actives.Range(func(key, value interface{}) bool { job := value.(*Job) + if job.GitRef.Timestamp.Sub(then) <= 0 { + return true + } + jobCopy := &Job{ StartedAt: job.StartedAt, ID: string(job.GitRef.GetURLSafeRefID()), @@ -163,6 +171,10 @@ func All() []*Job { Recents.Range(func(key, value interface{}) bool { job := value.(*Job) + if job.GitRef.Timestamp.Sub(then) <= 0 { + return true + } + jobCopy := &Job{ StartedAt: job.StartedAt, ID: string(job.GitRef.GetURLSafeRevID()), diff --git a/internal/jobs/job_test.go b/internal/jobs/job_test.go index 5488114..e7dc079 100644 --- a/internal/jobs/job_test.go +++ b/internal/jobs/job_test.go @@ -93,7 +93,7 @@ func TestDebounce(t *testing.T) { time.Sleep(debounceDelay) var jobMatch *Job - all := All() + all := All(time.Time{}) for i := range all { // WARN: lock value copied j := all[i] @@ -152,7 +152,7 @@ func TestDebounce(t *testing.T) { //var j *Job jobMatch = nil - all = All() + all = All(time.Time{}) for i := range all { j := all[i] //t.Logf("[TEST] B-Job[%d]: %s\n%#v\n", i, j.GitRef.Timestamp, *j.GitRef)