go-watchdog/watchdog.go

303 lines
6.8 KiB
Go
Raw Normal View History

package watchdog
2019-06-08 02:37:53 +00:00
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
2019-06-08 09:11:42 +00:00
"net/url"
2019-06-08 02:37:53 +00:00
"os/exec"
2019-06-08 09:11:42 +00:00
"strings"
2019-06-08 02:37:53 +00:00
"time"
)
type Dog struct {
Name string
CheckURL string
Keywords string
Recover string
2019-06-08 09:11:42 +00:00
Webhooks []string
2019-06-25 02:00:01 +00:00
AllWebhooks map[string]Webhook
Logger chan string
2019-06-08 02:37:53 +00:00
error error
failures int
passes int
lastFailed time.Time
lastPassed time.Time
lastNotified time.Time
}
2019-06-08 09:11:42 +00:00
func New(d *Dog) *Dog {
d.lastPassed = time.Now().Add(-5 * time.Minute)
return d
2019-06-08 02:37:53 +00:00
}
2019-06-08 09:11:42 +00:00
func (d *Dog) Watch() {
d.watch()
2019-06-08 02:37:53 +00:00
for {
// TODO set cancellable callback ?
time.Sleep(5 * time.Minute)
2019-06-08 09:11:42 +00:00
d.watch()
2019-06-08 02:37:53 +00:00
}
}
2019-06-08 09:11:42 +00:00
func (d *Dog) watch() {
d.Logger <- fmt.Sprintf("Check: '%s'", d.Name)
2019-06-08 02:37:53 +00:00
2019-06-08 09:11:42 +00:00
err := d.check()
2019-06-08 02:37:53 +00:00
if nil == err {
return
}
2019-06-20 23:36:33 +00:00
time.Sleep(time.Duration(2) * time.Second)
err2 := d.check()
if nil != err2 {
d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
} else {
d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
return
}
2019-06-08 02:37:53 +00:00
failure := false
t := 10
for {
2019-06-08 09:11:42 +00:00
d.recover()
2019-06-08 02:37:53 +00:00
time.Sleep(time.Duration(t) * time.Second)
// backoff
t *= 2
2019-06-08 09:11:42 +00:00
err := d.check()
2019-06-08 02:37:53 +00:00
if nil != err {
2019-06-20 23:36:33 +00:00
d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
2019-06-08 02:37:53 +00:00
failure = true
2019-06-12 06:52:33 +00:00
} else {
failure = false
2019-06-08 02:37:53 +00:00
}
2019-06-12 06:52:33 +00:00
2019-06-08 02:37:53 +00:00
// We should notify if
// * We've had success since the last notification
// * It's been at least 5 minutes since the last notification
2019-06-08 09:11:42 +00:00
fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
d.notify(failure)
2019-06-08 02:37:53 +00:00
}
2019-06-12 06:52:33 +00:00
if !failure || d.failures >= 5 {
2019-06-08 02:37:53 +00:00
// go back to the main 5-minute loop
break
}
}
}
2019-06-08 09:11:42 +00:00
func (d *Dog) check() error {
2019-06-08 02:37:53 +00:00
var err error
defer func() {
if nil != err {
2019-06-08 09:11:42 +00:00
d.failures += 1
d.lastFailed = time.Now()
2019-06-08 02:37:53 +00:00
} else {
2019-06-08 09:11:42 +00:00
d.lastPassed = time.Now()
d.passes += 1
2019-06-08 02:37:53 +00:00
}
}()
client := NewHTTPClient()
2019-06-08 09:11:42 +00:00
response, err := client.Get(d.CheckURL)
2019-06-08 02:37:53 +00:00
if nil != err {
2019-06-08 09:11:42 +00:00
d.error = fmt.Errorf("Connection Failure: " + err.Error())
2019-06-08 02:37:53 +00:00
return err
}
b, err := ioutil.ReadAll(response.Body)
if nil != err {
2019-06-08 09:11:42 +00:00
d.error = fmt.Errorf("Network Failure: " + err.Error())
2019-06-08 02:37:53 +00:00
return err
}
2019-06-08 09:11:42 +00:00
if !bytes.Contains(b, []byte(d.Keywords)) {
err = fmt.Errorf("Down: '%s' Not Found for '%s'", d.Keywords, d.Name)
d.Logger <- fmt.Sprintf("%s", err)
2019-06-08 09:11:42 +00:00
d.error = err
2019-06-08 02:37:53 +00:00
return err
} else {
d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
2019-06-08 02:37:53 +00:00
}
return nil
}
2019-06-08 09:11:42 +00:00
func (d *Dog) recover() {
if "" == d.Recover {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
cmd := exec.CommandContext(ctx, "bash")
pipe, err := cmd.StdinPipe()
pipe.Write([]byte(d.Recover))
if nil != err {
d.Logger <- fmt.Sprintf("[Recover] Could not write to bash '%s': %s", d.Recover, err)
2019-06-08 09:11:42 +00:00
}
err = cmd.Start()
if nil != err {
d.Logger <- fmt.Sprintf("[Recover] Could not start '%s': %s", d.Recover, err)
2019-06-08 09:11:42 +00:00
}
err = pipe.Close()
if nil != err {
d.Logger <- fmt.Sprintf("[Recover] Could not close '%s': %s", d.Recover, err)
2019-06-08 09:11:42 +00:00
}
err = cmd.Wait()
cancel()
if nil != err {
d.Logger <- fmt.Sprintf("[Recover] '%s' failed for '%s': %s", d.Recover, d.Name, err)
2019-06-08 09:11:42 +00:00
}
}
func (d *Dog) notify(hardFail bool) {
d.Logger <- fmt.Sprintf("Notifying the authorities of %s's failure", d.Name)
2019-06-08 09:11:42 +00:00
d.lastNotified = time.Now()
for i := range d.Webhooks {
name := d.Webhooks[i]
if "" == name {
continue
}
h, ok := d.AllWebhooks[name]
if !ok {
// TODO check in main when config is read
d.Webhooks[i] = ""
d.Logger <- fmt.Sprintf("[Warning] Could not find webhook '%s' for '%s'", name, h.Name)
2019-06-08 09:11:42 +00:00
continue
}
2019-06-25 02:00:01 +00:00
d.notifyOne(h, hardFail)
}
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
func (d *Dog) notifyOne(h Webhook, hardFail bool) {
// TODO do this in main on config init
if "" == h.Method {
h.Method = "POST"
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
var body *strings.Reader
var err error
// TODO real templates
if 0 != len(h.Form) {
form := url.Values{}
for k := range h.Form {
v := h.Form[k]
// because `{{` gets urlencoded
//k = strings.Replace(k, "{{ .Name }}", d.Name, -1)
v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
d.Logger <- fmt.Sprintf("[HEADER] %s: %s", k, v)
form.Set(k, v)
}
body = strings.NewReader(form.Encode())
} else if 0 != len(h.JSON) {
bodyBuf, err := json.Marshal(h.JSON)
2019-06-08 09:11:42 +00:00
if nil != err {
2019-06-25 02:00:01 +00:00
d.Logger <- fmt.Sprintf("[Notify] JSON Marshal Error for '%s': %s", h.Name, err)
return
2019-06-08 09:11:42 +00:00
}
2019-06-25 02:00:01 +00:00
// `{{` should be left alone
body = strings.NewReader(strings.Replace(string(bodyBuf), "{{ .Name }}", d.Name, -1))
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
client := NewHTTPClient()
req, err := http.NewRequest(h.Method, h.URL, body)
if nil != err {
d.Logger <- fmt.Sprintf("[Notify] HTTP Client Network Error for '%s': %s", h.Name, err)
return
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
if 0 != len(h.Form) {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
} else if 0 != len(h.JSON) {
req.Header.Set("Content-Type", "application/json")
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
if 0 != len(h.Auth) {
user := h.Auth["user"]
if "" == user {
user = h.Auth["username"]
}
2019-06-25 02:00:01 +00:00
pass := h.Auth["pass"]
if "" == user {
pass = h.Auth["password"]
2019-06-08 09:11:42 +00:00
}
2019-06-25 02:00:01 +00:00
req.SetBasicAuth(user, pass)
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
req.Header.Set("User-Agent", "Watchdog/1.0")
for k := range h.Headers {
req.Header.Set(k, h.Headers[k])
}
2019-06-08 09:11:42 +00:00
2019-06-25 02:00:01 +00:00
resp, err := client.Do(req)
if nil != err {
d.Logger <- fmt.Sprintf("[Notify] HTTP Client Error for '%s': %s", h.Name, err)
return
}
if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
d.Logger <- fmt.Sprintf("[Notify] Response Error for '%s': %s", h.Name, resp.Status)
return
}
2019-06-08 02:37:53 +00:00
2019-06-25 02:00:01 +00:00
// TODO json vs xml vs txt
var data map[string]interface{}
req.Header.Add("Accept", "application/json")
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(&data)
if err != nil {
d.Logger <- fmt.Sprintf("[Notify] Response Body Error for '%s': %s", h.Name, resp.Status)
return
2019-06-08 09:11:42 +00:00
}
2019-06-25 02:00:01 +00:00
// TODO some sort of way to determine if data is successful (keywords)
d.Logger <- fmt.Sprintf("[Notify] Success? %#v", data)
2019-06-08 02:37:53 +00:00
}
type Config struct {
2019-06-25 02:00:01 +00:00
Watches []ConfigWatch `json:"watches"`
Webhooks []Webhook `json:"webhooks"`
2019-06-08 02:37:53 +00:00
}
type ConfigWatch struct {
2019-06-08 09:11:42 +00:00
Name string `json:"name"`
URL string `json:"url"`
Keywords string `json:"keywords"`
Webhooks []string `json:"webhooks"`
RecoverScript string `json:"recover_script"`
}
2019-06-25 02:00:01 +00:00
type Webhook struct {
2019-06-08 09:11:42 +00:00
Name string `json:"name"`
Method string `json:"method"`
URL string `json:"url"`
Auth map[string]string `json:"auth"`
Headers map[string]string `json:"headers"`
2019-06-08 09:11:42 +00:00
Form map[string]string `json:"form"`
2019-06-12 06:52:33 +00:00
JSON map[string]string `json:"json"`
2019-06-08 09:11:42 +00:00
Config map[string]string `json:"config"`
Configs []map[string]string `json:"configs"`
}
// The default http client uses unsafe defaults
2019-06-08 09:11:42 +00:00
func NewHTTPClient() *http.Client {
transport := &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
}
client := &http.Client{
Timeout: time.Second * 5,
Transport: transport,
}
return client
}