// Copyright 2015 Matthew Holt // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package certmagic import ( "crypto/tls" "encoding/json" "fmt" "log" "net" "net/http" "path" "strings" "sync" "sync/atomic" "time" "github.com/go-acme/lego/v3/challenge" "github.com/go-acme/lego/v3/challenge/tlsalpn01" ) // httpSolver solves the HTTP challenge. It must be // associated with a config and an address to use // for solving the challenge. If multiple httpSolvers // are initialized concurrently, the first one to // begin will start the server, and the last one to // finish will stop the server. This solver must be // wrapped by a distributedSolver to work properly, // because the only way the HTTP challenge handler // can access the keyAuth material is by loading it // from storage, which is done by distributedSolver. type httpSolver struct { closed int32 // accessed atomically acmeManager *ACMEManager address string } // Present starts an HTTP server if none is already listening on s.address. func (s *httpSolver) Present(domain, token, keyAuth string) error { solversMu.Lock() defer solversMu.Unlock() si := getSolverInfo(s.address) si.count++ if si.listener != nil { return nil // already be served by us } // notice the unusual error handling here; we // only continue to start a challenge server if // we got a listener; in all other cases return ln, err := robustTryListen(s.address) if ln == nil { return err } // successfully bound socket, so save listener and start key auth HTTP server si.listener = ln go s.serve(si) return nil } // serve is an HTTP server that serves only HTTP challenge responses. func (s *httpSolver) serve(si *solverInfo) { defer close(si.done) httpServer := &http.Server{Handler: s.acmeManager.HTTPChallengeHandler(http.NewServeMux())} httpServer.SetKeepAlivesEnabled(false) err := httpServer.Serve(si.listener) if err != nil && atomic.LoadInt32(&s.closed) != 1 { log.Printf("[ERROR] key auth HTTP server: %v", err) } } // CleanUp cleans up the HTTP server if it is the last one to finish. func (s *httpSolver) CleanUp(domain, token, keyAuth string) error { solversMu.Lock() defer solversMu.Unlock() si := getSolverInfo(s.address) si.count-- if si.count == 0 { // last one out turns off the lights atomic.StoreInt32(&s.closed, 1) if si.listener != nil { si.listener.Close() <-si.done } delete(solvers, s.address) } return nil } // tlsALPNSolver is a type that can solve TLS-ALPN challenges. // It must have an associated config and address on which to // serve the challenge. type tlsALPNSolver struct { config *Config address string } // Present adds the certificate to the certificate cache and, if // needed, starts a TLS server for answering TLS-ALPN challenges. func (s *tlsALPNSolver) Present(domain, token, keyAuth string) error { // load the certificate into the cache; this isn't strictly necessary // if we're using the distributed solver since our GetCertificate // function will check storage for the keyAuth anyway, but it seems // like loading it into the cache is the right thing to do cert, err := tlsalpn01.ChallengeCert(domain, keyAuth) if err != nil { return err } certHash := hashCertificateChain(cert.Certificate) s.config.certCache.mu.Lock() s.config.certCache.cache[tlsALPNCertKeyName(domain)] = Certificate{ Certificate: *cert, Names: []string{domain}, hash: certHash, // perhaps not necesssary } s.config.certCache.mu.Unlock() // the rest of this function increments the // challenge count for the solver at this // listener address, and if necessary, starts // a simple TLS server solversMu.Lock() defer solversMu.Unlock() si := getSolverInfo(s.address) si.count++ if si.listener != nil { return nil // already be served by us } // notice the unusual error handling here; we // only continue to start a challenge server if // we got a listener; in all other cases return ln, err := robustTryListen(s.address) if ln == nil { return err } // we were able to bind the socket, so make it into a TLS // listener, store it with the solverInfo, and start the // challenge server si.listener = tls.NewListener(ln, s.config.TLSConfig()) go func() { defer close(si.done) for { conn, err := si.listener.Accept() if err != nil { if atomic.LoadInt32(&si.closed) == 1 { return } log.Printf("[ERROR] TLS-ALPN challenge server: accept: %v", err) continue } go s.handleConn(conn) } }() return nil } // handleConn completes the TLS handshake and then closes conn. func (*tlsALPNSolver) handleConn(conn net.Conn) { defer conn.Close() tlsConn, ok := conn.(*tls.Conn) if !ok { log.Printf("[ERROR] TLS-ALPN challenge server: expected tls.Conn but got %T: %#v", conn, conn) return } err := tlsConn.Handshake() if err != nil { log.Printf("[ERROR] TLS-ALPN challenge server: handshake: %v", err) return } } // CleanUp removes the challenge certificate from the cache, and if // it is the last one to finish, stops the TLS server. func (s *tlsALPNSolver) CleanUp(domain, token, keyAuth string) error { s.config.certCache.mu.Lock() delete(s.config.certCache.cache, tlsALPNCertKeyName(domain)) s.config.certCache.mu.Unlock() solversMu.Lock() defer solversMu.Unlock() si := getSolverInfo(s.address) si.count-- if si.count == 0 { // last one out turns off the lights atomic.StoreInt32(&si.closed, 1) if si.listener != nil { si.listener.Close() <-si.done } delete(solvers, s.address) } return nil } // tlsALPNCertKeyName returns the key to use when caching a cert // for use with the TLS-ALPN ACME challenge. It is simply to help // avoid conflicts (although at time of writing, there shouldn't // be, since the cert cache is keyed by hash of certificate chain). func tlsALPNCertKeyName(sniName string) string { return sniName + ":acme-tls-alpn" } // distributedSolver allows the ACME HTTP-01 and TLS-ALPN challenges // to be solved by an instance other than the one which initiated it. // This is useful behind load balancers or in other cluster/fleet // configurations. The only requirement is that the instance which // initiates the challenge shares the same storage and locker with // the others in the cluster. The storage backing the certificate // cache in distributedSolver.config is crucial. // // Obviously, the instance which completes the challenge must be // serving on the HTTPChallengePort for the HTTP-01 challenge or the // TLSALPNChallengePort for the TLS-ALPN-01 challenge (or have all // the packets port-forwarded) to receive and handle the request. The // server which receives the challenge must handle it by checking to // see if the challenge token exists in storage, and if so, decode it // and use it to serve up the correct response. HTTPChallengeHandler // in this package as well as the GetCertificate method implemented // by a Config support and even require this behavior. // // In short: the only two requirements for cluster operation are // sharing sync and storage, and using the facilities provided by // this package for solving the challenges. type distributedSolver struct { // The config with a certificate cache // with a reference to the storage to // use which is shared among all the // instances in the cluster - REQUIRED. acmeManager *ACMEManager // Since the distributedSolver is only a // wrapper over an actual solver, place // the actual solver here. providerServer challenge.Provider // The CA endpoint URL associated with // this solver. caURL string } // Present invokes the underlying solver's Present method // and also stores domain, token, and keyAuth to the storage // backing the certificate cache of dhs.acmeManager. func (dhs distributedSolver) Present(domain, token, keyAuth string) error { infoBytes, err := json.Marshal(challengeInfo{ Domain: domain, Token: token, KeyAuth: keyAuth, }) if err != nil { return err } err = dhs.acmeManager.config.Storage.Store(dhs.challengeTokensKey(domain), infoBytes) if err != nil { return err } err = dhs.providerServer.Present(domain, token, keyAuth) if err != nil { return fmt.Errorf("presenting with embedded provider: %v", err) } return nil } // CleanUp invokes the underlying solver's CleanUp method // and also cleans up any assets saved to storage. func (dhs distributedSolver) CleanUp(domain, token, keyAuth string) error { err := dhs.acmeManager.config.Storage.Delete(dhs.challengeTokensKey(domain)) if err != nil { return err } err = dhs.providerServer.CleanUp(domain, token, keyAuth) if err != nil { return fmt.Errorf("cleaning up embedded provider: %v", err) } return nil } // challengeTokensPrefix returns the key prefix for challenge info. func (dhs distributedSolver) challengeTokensPrefix() string { return path.Join(dhs.acmeManager.storageKeyCAPrefix(dhs.caURL), "challenge_tokens") } // challengeTokensKey returns the key to use to store and access // challenge info for domain. func (dhs distributedSolver) challengeTokensKey(domain string) string { return path.Join(dhs.challengeTokensPrefix(), StorageKeys.Safe(domain)+".json") } type challengeInfo struct { Domain, Token, KeyAuth string } // solverInfo associates a listener with the // number of challenges currently using it. type solverInfo struct { closed int32 // accessed atomically count int listener net.Listener done chan struct{} // used to signal when our own solver server is done } // getSolverInfo gets a valid solverInfo struct for address. func getSolverInfo(address string) *solverInfo { si, ok := solvers[address] if !ok { si = &solverInfo{done: make(chan struct{})} solvers[address] = si } return si } // robustTryListen calls net.Listen for a TCP socket at addr. // This function may return both a nil listener and a nil error! // If it was able to bind the socket, it returns the listener // and no error. If it wasn't able to bind the socket because // the socket is already in use, then it returns a nil listener // and nil error. If it had any other error, it returns the // error. The intended error handling logic for this function // is to proceed if the returned listener is not nil; otherwise // return err (which may also be nil). In other words, this // function ignores errors if the socket is already in use, // which is useful for our challenge servers, where we assume // that whatever is already listening can solve the challenges. func robustTryListen(addr string) (net.Listener, error) { var listenErr error for i := 0; i < 2; i++ { // doesn't hurt to sleep briefly before the second // attempt in case the OS has timing issues if i > 0 { time.Sleep(100 * time.Millisecond) } // if we can bind the socket right away, great! var ln net.Listener ln, listenErr = net.Listen("tcp", addr) if listenErr == nil { return ln, nil } // if it failed just because the socket is already in use, we // have no choice but to assume that whatever is using the socket // can answer the challenge already, so we ignore the error connectErr := dialTCPSocket(addr) if connectErr == nil { return nil, nil } // hmm, we couldn't connect to the socket, so something else must // be wrong, right? wrong!! we've had reports across multiple OSes // now that sometimes connections fail even though the OS told us // that the address was already in use; either the listener is // fluctuating between open and closed very, very quickly, or the // OS is inconsistent and contradicting itself; I have been unable // to reproduce this, so I'm now resorting to hard-coding substring // matching in error messages as a really hacky and unreliable // safeguard against this, until we can idenify exactly what was // happening; see the following threads for more info: // https://caddy.community/t/caddy-retry-error/7317 // https://caddy.community/t/v2-upgrade-to-caddy2-failing-with-errors/7423 if strings.Contains(listenErr.Error(), "address already in use") || strings.Contains(listenErr.Error(), "one usage of each socket address") { log.Printf("[WARNING] OS reports a contradiction: %v - but we cannot connect to it, with this error: %v; continuing anyway 🤞 (I don't know what causes this... if you do, please help?)", listenErr, connectErr) return nil, nil } } return nil, fmt.Errorf("could not start listener for challenge server at %s: %v", addr, listenErr) } // dialTCPSocket connects to a TCP address just for the sake of // seeing if it is open. It returns a nil error if a TCP connection // can successfully be made to addr within a short timeout. func dialTCPSocket(addr string) error { conn, err := net.DialTimeout("tcp", addr, 250*time.Millisecond) if err == nil { conn.Close() } return err } // The active challenge solvers, keyed by listener address, // and protected by a mutex. Note that the creation of // solver listeners and the incrementing of their counts // are atomic operations guarded by this mutex. var ( solvers = make(map[string]*solverInfo) solversMu sync.Mutex )