diff --git a/cmd/serve_unix.go b/cmd/serve_unix.go index d234a7fb..49133bf1 100644 --- a/cmd/serve_unix.go +++ b/cmd/serve_unix.go @@ -33,7 +33,8 @@ func sigHandlerConfigReload(config string) { // pending writes such as the message cache batch are persisted. It closes stopping first, so // the caller can tell the resulting Run error apart from a real failure. // -// A cluster node drains first (see Server.Drain); a second signal skips the rest of the drain. +// A cluster node first enters maintenance mode and drains (see Server.Drain); a second signal +// skips the rest of the drain. func sigHandlerShutdown(s *server.Server, stopping chan<- struct{}) { sigs := make(chan os.Signal, 2) signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) diff --git a/docs/config.md b/docs/config.md index ad8cdaaf..39a16075 100644 --- a/docs/config.md +++ b/docs/config.md @@ -2278,9 +2278,10 @@ A few important notes: `passive` node never leads and never becomes ready, so it stays out of the load balancer; use it for test clients only, since while it is cut off from the database, its clients can miss messages published elsewhere. * **Health checks:** point your load balancer at `/v1/ready`, not `/v1/health` (see [Health checks](#health-checks)). -* **Draining on shutdown:** when an active node gets `SIGTERM` (e.g. `systemctl restart ntfy` during a deploy) and - another active node is up, it drains first: it answers new subscriptions with 503, fails `/v1/ready`, and closes - its open subscriber connections evenly over `experimental-cluster-drain-duration` (default: 1 minute). Clients +* **Maintenance mode on shutdown:** when an active node gets `SIGTERM` (e.g. `systemctl restart ntfy` during a deploy) + and another active node is up, it enters maintenance mode and drains before it exits: it answers new subscriptions + with 503, fails `/v1/ready`, and closes its open subscriber connections evenly over + `experimental-cluster-drain-duration` (default: 1 minute). Clients reconnect a few at a time, through the load balancer, to the other nodes, and catch up on what they missed, instead of all reconnecting at once. Publishing and polling keep working while a node drains. Set your load balancer to retry on 503 (nginx: `proxy_next_upstream error timeout http_503;`), and give the service manager enough time to wait for @@ -2355,12 +2356,12 @@ few seconds, has failed for 30 seconds. For load balancers, use `/v1/ready` instead: it returns 200 only if the server should receive clients. Without clustering that is always the case. With clustering, a passive node is never ready, and an unhealthy active node is not ready while another active node can take its clients. The only active node stays ready even when it is unhealthy, since there is -nowhere better for its clients. A node that is [draining](#clustering-experimental) before a shutdown is not ready either, -and says so: +nowhere better for its clients. A node in [maintenance mode](#clustering-experimental) (e.g. while it drains before a +shutdown) is not ready either, and says so: ```json {"ready":true,"role":"active"} -{"ready":false,"role":"active","draining":true} +{"ready":false,"role":"active","maintenance":true} ``` See [Installation for Docker](install.md#docker) for an example of how this could be used in a `docker-compose` environment. @@ -2496,7 +2497,7 @@ variable before running the `ntfy` command (e.g. `export NTFY_LISTEN_HTTP=:80`). | `experimental-cluster-secret` | `NTFY_EXPERIMENTAL_CLUSTER_SECRET` | *string* | - | Experimental, not ready for production. Shared secret authenticating node-to-node traffic; must match on all nodes. | | `experimental-cluster-batch-linger` | `NTFY_EXPERIMENTAL_CLUSTER_BATCH_LINGER` | *duration* | `500ms` | Experimental, not ready for production. How long messages wait to form a node-to-node delivery batch; `0` sends immediately. | | `experimental-cluster-role` | `NTFY_EXPERIMENTAL_CLUSTER_ROLE` | `active` or `passive` | `active` | Experimental, not ready for production. A `passive` node never leads the cluster's singleton jobs and never closes its subscribers to send them to another node; other active nodes do not count it as a place for theirs. Use it for a node that is not behind the load balancer (check `/v1/ready`, see [Health checks](#health-checks)). While a passive node is cut off from the database, its clients can miss messages published on other nodes, so give it test clients only. | -| `experimental-cluster-drain-duration` | `NTFY_EXPERIMENTAL_CLUSTER_DRAIN_DURATION` | *duration* | `1m` | Experimental, not ready for production. On shutdown (`SIGTERM`), how long an active node takes to close its subscriber connections, so they reconnect to the other nodes gradually; `0` disables draining. A node drains only if another active node is up. The service manager's stop timeout must be longer. | +| `experimental-cluster-drain-duration` | `NTFY_EXPERIMENTAL_CLUSTER_DRAIN_DURATION` | *duration* | `1m` | Experimental, not ready for production. On shutdown (`SIGTERM`), an active node enters maintenance mode first; this is how long it takes to close its subscriber connections, so they reconnect to the other nodes gradually; `0` disables draining. A node drains only if another active node is up. The service manager's stop timeout must be longer. | | `database-replica-urls` | `NTFY_DATABASE_REPLICA_URLS` | *list of strings (connection URLs)* | - | PostgreSQL read replica connection strings. Non-critical read-only queries are distributed across replicas (round-robin) with automatic fallback to primary. Requires `database-url`. | | `cache-file` | `NTFY_CACHE_FILE` | *filename* | - | If set, messages are cached in a local SQLite database instead of only in-memory. This allows for service restarts without losing messages in support of the since= parameter. See [message cache](#message-cache). | | `cache-duration` | `NTFY_CACHE_DURATION` | *duration* | 12h | Duration for which messages will be buffered before they are deleted. This is required to support the `since=...` and `poll=1` parameter. Set this to `0` to disable the cache entirely. | diff --git a/docs/releases.md b/docs/releases.md index 85b68f62..bb3114d0 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2247,7 +2247,7 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release **Features:** * Allow several comma-separated addresses in `listen-http`, so a load balancer can hold more than ~63k connections to one ntfy node -* Cluster nodes drain on shutdown: an active node closes its subscriber connections gradually over `experimental-cluster-drain-duration` (default 1 minute), so its clients move to the other nodes without a reconnect storm (experimental, see [clustering](config.md#clustering-experimental)) +* Cluster nodes drain on shutdown: an active node enters maintenance mode and closes its subscriber connections gradually over `experimental-cluster-drain-duration` (default 1 minute), so its clients move to the other nodes without a reconnect storm (experimental, see [clustering](config.md#clustering-experimental)) ### ntfy iOS app v1.8.0 (UNRELEASED) diff --git a/server/errors.go b/server/errors.go index a4474487..841cf1ca 100644 --- a/server/errors.go +++ b/server/errors.go @@ -184,6 +184,6 @@ var ( errHTTPInternalErrorInvalidPath = &errHTTP{50002, http.StatusInternalServerError, "internal server error: invalid path", "", nil} errHTTPInternalErrorMissingBaseURL = &errHTTP{50003, http.StatusInternalServerError, "internal server error: base-url must be be configured for this feature", "https://ntfy.sh/docs/config/", nil} errHTTPInternalErrorWebPushUnableToPublish = &errHTTP{50004, http.StatusInternalServerError, "internal server error: unable to publish web push message", "", nil} - errHTTPServiceUnavailableDraining = &errHTTP{50301, http.StatusServiceUnavailable, "service unavailable: this node is shutting down, please reconnect", "", nil} + errHTTPServiceUnavailableMaintenance = &errHTTP{50301, http.StatusServiceUnavailable, "service unavailable: this node is in maintenance mode, please reconnect", "", nil} errHTTPInsufficientStorageUnifiedPush = &errHTTP{50701, http.StatusInsufficientStorage, "cannot publish to UnifiedPush topic without previously active subscriber", "", nil} ) diff --git a/server/server.go b/server/server.go index d121adb0..a17be0cf 100644 --- a/server/server.go +++ b/server/server.go @@ -81,7 +81,7 @@ type Server struct { tail *message.Tail // Reads what was written, see runTail; nil without a message cache stopOnce sync.Once stopped atomic.Bool // Set by Stop; read by Run, which must not start listeners afterwards - draining atomic.Bool // Set by Drain: new subscriptions get 503, and /v1/ready fails + maintenance atomic.Bool // Maintenance mode, set by Drain: new subscriptions get 503, and /v1/ready fails streams *streamRegistry // Open subscriber connections, for Drain mu sync.RWMutex } @@ -634,7 +634,7 @@ func (s *Server) handleError(w http.ResponseWriter, r *http.Request, v *visitor, metrics.HTTPRequests.WithLabelValues(strconv.Itoa(httpErr.HTTPCode), strconv.Itoa(httpErr.Code), r.Method).Inc() isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode) isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode) || - httpErr.Code == errHTTPServiceUnavailableDraining.Code // A draining node rejects every new subscription, by design + httpErr.Code == errHTTPServiceUnavailableMaintenance.Code // A node in maintenance mode rejects every new subscription, by design ev := logvr(v, r).Err(err) if websocket.IsWebSocketUpgrade(r) { ev.Tag(tagWebsocket).Fields(websocketErrorContext(err)) @@ -845,15 +845,15 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request, _ *visitor // handleReady answers whether a load balancer should send clients here (see cluster.Cluster's // Ready). A passive node is healthy (see handleHealth) but never ready. func (s *Server) handleReady(w http.ResponseWriter, _ *http.Request, _ *visitor) error { - draining := s.draining.Load() - ready := s.cluster.Ready() && !draining + maintenance := s.maintenance.Load() + ready := s.cluster.Ready() && !maintenance if !ready { w.WriteHeader(http.StatusServiceUnavailable) } return s.writeJSON(w, &apiReadyResponse{ - Ready: ready, - Role: s.cluster.Role(), - Draining: draining, + Ready: ready, + Role: s.cluster.Role(), + Maintenance: maintenance, }) } @@ -1582,8 +1582,8 @@ func (s *Server) handleSubscribeHTTP(w http.ResponseWriter, r *http.Request, v * poll, since, scheduled, filters, err := parseSubscribeParams(r) if err != nil { return err - } else if !poll && s.draining.Load() { - return errHTTPServiceUnavailableDraining + } else if !poll && s.maintenance.Load() { + return errHTTPServiceUnavailableMaintenance } var wlock sync.Mutex var closed bool @@ -1701,8 +1701,8 @@ func (s *Server) handleSubscribeWS(w http.ResponseWriter, r *http.Request, v *vi poll, since, scheduled, filters, err := parseSubscribeParams(r) if err != nil { return err - } else if !poll && s.draining.Load() { - return errHTTPServiceUnavailableDraining // Before the upgrade, so the load balancer can retry elsewhere + } else if !poll && s.maintenance.Load() { + return errHTTPServiceUnavailableMaintenance // Before the upgrade, so the load balancer can retry elsewhere } upgrader := &websocket.Upgrader{ ReadBufferSize: wsBufferSize, diff --git a/server/server.yml b/server/server.yml index f75c4ca8..4223f874 100644 --- a/server/server.yml +++ b/server/server.yml @@ -89,9 +89,10 @@ # balancer agents can route to active nodes only. # - experimental-cluster-drain-duration is how long an active node takes, on shutdown (SIGTERM), # to close its subscriber connections, so they reconnect to the other nodes gradually instead -# of all at once. While draining, new subscriptions get 503 and /v1/ready fails, so the load -# balancer must retry on 503. A node drains only if another active node is up. Make sure the -# service manager waits long enough (systemd: TimeoutStopSec, default 90s). Set to 0 to disable. +# of all at once. It is in maintenance mode meanwhile: new subscriptions get 503 and /v1/ready +# fails, so the load balancer must retry on 503. A node drains only if another active node is +# up. Make sure the service manager waits long enough (systemd: TimeoutStopSec, default 90s). +# Set to 0 to disable. # # Clustering also requires "database-url": the shared PostgreSQL database carries the durable # state and the cluster's own coordination tables. diff --git a/server/server_drain.go b/server/server_maintenance.go similarity index 70% rename from server/server_drain.go rename to server/server_maintenance.go index fa059fc2..b4b60c3c 100644 --- a/server/server_drain.go +++ b/server/server_maintenance.go @@ -10,7 +10,7 @@ import ( ) const ( - tagDrain = "drain" + tagMaintenance = "maintenance" // drainTickInterval is how often a drain closes its next share of streams; small enough that // the closes look continuous to the load balancers, rather than bursts @@ -75,11 +75,12 @@ func (r *streamRegistry) CancelSome(n int) int { return canceled } -// Drain moves this node's clients to the other nodes before a shutdown: it stops taking new -// subscriptions (503, which a load balancer retries on another node; /v1/ready fails too) and -// closes the open streams evenly over ClusterDrainDuration, so their clients reconnect to the -// other nodes a few at a time instead of all at once. Clients resume with since=, so nothing is -// lost. It returns when no streams are left, or right away when ctx is canceled. +// Drain puts this node into maintenance mode and moves its clients to the other nodes: it stops +// taking new subscriptions (503, which a load balancer retries on another node; /v1/ready fails +// too) and closes the open streams evenly over ClusterDrainDuration, so their clients reconnect to +// the other nodes a few at a time instead of all at once. Clients resume with since=, so nothing is +// lost. Publishes, polls and the cluster mesh keep working. It returns when no streams are left, or +// right away when ctx is canceled; the node stays in maintenance mode. A shutdown calls it first. // // It drains only an active node with at least one other healthy active node: otherwise there is // nowhere for the clients to go, and a drain would only delay the shutdown. @@ -87,10 +88,10 @@ func (s *Server) Drain(ctx context.Context) { if !s.shouldDrain() { return } - s.draining.Store(true) + s.maintenance.Store(true) total := s.streams.Len() duration := s.config.ClusterDrainDuration - log.Tag(tagDrain).Info("Draining: closing %d stream(s) over %v, new subscriptions are rejected", total, duration) + log.Tag(tagMaintenance).Info("Entering maintenance mode: closing %d stream(s) over %v, new subscriptions are rejected", total, duration) start, lastLog := time.Now(), time.Now() closed := 0 ticker := time.NewTicker(drainTickInterval) @@ -98,12 +99,12 @@ func (s *Server) Drain(ctx context.Context) { for s.streams.Len() > 0 { select { case <-ctx.Done(): - log.Tag(tagDrain).Info("Draining canceled, %d stream(s) left", s.streams.Len()) + log.Tag(tagMaintenance).Info("Drain canceled, %d stream(s) left", s.streams.Len()) return case <-ticker.C: } // Close the share of the streams that is due by now. Streams that opened right as the drain - // started (past the draining check, but not counted in total) go at the end. + // started (past the maintenance check, but not counted in total) go at the end. elapsed := time.Since(start) due := int(float64(total)*elapsed.Seconds()/duration.Seconds()) - closed if elapsed >= duration { @@ -113,11 +114,11 @@ func (s *Server) Drain(ctx context.Context) { closed += s.streams.CancelSome(due) } if time.Since(lastLog) >= drainLogInterval { - log.Tag(tagDrain).Info("Draining: closed %d stream(s), %d left", closed, s.streams.Len()) + log.Tag(tagMaintenance).Info("Maintenance mode: closed %d stream(s), %d left", closed, s.streams.Len()) lastLog = time.Now() } } - log.Tag(tagDrain).Info("Drained: closed %d stream(s) in %v", closed, time.Since(start).Round(time.Millisecond)) + log.Tag(tagMaintenance).Info("Maintenance mode: drained, closed %d stream(s) in %v", closed, time.Since(start).Round(time.Millisecond)) } func (s *Server) shouldDrain() bool { @@ -129,6 +130,6 @@ func (s *Server) shouldDrain() bool { return true } } - log.Tag(tagDrain).Info("Not draining: no other active node to take this node's clients") + log.Tag(tagMaintenance).Info("Not entering maintenance mode: no other active node to take this node's clients") return false } diff --git a/server/server_drain_test.go b/server/server_maintenance_test.go similarity index 85% rename from server/server_drain_test.go rename to server/server_maintenance_test.go index bc3f9fe7..f7323756 100644 --- a/server/server_drain_test.go +++ b/server/server_maintenance_test.go @@ -11,7 +11,7 @@ import ( "heckel.io/ntfy/v2/cluster" ) -func TestServer_Drain_ClosesStreamsGradually(t *testing.T) { +func TestServer_Maintenance_ClosesStreamsGradually(t *testing.T) { s, _ := newDrainTestServer(t, 2*time.Second, drainTestMember("node-1", cluster.RoleActive), drainTestMember("node-2", cluster.RoleActive)) streams := make([]<-chan struct{}, 0) for i := 0; i < 20; i++ { @@ -46,8 +46,8 @@ func TestServer_Drain_ClosesStreamsGradually(t *testing.T) { require.Equal(t, 0, s.streams.Len()) } -func TestServer_Drain_RejectsNewSubscriptionsOnly(t *testing.T) { - // While draining, new streams get 503 (the load balancer retries them on another node), but +func TestServer_Maintenance_RejectsNewSubscriptionsOnly(t *testing.T) { + // In maintenance mode, new streams get 503 (the load balancer retries them on another node), but // publishes and polls are short and still served: a load balancer does not retry a POST s, _ := newDrainTestServer(t, 3*time.Second, drainTestMember("node-1", cluster.RoleActive), drainTestMember("node-2", cluster.RoleActive)) openDrainTestStream(s, "/othertopic/json") // Keeps the drain going for the whole test @@ -58,12 +58,12 @@ func TestServer_Drain_RejectsNewSubscriptionsOnly(t *testing.T) { defer cancel() go s.Drain(ctx) waitFor(t, func() bool { - return s.draining.Load() + return s.maintenance.Load() }) rr := request(t, s, "GET", "/mytopic/json", "", nil) require.Equal(t, 503, rr.Code) - require.Equal(t, errHTTPServiceUnavailableDraining.Code, toHTTPError(t, rr.Body.String()).Code) + require.Equal(t, errHTTPServiceUnavailableMaintenance.Code, toHTTPError(t, rr.Body.String()).Code) rr = request(t, s, "GET", "/mytopic/ws", "", map[string]string{ "Upgrade": "websocket", }) @@ -77,10 +77,10 @@ func TestServer_Drain_RejectsNewSubscriptionsOnly(t *testing.T) { rr = request(t, s, "GET", "/v1/ready", "", nil) require.Equal(t, 503, rr.Code) - require.JSONEq(t, `{"ready":false,"role":"active","draining":true}`, rr.Body.String()) + require.JSONEq(t, `{"ready":false,"role":"active","maintenance":true}`, rr.Body.String()) } -func TestServer_Drain_SkippedWithoutAnotherActiveNode(t *testing.T) { +func TestServer_Maintenance_SkippedWithoutAnotherActiveNode(t *testing.T) { // Draining the last active node would send its clients nowhere: a passive peer does not count s, _ := newDrainTestServer(t, 2*time.Second, drainTestMember("node-1", cluster.RoleActive), drainTestMember("node-2", cluster.RolePassive)) stream := openDrainTestStream(s, "/mytopic/json") @@ -90,22 +90,22 @@ func TestServer_Drain_SkippedWithoutAnotherActiveNode(t *testing.T) { start := time.Now() s.Drain(context.Background()) require.Less(t, time.Since(start), 500*time.Millisecond) - require.False(t, s.draining.Load()) + require.False(t, s.maintenance.Load()) require.Equal(t, 0, countClosed([]<-chan struct{}{stream})) } -func TestServer_Drain_SkippedWithZeroDuration(t *testing.T) { +func TestServer_Maintenance_SkippedWithZeroDuration(t *testing.T) { s, _ := newDrainTestServer(t, 0, drainTestMember("node-1", cluster.RoleActive), drainTestMember("node-2", cluster.RoleActive)) stream := openDrainTestStream(s, "/mytopic/json") waitFor(t, func() bool { return s.streams.Len() == 1 }) s.Drain(context.Background()) - require.False(t, s.draining.Load()) + require.False(t, s.maintenance.Load()) require.Equal(t, 0, countClosed([]<-chan struct{}{stream})) } -func TestServer_Drain_SkippedOnPassiveNode(t *testing.T) { +func TestServer_Maintenance_SkippedOnPassiveNode(t *testing.T) { s, cl := newDrainTestServer(t, 2*time.Second, drainTestMember("node-1", cluster.RolePassive), drainTestMember("node-2", cluster.RoleActive)) cl.mu.Lock() cl.role = cluster.RolePassive @@ -115,10 +115,10 @@ func TestServer_Drain_SkippedOnPassiveNode(t *testing.T) { return s.streams.Len() == 1 }) s.Drain(context.Background()) - require.False(t, s.draining.Load()) + require.False(t, s.maintenance.Load()) } -func TestServer_Drain_StopsWhenCanceled(t *testing.T) { +func TestServer_Maintenance_StopsWhenCanceled(t *testing.T) { // A second SIGTERM cancels the drain, and the shutdown goes on at once s, _ := newDrainTestServer(t, time.Minute, drainTestMember("node-1", cluster.RoleActive), drainTestMember("node-2", cluster.RoleActive)) openDrainTestStream(s, "/mytopic/json") @@ -132,7 +132,7 @@ func TestServer_Drain_StopsWhenCanceled(t *testing.T) { close(drained) }() waitFor(t, func() bool { - return s.draining.Load() + return s.maintenance.Load() }) cancel() select { @@ -142,7 +142,7 @@ func TestServer_Drain_StopsWhenCanceled(t *testing.T) { } } -func TestServer_Drain_StreamsCountsConnectionsNotTopics(t *testing.T) { +func TestServer_Maintenance_StreamsCountsConnectionsNotTopics(t *testing.T) { // One connection subscribed to three topics is one stream to close, not three s, _ := newDrainTestServer(t, time.Second) streamHTTP := openDrainTestStream(s, "/a,b,c/json") diff --git a/server/types.go b/server/types.go index a779db3f..5a269729 100644 --- a/server/types.go +++ b/server/types.go @@ -151,9 +151,9 @@ type apiHealthResponse struct { } type apiReadyResponse struct { - Ready bool `json:"ready"` - Role cluster.Role `json:"role"` - Draining bool `json:"draining,omitempty"` + Ready bool `json:"ready"` + Role cluster.Role `json:"role"` + Maintenance bool `json:"maintenance,omitempty"` } type apiVersionResponse struct {