diff --git a/src/server/dflycmd.cc b/src/server/dflycmd.cc index 09d5bd800..6adc47f36 100644 --- a/src/server/dflycmd.cc +++ b/src/server/dflycmd.cc @@ -719,10 +719,10 @@ void DflyCmd::GetReplicationMemoryStats(ReplicationMemoryStats* stats) const { const auto& flow = info->flows[shard->shard_id()]; if (flow.streamer) - stats->streamer_buf_capacity_bytes_ += flow.streamer->GetTotalBufferCapacities(); + stats->streamer_buf_capacity_bytes += flow.streamer->GetTotalBufferCapacities(); if (flow.saver) - stats->full_sync_buf_bytes_ += flow.saver->GetTotalBuffersSize(); + stats->full_sync_buf_bytes += flow.saver->GetTotalBuffersSize(); } }; shard_set->RunBlockingInParallel(cb); diff --git a/src/server/server_family.cc b/src/server/server_family.cc index dca042051..adbb09b36 100644 --- a/src/server/server_family.cc +++ b/src/server/server_family.cc @@ -1051,7 +1051,7 @@ void AppendMetricWithoutLabels(string_view name, string_view help, const absl::A AppendMetricValue(name, value, {}, {}, dest); } -void PrintPrometheusMetrics(const Metrics& m, StringResponse* resp) { +void PrintPrometheusMetrics(const Metrics& m, DflyCmd* dfly_cmd, StringResponse* resp) { // Server metrics AppendMetricHeader("version", "", MetricType::GAUGE, &resp->body()); AppendMetricValue("version", 1, {"version"}, {GetVersion()}, &resp->body()); @@ -1130,6 +1130,15 @@ void PrintPrometheusMetrics(const Metrics& m, StringResponse* resp) { if (added) absl::StrAppend(&resp->body(), type_used_memory_metric); } + if (!m.replication_metrics.empty()) { + ReplicationMemoryStats repl_mem; + dfly_cmd->GetReplicationMemoryStats(&repl_mem); + AppendMetricWithoutLabels( + "replication_streaming_bytes", "Stable sync replication memory usage", + repl_mem.streamer_buf_capacity_bytes, MetricType::GAUGE, &resp->body()); + AppendMetricWithoutLabels("replication_full_sync_bytes", "Full sync memory usage", + repl_mem.full_sync_buf_bytes, MetricType::GAUGE, &resp->body()); + } // Stats metrics AppendMetricWithoutLabels("connections_received_total", "", conn_stats.conn_received_cnt, @@ -1262,7 +1271,7 @@ void ServerFamily::ConfigureMetrics(util::HttpListenerBase* http_base) { auto cb = [this](const util::http::QueryArgs& args, util::HttpContext* send) { StringResponse resp = util::http::MakeStringResponse(boost::beast::http::status::ok); - PrintPrometheusMetrics(this->GetMetrics(), &resp); + PrintPrometheusMetrics(this->GetMetrics(), this->dfly_cmd_.get(), &resp); return send->Invoke(std::move(resp)); }; @@ -2027,8 +2036,8 @@ void ServerFamily::Info(CmdArgList args, ConnectionContext* cntx) { if (!m.replication_metrics.empty()) { ReplicationMemoryStats repl_mem; dfly_cmd_->GetReplicationMemoryStats(&repl_mem); - append("replication_streaming_buffer_bytes", repl_mem.streamer_buf_capacity_bytes_); - append("replication_full_sync_buffer_bytes", repl_mem.full_sync_buf_bytes_); + append("replication_streaming_buffer_bytes", repl_mem.streamer_buf_capacity_bytes); + append("replication_full_sync_buffer_bytes", repl_mem.full_sync_buf_bytes); } { diff --git a/src/server/server_family.h b/src/server/server_family.h index a064b4afb..2f8a151c8 100644 --- a/src/server/server_family.h +++ b/src/server/server_family.h @@ -62,8 +62,8 @@ struct ReplicaRoleInfo { }; struct ReplicationMemoryStats { - size_t streamer_buf_capacity_bytes_ = 0; // total capacities of streamer buffers - size_t full_sync_buf_bytes_ = 0; // total bytes used for full sync buffers + size_t streamer_buf_capacity_bytes = 0; // total capacities of streamer buffers + size_t full_sync_buf_bytes = 0; // total bytes used for full sync buffers }; // Global peak stats recorded after aggregating metrics over all shards. diff --git a/tools/local/monitoring/docker-compose.yml b/tools/local/monitoring/docker-compose.yml index 7202cff5b..eaf09ec0d 100644 --- a/tools/local/monitoring/docker-compose.yml +++ b/tools/local/monitoring/docker-compose.yml @@ -41,7 +41,7 @@ services: mode: global grafana: - image: grafana/grafana + image: grafana/grafana:10.1.10 user: '472' restart: always environment: diff --git a/tools/local/monitoring/grafana/provisioning/dashboards/dashboard.json b/tools/local/monitoring/grafana/provisioning/dashboards/dashboard.json index 3a7575795..abc86a5f6 100644 --- a/tools/local/monitoring/grafana/provisioning/dashboards/dashboard.json +++ b/tools/local/monitoring/grafana/provisioning/dashboards/dashboard.json @@ -495,6 +495,19 @@ "legendFormat": "max", "refId": "B", "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "dragonfly_used_memory_rss_bytes{instance=~\"$instance\"} ", + "range": true, + "format": "time_series", + "hide": false, + "legendFormat": "RSS", + "refId": "C" } ], "thresholds": [],