From e66a925d64606cc21533eb361380ab691c078e42 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 13 Jan 2024 11:47:37 +0700 Subject: [PATCH] metrics: add a basic setup, collect metrics for the server --- go.mod | 8 +- go.sum | 25 ++++- integrationtests/gomodvendor/go.sum | 4 +- metrics/dashboards/README.md | 25 +++++ metrics/dashboards/datasources.yml | 13 +++ metrics/dashboards/docker-compose.yml | 25 +++++ metrics/dashboards/prometheus.yml | 9 ++ metrics/pool.go | 27 ++++++ metrics/tracer.go | 133 ++++++++++++++++++++++++++ 9 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 metrics/dashboards/README.md create mode 100644 metrics/dashboards/datasources.yml create mode 100644 metrics/dashboards/docker-compose.yml create mode 100644 metrics/dashboards/prometheus.yml create mode 100644 metrics/pool.go create mode 100644 metrics/tracer.go diff --git a/go.mod b/go.mod index 00abc69a4..8cc39ebd2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/francoispqt/gojay v1.2.13 github.com/onsi/ginkgo/v2 v2.9.5 github.com/onsi/gomega v1.27.6 + github.com/prometheus/client_golang v1.19.1 github.com/quic-go/qpack v0.4.0 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.23.0 @@ -17,13 +18,18 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8fbf1c8a5..fda9cbfcc 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,12 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -61,8 +65,9 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -84,11 +89,21 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -199,11 +214,11 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/integrationtests/gomodvendor/go.sum b/integrationtests/gomodvendor/go.sum index 260b2088e..6d99bc8b7 100644 --- a/integrationtests/gomodvendor/go.sum +++ b/integrationtests/gomodvendor/go.sum @@ -47,8 +47,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/metrics/dashboards/README.md b/metrics/dashboards/README.md new file mode 100644 index 000000000..9d1a68d1e --- /dev/null +++ b/metrics/dashboards/README.md @@ -0,0 +1,25 @@ +# quic-go Prometheus / Grafana setup + +Expose a Grafana endpoint on `http://localhost:5001/prometheus`: +```go +import "github.com/prometheus/client_golang/prometheus/promhttp" + +go func() { + http.Handle("/prometheus", promhttp.Handler()) + log.Fatal(http.ListenAndServe(":5001", nil)) +}() +``` + +Set a metrics tracer on the `Transport`: +```go +quic.Transport{ + Tracer: metrics.NewTracer(), +} +``` + +When using multiple `Transport`s, it is recommended to use the metrics tracer struct for all of them. + +Running: +```shell +docker-compose up +``` diff --git a/metrics/dashboards/datasources.yml b/metrics/dashboards/datasources.yml new file mode 100644 index 000000000..ed47ec11a --- /dev/null +++ b/metrics/dashboards/datasources.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + +datasources: + - name: Prometheus + orgId: 1 + type: prometheus + access: proxy + url: http://prometheus:9090 + editable: false diff --git a/metrics/dashboards/docker-compose.yml b/metrics/dashboards/docker-compose.yml new file mode 100644 index 000000000..8e3a8a55c --- /dev/null +++ b/metrics/dashboards/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.8' + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + expose: + - 9090 + grafana: + image: grafana/grafana:latest + container_name: grafana + volumes: + - grafana_data:/var/lib/grafana + - ./datasources.yml:/etc/grafana/provisioning/datasources/prom.yml + ports: + - "3000:3000" diff --git a/metrics/dashboards/prometheus.yml b/metrics/dashboards/prometheus.yml new file mode 100644 index 000000000..5018097ec --- /dev/null +++ b/metrics/dashboards/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'quic-go' + scrape_interval: 15s + static_configs: + - targets: ['host.docker.internal:5001'] + metrics_path: '/prometheus' diff --git a/metrics/pool.go b/metrics/pool.go new file mode 100644 index 000000000..75868beae --- /dev/null +++ b/metrics/pool.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "fmt" + "sync" +) + +const capacity = 4 + +// The stringPool is used to avoid allocations when passing labels to Prometheus. +var stringPool = sync.Pool{New: func() any { + s := make([]string, 0, capacity) + return &s +}} + +func getStringSlice() *[]string { + s := stringPool.Get().(*[]string) + *s = (*s)[:0] + return s +} + +func putStringSlice(s *[]string) { + if c := cap(*s); c < capacity { + panic(fmt.Sprintf("unexpected slice cap: %d", c)) + } + stringPool.Put(s) +} diff --git a/metrics/tracer.go b/metrics/tracer.go new file mode 100644 index 000000000..e069db727 --- /dev/null +++ b/metrics/tracer.go @@ -0,0 +1,133 @@ +package metrics + +import ( + "errors" + "fmt" + "net" + + "github.com/quic-go/quic-go/internal/protocol" + "github.com/quic-go/quic-go/internal/qerr" + "github.com/quic-go/quic-go/logging" + + "github.com/prometheus/client_golang/prometheus" +) + +const metricNamespace = "quicgo" + +func getIPVersion(addr net.Addr) string { + udpAddr, ok := addr.(*net.UDPAddr) + if !ok { + return "" + } + if udpAddr.IP.To4() != nil { + return "ipv4" + } + return "ipv6" +} + +var ( + connsRejected = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "server_connections_rejected_total", + Help: "Connections Rejected", + }, + []string{"ip_version", "reason"}, + ) + packetDropped = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "server_received_packets_dropped_total", + Help: "packets dropped", + }, + []string{"ip_version", "reason"}, + ) +) + +func NewTracer() *logging.Tracer { + for _, c := range [...]prometheus.Collector{ + connsRejected, + packetDropped, + } { + if err := prometheus.DefaultRegisterer.Register(c); err != nil { + if ok := errors.As(err, &prometheus.AlreadyRegisteredError{}); !ok { + panic(err) + } + } + } + + return &logging.Tracer{ + SentPacket: func(addr net.Addr, hdr *logging.Header, _ logging.ByteCount, frames []logging.Frame) { + tags := getStringSlice() + defer putStringSlice(tags) + + var reason string + switch { + case hdr.Type == protocol.PacketTypeRetry: + reason = "retry" + case hdr.Type == protocol.PacketTypeInitial: + var ccf *logging.ConnectionCloseFrame + for _, f := range frames { + cc, ok := f.(*logging.ConnectionCloseFrame) + if ok { + ccf = cc + break + } + } + if ccf.IsApplicationError { + //nolint:exhaustive // Only a few error codes applicable. + switch qerr.TransportErrorCode(ccf.ErrorCode) { + case qerr.ConnectionRefused: + reason = "connection_refused" + case qerr.InvalidToken: + reason = "invalid_token" + default: + // This shouldn't happen, the server doesn't send CONNECTION_CLOSE frames with different errors. + reason = fmt.Sprintf("transport_error: %d", ccf.ErrorCode) + } + } else { + // This shouldn't happen, the server doesn't send application-level CONNECTION_CLOSE frames. + reason = "application_error" + } + } + *tags = append(*tags, getIPVersion(addr)) + *tags = append(*tags, reason) + connsRejected.WithLabelValues(*tags...).Inc() + }, + SentVersionNegotiationPacket: func(addr net.Addr, _, _ logging.ArbitraryLenConnectionID, _ []logging.VersionNumber) { + tags := getStringSlice() + defer putStringSlice(tags) + + *tags = append(*tags, getIPVersion(addr)) + *tags = append(*tags, "version_negotiation") + connsRejected.WithLabelValues(*tags...).Inc() + }, + DroppedPacket: func(addr net.Addr, pt logging.PacketType, _ logging.ByteCount, reason logging.PacketDropReason) { + tags := getStringSlice() + defer putStringSlice(tags) + + var dropReason string + //nolint:exhaustive // Only a few drop reasons applicable. + switch reason { + case logging.PacketDropDOSPrevention: + if pt == logging.PacketType0RTT { + dropReason = "0rtt_dos_prevention" + } else { + dropReason = "dos_prevention" + } + case logging.PacketDropHeaderParseError: + dropReason = "header_parsing" + case logging.PacketDropPayloadDecryptError: + dropReason = "payload_decrypt" + case logging.PacketDropUnexpectedPacket: + dropReason = "unexpected_packet" + default: + dropReason = "unknown" + } + + *tags = append(*tags, getIPVersion(addr)) + *tags = append(*tags, dropReason) + packetDropped.WithLabelValues(*tags...).Inc() + }, + } +}