feat(observability): add local grafana+prom stack for metrics insights

observability/tempo/tempo.yaml

@@ -0,0 +1,36 @@
+server:
+  http_listen_port: 3101
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+
+ingester:
+  trace_idle_period: 10s # the length of time after a trace has not received spans to consider it complete and flush it
+  max_block_bytes: 1_000_000 # cut the head block when it hits this size or ...
+  max_block_duration: 5m #   this much time passes
+
+compactor:
+  compaction:
+    compaction_window: 1h # blocks in this time window will be compacted together
+    max_block_bytes: 100_000_000 # maximum size of compacted blocks
+    block_retention: 1h
+    compacted_block_retention: 10m
+
+storage:
+  trace:
+    backend: local # backend configuration to use
+    block:
+      bloom_filter_false_positive: .05 # bloom filter false positive rate.  lower values create larger filters but fewer false positives
+      index_downsample_bytes: 1000 # number of bytes per index record
+      encoding: zstd # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+    wal:
+      path: /tmp/tempo/wal # where to store the the wal locally
+      encoding: none # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+    local:
+      path: /tmp/tempo/blocks
+    pool:
+      max_workers: 100 # the worker pool mainly drives querying, but is also used for polling the blocklist
+      queue_depth: 10000