diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md new file mode 100644 index 0000000000..7d911b7399 --- /dev/null +++ b/.planning/PROJECT.md @@ -0,0 +1,56 @@ +# Add Start Time (ST) to Histogram WAL Records + +## What This Is + +Extend the Prometheus TSDB WAL record encoding layer to support per-sample start time (ST) for histogram types. This mirrors the existing ST support already implemented for float samples (RefSample / SamplesV2). + +## Core Value + +Histogram samples carry accurate start time metadata through the WAL, enabling correct staleness and reset detection downstream. + +## Requirements + +### Validated + +- RefSample already has ST field with V1/V2 encoding (SamplesV2, type 11) + +### Active + +- [ ] Add ST field to RefHistogramSample struct +- [ ] Add ST field to RefFloatHistogramSample struct +- [ ] New V2 record types for all four histogram variants +- [ ] V2 encoder/decoder using same ST marker scheme as samplesV2 +- [ ] Backward-compatible: V1 decoder still works, EnableSTStorage gates V2 + +### Out of Scope + +- WAL replay changes. Record layer only. +- Remote write integration. Separate follow-up. +- Head append path changes. Separate follow-up. +- Any non-record-layer changes. + +## Context + +- `tsdb/record/record.go` is the primary file +- Existing pattern: `Encoder.Samples()` checks `EnableSTStorage` to pick V1 vs V2 +- V2 encoding uses marker bytes: `noST` (0), `sameST` (1), `explicitST` (2) +- Four histogram encoder functions exist: `HistogramSamples`, `CustomBucketsHistogramSamples`, `FloatHistogramSamples`, `CustomBucketsFloatHistogramSamples` +- Each has a corresponding decoder in `Decoder.HistogramSamples` and `Decoder.FloatHistogramSamples` +- Current histogram encoding: `[type(1)] [baseRef(8)] [baseTime(8)] [dRef(varint) dTime(varint) histogramPayload]...` + +## Constraints + +- **Backward compat**: Old decoders must still read V1 records. New decoders must read both V1 and V2. +- **Gating**: V2 encoding gated behind `EnableSTStorage` flag, same as float samples. +- **Pattern consistency**: Must follow the exact same ST marker scheme used in samplesV2. + +## Key Decisions + +| Decision | Rationale | Outcome | +|----------|-----------|---------| +| New record types (V2) rather than in-place changes | Mirrors Samples/SamplesV2 pattern, clean backward compat | -- Pending | +| All four histogram variants get V2 | Consistent coverage, avoids partial support | -- Pending | +| Both RefHistogramSample and RefFloatHistogramSample get ST field | All four record types need the struct field to encode ST | -- Pending | + +--- +*Last updated: 2026-03-02 after initial project setup* diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md new file mode 100644 index 0000000000..d814611816 --- /dev/null +++ b/.planning/REQUIREMENTS.md @@ -0,0 +1,90 @@ +# Requirements: Histogram ST WAL Records + +**Defined:** 2026-03-02 +**Core Value:** Histogram samples carry accurate start time through the WAL + +## v1 Requirements + +### Struct Changes + +- [x] **STRUCT-01**: RefHistogramSample has ST field (int64) alongside T +- [x] **STRUCT-02**: RefFloatHistogramSample has ST field (int64) alongside T + +### Record Types + +- [x] **TYPE-01**: HistogramSamplesV2 record type constant added +- [x] **TYPE-02**: FloatHistogramSamplesV2 record type constant added +- [x] **TYPE-03**: CustomBucketsHistogramSamplesV2 record type constant added +- [x] **TYPE-04**: CustomBucketsFloatHistogramSamplesV2 record type constant added +- [x] **TYPE-05**: Type.String() returns correct names for new types + +### Encoding + +- [x] **ENC-01**: Encoder.HistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled +- [x] **ENC-02**: Encoder.FloatHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled +- [x] **ENC-03**: Encoder.CustomBucketsHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled +- [x] **ENC-04**: Encoder.CustomBucketsFloatHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled +- [x] **ENC-05**: V2 histogram encoding uses noST/sameST/explicitST marker scheme + +### Decoding + +- [x] **DEC-01**: Decoder.HistogramSamples() accepts both V1 and V2 record types +- [x] **DEC-02**: Decoder.FloatHistogramSamples() accepts both V1 and V2 record types +- [x] **DEC-03**: V2 histogram decoding correctly reads ST marker bytes and reconstructs ST values +- [x] **DEC-04**: V1 records decoded with ST=0 (backward compat) + +### Testing + +- [x] **TEST-01**: Round-trip encode/decode for histogram V2 with no ST +- [x] **TEST-02**: Round-trip encode/decode for histogram V2 with constant ST +- [x] **TEST-03**: Round-trip encode/decode for histogram V2 with varying ST +- [x] **TEST-04**: Round-trip encode/decode for float histogram V2 (same ST scenarios) +- [x] **TEST-05**: Round-trip encode/decode for custom buckets variants V2 +- [x] **TEST-06**: V1 records still decode correctly (backward compat test) +- [x] **TEST-07**: Type() correctly identifies new record types + +## Out of Scope + +| Feature | Reason | +|---------|--------| +| WAL replay changes | Record layer only per scope decision | +| Remote write integration | Separate follow-up | +| Head append path | Separate follow-up | +| Exemplar ST support | Not requested | + +## Traceability + +| Requirement | Phase | Status | +|-------------|-------|--------| +| STRUCT-01 | Phase 1 | Complete | +| STRUCT-02 | Phase 1 | Complete | +| TYPE-01 | Phase 1 | Complete | +| TYPE-02 | Phase 1 | Complete | +| TYPE-03 | Phase 1 | Complete | +| TYPE-04 | Phase 1 | Complete | +| TYPE-05 | Phase 1 | Complete | +| ENC-01 | Phase 2 | Complete | +| ENC-02 | Phase 2 | Complete | +| ENC-03 | Phase 2 | Complete | +| ENC-04 | Phase 2 | Complete | +| ENC-05 | Phase 2 | Complete | +| DEC-01 | Phase 3 | Complete | +| DEC-02 | Phase 3 | Complete | +| DEC-03 | Phase 3 | Complete | +| DEC-04 | Phase 3 | Complete | +| TEST-01 | Phase 4 | Complete | +| TEST-02 | Phase 4 | Complete | +| TEST-03 | Phase 4 | Complete | +| TEST-04 | Phase 4 | Complete | +| TEST-05 | Phase 4 | Complete | +| TEST-06 | Phase 4 | Complete | +| TEST-07 | Phase 4 | Complete | + +**Coverage:** +- v1 requirements: 23 total +- Mapped to phases: 23 +- Unmapped: 0 + +--- +*Requirements defined: 2026-03-02* +*Last updated: 2026-03-02 after Phase 3 Plan 02 complete* diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md new file mode 100644 index 0000000000..7edce03482 --- /dev/null +++ b/.planning/ROADMAP.md @@ -0,0 +1,89 @@ +# Roadmap: Histogram ST WAL Records + +**Created:** 2026-03-02 +**Phases:** 4 +**Scope:** tsdb/record/ only (record layer) + +## Phase 1: Struct and Type Definitions + +**Goal:** All types and structs exist so encoding/decoding can reference them. + +**Requirements:** STRUCT-01, STRUCT-02, TYPE-01..05 + +**Plans:** 1/1 plans complete + +Plans: +- [x] 01-01-PLAN.md -- Add ST fields to histogram structs and declare V2 record type constants + +**Files:** `tsdb/record/record.go` + +**Verification:** Code compiles. `go vet ./tsdb/record/...` passes. + +--- + +## Phase 2: V2 Encoders + +**Goal:** Encoder can produce V2 histogram records with ST when EnableSTStorage is true. + +**Requirements:** ENC-01..05 + +**Plans:** 2/2 plans complete + +Plans: +- [x] 02-01-PLAN.md -- Int-histogram V2 encoders (HistogramSamples + CustomBucketsHistogramSamples dispatch and V2 methods) +- [x] 02-02-PLAN.md -- Float-histogram V2 encoders (FloatHistogramSamples + CustomBucketsFloatHistogramSamples dispatch and V2 methods) + +**Files:** `tsdb/record/record.go` + +**Verification:** Code compiles. Encoder produces valid V2 byte sequences. + +--- + +## Phase 3: V2 Decoders + +**Goal:** Decoder reads both V1 and V2 histogram records correctly. + +**Requirements:** DEC-01..04 + +**Plans:** 2/2 plans complete + +Plans: +- [x] 03-01-PLAN.md -- Int-histogram V2 decoder (histogramSamplesV2 + HistogramSamples dispatch) +- [x] 03-02-PLAN.md -- Float-histogram V2 decoder (floatHistogramSamplesV2 + FloatHistogramSamples dispatch) + +**Files:** `tsdb/record/record.go` + +**Verification:** Code compiles. Decoder correctly round-trips V2 records. + +--- + +## Phase 4: Tests + +**Goal:** Full test coverage for V2 histogram encoding, backward compat verified. + +**Requirements:** TEST-01..07 + +**Plans:** 2 plans + +Plans: +- [ ] 04-01-PLAN.md -- V2 histogram round-trip tests (all ST scenarios, custom-bucket, gauge, backward compat) +- [ ] 04-02-PLAN.md -- V2 histogram type recognition assertions in TestRecord_Type + +**Files:** `tsdb/record/record_test.go` + +**Verification:** `go test ./tsdb/record/... -count=1` passes. All new test cases green. + +--- + +## Phase Dependencies + +``` +Phase 1 (types/structs) + └─> Phase 2 (encoders) + └─> Phase 3 (decoders) [depends on Phase 2 for round-trip verification] + └─> Phase 4 (tests) [depends on Phase 2 + Phase 3] +``` + +--- +*Created: 2026-03-02* +*Last updated: 2026-03-03 after Phase 4 planning complete* diff --git a/.planning/STATE.md b/.planning/STATE.md new file mode 100644 index 0000000000..d7e139a11a --- /dev/null +++ b/.planning/STATE.md @@ -0,0 +1,54 @@ +--- +gsd_state_version: 1.0 +milestone: v1.0 +milestone_name: milestone +status: unknown +last_updated: "2026-03-03T14:24:27.737Z" +progress: + total_phases: 4 + completed_phases: 4 + total_plans: 7 + completed_plans: 7 +--- + +# Project State: Histogram ST WAL Records + +## Project Reference + +See: .planning/PROJECT.md (updated 2026-03-02) + +**Core value:** Histogram samples carry accurate start time through the WAL +**Current focus:** All phases complete. Histogram ST WAL records fully implemented and tested. + +## Phase Progress + +| Phase | Name | Status | +|-------|------|--------| +| 1 | Struct and Type Definitions | Plan 01 complete | +| 2 | V2 Encoders | Complete | +| 3 | V2 Decoders | Complete | +| 4 | Tests | Complete | + +## Current Phase + +Phase 4 complete. Plan 01 (021b0d9e4, b05e0d328), Plan 02 (2523b681e). All phases done. + +## Decisions Log + +- **01-struct-and-type-definitions:** V2 histogram String() names use underscores (histogram_samples_v2), consistent with V1 histogram naming convention. SamplesV2 uses hyphens but that pre-existing inconsistency is not followed for histogram V2 types. +- **01-struct-and-type-definitions:** ST field declared as `ST, T int64` on one line, ST before T, matching RefSample layout exactly. Zero-value default is backward-compatible with all existing named-field callers. +- **02-v2-encoders:** V2 int-histogram encoder uses all-varint first-sample, ref-delta-to-prev, T-delta-to-first, ST marker scheme (noST/sameST/explicitST). Deliberately breaks from V1 BE64 wire format. +- **02-v2-encoders (plan 02):** V2 float-histogram encoder follows identical pattern to int-histogram V2. All four public histogram encoder methods now dispatch to V1/V2 based on EnableSTStorage. + +- **03-v2-decoders (plan 01):** Extracted V1 body into histogramSamplesV1 private method for clean switch dispatch. V2 decoder mirrors samplesV2 exactly. +- **03-v2-decoders (plan 02):** Extracted V1 body into floatHistogramSamplesV1 private method, mirroring Plan 01. V2 float-histogram decoder mirrors histogramSamplesV2 with FloatHistogram payload. + +- **04-tests (plan 01):** Used t.Run subtests for each ST scenario and gauge/backward-compat block for failure isolation. Float histogram derivation includes explicit ST copy. V2 blocks placed before gauge mutation to avoid shared pointer corruption. +- **04-tests (plan 02):** Reset enc to zero-value Encoder{} before V1 histogram assertions to fix pre-existing EnableSTStorage leak from SamplesV2 test block. + +## Blockers + +(None) + +--- +*Last updated: 2026-03-03 after Phase 4 Plan 02 (04-02-PLAN.md). All plans complete.* diff --git a/.planning/config.json b/.planning/config.json new file mode 100644 index 0000000000..a9a617f097 --- /dev/null +++ b/.planning/config.json @@ -0,0 +1,10 @@ +{ + "project_name": "histogram-st-wal-records", + "created": "2026-03-02", + "researcher_model": "sonnet", + "synthesizer_model": "sonnet", + "roadmapper_model": "sonnet", + "commit_docs": true, + "is_brownfield": true, + "skip_research": true +} diff --git a/.planning/phases/01-struct-and-type-definitions/01-01-PLAN.md b/.planning/phases/01-struct-and-type-definitions/01-01-PLAN.md new file mode 100644 index 0000000000..0c6a4f42e4 --- /dev/null +++ b/.planning/phases/01-struct-and-type-definitions/01-01-PLAN.md @@ -0,0 +1,284 @@ +--- +phase: 01-struct-and-type-definitions +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - tsdb/record/record.go +autonomous: true +requirements: + - STRUCT-01 + - STRUCT-02 + - TYPE-01 + - TYPE-02 + - TYPE-03 + - TYPE-04 + - TYPE-05 + +must_haves: + truths: + - "RefHistogramSample has an ST int64 field before T, matching RefSample layout" + - "RefFloatHistogramSample has an ST int64 field before T, matching RefSample layout" + - "Four new V2 record type constants exist with values 12-15" + - "Type.String() returns correct snake_case_v2 names for all four new types" + - "Decoder.Type() recognizes all four new types as valid (does not return Unknown)" + - "TODO comments about ST support are removed from both histogram structs" + artifacts: + - path: "tsdb/record/record.go" + provides: "V2 histogram type constants and ST-enabled histogram structs" + contains: "HistogramSamplesV2 Type = 12" + key_links: + - from: "tsdb/record/record.go constants block" + to: "tsdb/record/record.go Decoder.Type() switch" + via: "New type constants referenced in validity check" + pattern: "HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2" +--- + + +Add ST fields to histogram sample structs and declare V2 record type constants. + +Purpose: All types and structs must exist before Phase 2 (encoders) and Phase 3 (decoders) can reference them. +Output: Updated tsdb/record/record.go with ST fields, four new type constants, updated String() and Type() functions. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/01-struct-and-type-definitions/01-CONTEXT.md +@.planning/phases/01-struct-and-type-definitions/01-RESEARCH.md + +@tsdb/record/record.go + + + + +From tsdb/record/record.go (lines 36-63, constants block): +```go +type Type uint8 + +const ( + Unknown Type = 255 + Series Type = 1 + Samples Type = 2 + Tombstones Type = 3 + Exemplars Type = 4 + MmapMarkers Type = 5 + Metadata Type = 6 + HistogramSamples Type = 7 + FloatHistogramSamples Type = 8 + CustomBucketsHistogramSamples Type = 9 + CustomBucketsFloatHistogramSamples Type = 10 + SamplesV2 Type = 11 +) +``` + +From tsdb/record/record.go (lines 166-170, RefSample - the pattern to mirror): +```go +type RefSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + V float64 +} +``` + +From tsdb/record/record.go (lines 188-202, structs to modify): +```go +// RefHistogramSample is a histogram. +// TODO(owilliams): Add support for ST. +type RefHistogramSample struct { + Ref chunks.HeadSeriesRef + T int64 + H *histogram.Histogram +} + +// RefFloatHistogramSample is a float histogram. +// TODO(owilliams): Add support for ST. +type RefFloatHistogramSample struct { + Ref chunks.HeadSeriesRef + T int64 + FH *histogram.FloatHistogram +} +``` + +From tsdb/record/record.go (lines 224-233, Decoder.Type()): +```go +func (*Decoder) Type(rec []byte) Type { + if len(rec) < 1 { + return Unknown + } + switch t := Type(rec[0]); t { + case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples: + return t + } + return Unknown +} +``` + + + + + + + Task 1: Add ST field to histogram sample structs + tsdb/record/record.go + +Modify two structs in tsdb/record/record.go to add the ST field, matching the RefSample pattern at line 166-170. + +1. RefHistogramSample (lines 188-194): + - Remove the TODO comment line: `// TODO(owilliams): Add support for ST.` + - Keep the doc comment: `// RefHistogramSample is a histogram.` + - Replace `T int64` with `ST, T int64` (ST before T, on one line, matching RefSample) + - Align field spacing to match RefSample style (Ref gets extra spacing for column alignment) + + Before: + ```go + // RefHistogramSample is a histogram. + // TODO(owilliams): Add support for ST. + type RefHistogramSample struct { + Ref chunks.HeadSeriesRef + T int64 + H *histogram.Histogram + } + ``` + + After: + ```go + // RefHistogramSample is a histogram. + type RefHistogramSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + H *histogram.Histogram + } + ``` + +2. RefFloatHistogramSample (lines 196-202): + - Remove the TODO comment line: `// TODO(owilliams): Add support for ST.` + - Keep the doc comment: `// RefFloatHistogramSample is a float histogram.` + - Replace `T int64` with `ST, T int64` + - Align field spacing + + Before: + ```go + // RefFloatHistogramSample is a float histogram. + // TODO(owilliams): Add support for ST. + type RefFloatHistogramSample struct { + Ref chunks.HeadSeriesRef + T int64 + FH *histogram.FloatHistogram + } + ``` + + After: + ```go + // RefFloatHistogramSample is a float histogram. + type RefFloatHistogramSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + FH *histogram.FloatHistogram + } + ``` + +NOTE: All existing callers use named field initialization (verified by grep), so adding ST with zero-value default is backward-compatible. No other files need changes. + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... + + Both RefHistogramSample and RefFloatHistogramSample have `ST, T int64` field (ST before T). TODO comments removed. Code compiles. + + + + Task 2: Add V2 record type constants and update String/Type switches + tsdb/record/record.go + +Add four new record type constants and update two switch statements in tsdb/record/record.go. + +1. Constants block (after SamplesV2 = 11 at line 62). Insert these four constants before the closing paren: + + ```go + // HistogramSamplesV2 is an enhanced histogram record that supports start time per sample. + HistogramSamplesV2 Type = 12 + // FloatHistogramSamplesV2 is an enhanced float histogram record that supports start time per sample. + FloatHistogramSamplesV2 Type = 13 + // CustomBucketsHistogramSamplesV2 is an enhanced custom-buckets histogram record that supports start time per sample. + CustomBucketsHistogramSamplesV2 Type = 14 + // CustomBucketsFloatHistogramSamplesV2 is an enhanced custom-buckets float histogram record that supports start time per sample. + CustomBucketsFloatHistogramSamplesV2 Type = 15 + ``` + +2. Type.String() switch (lines 65-92). Add four new cases BEFORE the default case. Insert after the CustomBucketsFloatHistogramSamples case (line 84) and before the MmapMarkers case (line 85), or alternatively group them together after all existing histogram cases. The exact position within the switch does not matter functionally, but grouping with other histogram types is cleanest: + + ```go + case HistogramSamplesV2: + return "histogram_samples_v2" + case FloatHistogramSamplesV2: + return "float_histogram_samples_v2" + case CustomBucketsHistogramSamplesV2: + return "custom_buckets_histogram_samples_v2" + case CustomBucketsFloatHistogramSamplesV2: + return "custom_buckets_float_histogram_samples_v2" + ``` + + IMPORTANT: Use underscores, NOT hyphens. The existing SamplesV2 uses "samples-v2" (hyphen) but per user decision, histogram V2 types use underscores to stay consistent with histogram V1 naming ("histogram_samples" etc.). + +3. Decoder.Type() switch (line 229). Append the four new types to the existing case clause. The line currently reads: + ```go + case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples: + ``` + + Change to (split across lines for readability): + ```go + case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, + HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples, + HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2: + ``` + + CRITICAL: If you forget this step, Decoder.Type() will return Unknown for types 12-15, silently breaking Phase 3 decoders. + + + cd /home/owilliams/src/grafana/prometheus && go vet ./tsdb/record/... + + Four new type constants (12-15) declared. Type.String() returns "histogram_samples_v2", "float_histogram_samples_v2", "custom_buckets_histogram_samples_v2", "custom_buckets_float_histogram_samples_v2". Decoder.Type() recognizes all four new types. go vet passes. + + + + + +Run both build and vet to confirm the entire phase: + +```bash +cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... +``` + +Then confirm the specific changes: +```bash +cd /home/owilliams/src/grafana/prometheus && grep -n "ST, T" tsdb/record/record.go +# Should show ST, T in RefHistogramSample, RefFloatHistogramSample, and RefSample + +cd /home/owilliams/src/grafana/prometheus && grep -n "V2 Type" tsdb/record/record.go +# Should show HistogramSamplesV2 = 12, FloatHistogramSamplesV2 = 13, etc. + +cd /home/owilliams/src/grafana/prometheus && grep -c "TODO.*owilliams.*ST" tsdb/record/record.go +# Should return 0 (no remaining TODO comments) +``` + + + +1. `go build ./tsdb/record/...` succeeds (no compilation errors) +2. `go vet ./tsdb/record/...` succeeds (no static analysis warnings) +3. RefHistogramSample and RefFloatHistogramSample both have `ST, T int64` field +4. Four new constants: HistogramSamplesV2=12, FloatHistogramSamplesV2=13, CustomBucketsHistogramSamplesV2=14, CustomBucketsFloatHistogramSamplesV2=15 +5. Type.String() returns correct snake_case_v2 strings for all four +6. Decoder.Type() returns the correct type (not Unknown) for byte values 12-15 +7. No TODO(owilliams) comments remain on the histogram structs + + + +After completion, create `.planning/phases/01-struct-and-type-definitions/01-01-SUMMARY.md` + diff --git a/.planning/phases/01-struct-and-type-definitions/01-01-SUMMARY.md b/.planning/phases/01-struct-and-type-definitions/01-01-SUMMARY.md new file mode 100644 index 0000000000..7dd477aca1 --- /dev/null +++ b/.planning/phases/01-struct-and-type-definitions/01-01-SUMMARY.md @@ -0,0 +1,113 @@ +--- +phase: 01-struct-and-type-definitions +plan: 01 +subsystem: tsdb +tags: [wal, histogram, record-types, go] + +# Dependency graph +requires: [] +provides: + - "RefHistogramSample.ST int64 field (ST before T, matching RefSample layout)" + - "RefFloatHistogramSample.ST int64 field (ST before T, matching RefSample layout)" + - "HistogramSamplesV2 Type = 12" + - "FloatHistogramSamplesV2 Type = 13" + - "CustomBucketsHistogramSamplesV2 Type = 14" + - "CustomBucketsFloatHistogramSamplesV2 Type = 15" + - "Type.String() returns snake_case_v2 names for all four new types" + - "Decoder.Type() recognizes types 12-15 as valid (not Unknown)" +affects: + - 02-v2-encoders + - 03-v2-decoders + - 04-tests + +# Tech tracking +tech-stack: + added: [] + patterns: + - "V2 histogram type constants follow HistogramSamplesV2 naming (suffix, not prefix)" + - "V2 histogram String() uses snake_case_v2 (underscores, not hyphens)" + - "ST field declared as ST, T int64 on one line, ST before T, matching RefSample" + +key-files: + created: [] + modified: + - tsdb/record/record.go + +key-decisions: + - "V2 histogram String() names use underscores (histogram_samples_v2) not hyphens, consistent with V1 histogram naming even though SamplesV2 uses hyphens" + - "ST field placement: ST, T int64 on single line with ST before T, matching RefSample layout exactly" + - "Four separate V2 type constants (one per V1 type) to keep type space consistent" + +patterns-established: + - "ST field: declare as ST, T int64 (one line, ST before T) in all histogram sample structs" + - "V2 type constants: sequential integers after SamplesV2 (11), starting at 12" + +requirements-completed: [STRUCT-01, STRUCT-02, TYPE-01, TYPE-02, TYPE-03, TYPE-04, TYPE-05] + +# Metrics +duration: 2min +completed: 2026-03-02 +--- + +# Phase 1 Plan 1: Struct and Type Definitions Summary + +**ST fields added to RefHistogramSample and RefFloatHistogramSample, four V2 WAL record type constants (12-15) declared with String() and Decoder.Type() support** + +## Performance + +- **Duration:** 2 min +- **Started:** 2026-03-02T21:16:51Z +- **Completed:** 2026-03-02T21:19:00Z +- **Tasks:** 2 +- **Files modified:** 1 + +## Accomplishments +- Added `ST, T int64` field to `RefHistogramSample` and `RefFloatHistogramSample` matching the `RefSample` layout exactly, removed TODO(owilliams) comments +- Declared four new V2 record type constants: `HistogramSamplesV2=12`, `FloatHistogramSamplesV2=13`, `CustomBucketsHistogramSamplesV2=14`, `CustomBucketsFloatHistogramSamplesV2=15` +- Updated `Type.String()` with four new cases returning underscore-separated snake_case_v2 strings grouped with V1 histogram types +- Updated `Decoder.Type()` to recognize all four new types as valid (not Unknown), split across three lines for readability + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Add ST field to histogram sample structs** - `91daad4f8` (feat) +2. **Task 2: Add V2 record type constants and update String/Type switches** - `0c33819e5` (feat) + +**Plan metadata:** (pending docs commit) + +## Files Created/Modified +- `tsdb/record/record.go` - ST fields added to two structs, four V2 constants added, String() and Decoder.Type() switches updated + +## Decisions Made +- V2 histogram String() names use underscores (`histogram_samples_v2`) not hyphens, for consistency with V1 histogram names (`histogram_samples`). SamplesV2 uses `"samples-v2"` (hyphen) but that is a pre-existing inconsistency; histogram V2 types follow the histogram V1 convention. +- No other files needed changes because all existing callers use named field initialization, making the new ST field backward-compatible with a zero-value default. + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered + +None. + +## User Setup Required + +None - no external service configuration required. + +## Next Phase Readiness +- All struct fields and type constants are in place for Phase 2 (V2 encoders) and Phase 3 (V2 decoders) to reference +- Phase 2 can use `ST, T int64` fields on both histogram sample structs directly +- Phase 2 can use `HistogramSamplesV2`, `FloatHistogramSamplesV2`, `CustomBucketsHistogramSamplesV2`, `CustomBucketsFloatHistogramSamplesV2` constants as the record type byte +- No blockers + +--- +*Phase: 01-struct-and-type-definitions* +*Completed: 2026-03-02* + +## Self-Check: PASSED + +- tsdb/record/record.go: FOUND +- 01-01-SUMMARY.md: FOUND +- Commit 91daad4 (Task 1): FOUND +- Commit 0c33819 (Task 2): FOUND diff --git a/.planning/phases/01-struct-and-type-definitions/01-CONTEXT.md b/.planning/phases/01-struct-and-type-definitions/01-CONTEXT.md new file mode 100644 index 0000000000..2eed0486f1 --- /dev/null +++ b/.planning/phases/01-struct-and-type-definitions/01-CONTEXT.md @@ -0,0 +1,79 @@ +# Phase 1: Struct and Type Definitions - Context + +**Gathered:** 2026-03-02 +**Status:** Ready for planning + + +## Phase Boundary + +Add ST fields to RefHistogramSample and RefFloatHistogramSample structs, and declare four new V2 record type constants. No encoding/decoding logic. Code must compile and pass vet. + + + + +## Implementation Decisions + +### ST field style +- Use `ST, T int64` on one line, matching RefSample pattern exactly +- ST comes before T (same ordering as RefSample) +- Both RefHistogramSample and RefFloatHistogramSample get the field + +### V2 type naming +- Use `V2` suffix: HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2 +- Matches existing SamplesV2 naming convention +- String() returns snake_case with `_v2` suffix (e.g., "histogram_samples_v2") + +### V2 type numbering +- Sequential after SamplesV2 (11): types 12, 13, 14, 15 +- Order: HistogramSamplesV2 (12), FloatHistogramSamplesV2 (13), CustomBucketsHistogramSamplesV2 (14), CustomBucketsFloatHistogramSamplesV2 (15) + +### CustomBuckets V2 strategy +- Mirror V1: four separate V2 types, one for each V1 type +- Keeps the type space consistent and predictable +- CustomBuckets encoder functions already exist separately. V2 versions will too. + +### Claude's Discretion +- Exact comment wording on new constants +- Whether to update the struct doc comment beyond removing the TODO + + + + +## Existing Code Insights + +### Reusable Assets +- `RefSample` struct at line 166: direct pattern to mirror for field placement +- `SamplesV2` constant at line 62: naming and numbering pattern to follow +- `Type.String()` switch at line 65: add new cases here +- `Decoder.Type()` switch at line 229: add new types to the valid set + +### Established Patterns +- Record type constants are sequential integers starting from 1 +- `Type.String()` uses snake_case (e.g., "histogram_samples", "samples-v2") +- Decoder.Type() lists all valid types in a single case clause +- Struct comments are brief, one-line descriptions + +### Integration Points +- New types will be referenced by Phase 2 (encoders) and Phase 3 (decoders) +- EnableSTStorage flag on Encoder struct gates V1 vs V2 dispatch + + + + +## Specific Ideas + +- The existing SamplesV2 String() uses "samples-v2" (hyphen, not underscore). Histogram V1 types use underscores ("histogram_samples"). V2 histogram types should use underscores with v2: "histogram_samples_v2". This keeps consistency within histogram types while acknowledging the SamplesV2 hyphen is a minor inconsistency. + + + + +## Deferred Ideas + +None. Discussion stayed within phase scope. + + + +--- + +*Phase: 01-struct-and-type-definitions* +*Context gathered: 2026-03-02* diff --git a/.planning/phases/01-struct-and-type-definitions/01-RESEARCH.md b/.planning/phases/01-struct-and-type-definitions/01-RESEARCH.md new file mode 100644 index 0000000000..f671f68d19 --- /dev/null +++ b/.planning/phases/01-struct-and-type-definitions/01-RESEARCH.md @@ -0,0 +1,308 @@ +# Phase 1: Struct and Type Definitions - Research + +**Researched:** 2026-03-02 +**Domain:** Go struct field additions and iota-style constant declarations in `tsdb/record/record.go` +**Confidence:** HIGH + + +## User Constraints (from CONTEXT.md) + +### Locked Decisions + +- Use `ST, T int64` on one line, matching RefSample pattern exactly. ST comes before T. +- Both RefHistogramSample and RefFloatHistogramSample get the ST field. +- V2 type naming: `HistogramSamplesV2`, `FloatHistogramSamplesV2`, `CustomBucketsHistogramSamplesV2`, `CustomBucketsFloatHistogramSamplesV2`. +- Sequential numbering after SamplesV2 (11): types 12, 13, 14, 15. +- Order: HistogramSamplesV2 (12), FloatHistogramSamplesV2 (13), CustomBucketsHistogramSamplesV2 (14), CustomBucketsFloatHistogramSamplesV2 (15). +- Mirror V1: four separate V2 types, one for each V1 type. +- String() returns snake_case with `_v2` suffix (e.g., "histogram_samples_v2"). + +### Claude's Discretion + +- Exact comment wording on new constants. +- Whether to update the struct doc comment beyond removing the TODO. + +### Deferred Ideas (OUT OF SCOPE) + +None. Discussion stayed within phase scope. + + + +## Phase Requirements + +| ID | Description | Research Support | +|----|-------------|-----------------| +| STRUCT-01 | RefHistogramSample has ST field (int64) alongside T | Lines 188-194 in record.go show current struct; RefSample (lines 166-170) shows exact pattern to mirror | +| STRUCT-02 | RefFloatHistogramSample has ST field (int64) alongside T | Lines 196-202 in record.go show current struct; same pattern as STRUCT-01 | +| TYPE-01 | HistogramSamplesV2 record type constant added (value 12) | Constants block lines 38-63; SamplesV2 = 11 is the predecessor | +| TYPE-02 | FloatHistogramSamplesV2 record type constant added (value 13) | Same constants block | +| TYPE-03 | CustomBucketsHistogramSamplesV2 record type constant added (value 14) | Same constants block | +| TYPE-04 | CustomBucketsFloatHistogramSamplesV2 record type constant added (value 15) | Same constants block | +| TYPE-05 | Type.String() returns correct names for new types | Switch at lines 65-92; add four new cases following histogram_samples naming pattern | + + +--- + +## Summary + +This phase is pure Go declaration work in a single file: `tsdb/record/record.go`. No logic changes, no new functions, no encoding or decoding. The task is to add `ST int64` fields to two structs, declare four new type constants, extend the `String()` switch, update the `Type()` validity check, and remove two TODO comments. + +The existing code provides exact templates for everything. `RefSample` (line 166) shows the `ST, T int64` field layout. The `SamplesV2 = 11` constant (line 62) shows the comment and numbering style. The `Type.String()` switch (lines 65-92) shows the case pattern. The `Decoder.Type()` switch case (line 229) shows where new types are registered as valid. + +The only judgment call the planner needs to make is comment wording (explicitly left to Claude's discretion in CONTEXT.md) and whether to refresh the struct doc comments beyond removing the TODOs. Both are trivial. + +**Primary recommendation:** Make all changes in a single commit to `tsdb/record/record.go`. Verify with `go build ./tsdb/record/...` and `go vet ./tsdb/record/...`. + +## Standard Stack + +### Core +| Tool | Version | Purpose | Why Standard | +|------|---------|---------|--------------| +| Go stdlib | 1.23+ (module) | Language and type system | This is a Go project | +| `go build` | same | Compile-time verification | Catches type errors immediately | +| `go vet` | same | Static analysis | Project's stated verification method | + +No external libraries are added in this phase. All changes are pure Go declarations. + +**Verification commands:** +```bash +go build ./tsdb/record/... +go vet ./tsdb/record/... +``` + +## Architecture Patterns + +### Pattern 1: Multi-field declaration on one line + +The project uses grouped field declarations where fields share a type. The canonical example is `RefSample`: + +```go +// Source: tsdb/record/record.go line 166-170 +type RefSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + V float64 +} +``` + +Apply this exactly to both histogram structs. `ST` precedes `T` on the same line. + +### Pattern 2: Typed integer constants without iota + +New record type constants follow the existing block style - explicit integer literals, not iota, with a comment for each: + +```go +// Source: tsdb/record/record.go lines 38-63 +// SamplesV2 is an enhanced sample record with an encoding scheme that allows storing float samples with timestamp and an optional ST per sample. +SamplesV2 Type = 11 +``` + +New constants slot directly after `SamplesV2 = 11`. Each gets a brief comment describing the record type. + +### Pattern 3: Type.String() switch case + +New cases are added in the `func (rt Type) String() string` switch. Histogram V1 types use snake_case with underscores. V2 histogram types follow the same convention with `_v2` appended: + +```go +// Source: tsdb/record/record.go lines 65-92 +// Existing examples: +case HistogramSamples: + return "histogram_samples" +case FloatHistogramSamples: + return "float_histogram_samples" +case CustomBucketsHistogramSamples: + return "custom_buckets_histogram_samples" +case CustomBucketsFloatHistogramSamples: + return "custom_buckets_float_histogram_samples" + +// New cases to add: +case HistogramSamplesV2: + return "histogram_samples_v2" +case FloatHistogramSamplesV2: + return "float_histogram_samples_v2" +case CustomBucketsHistogramSamplesV2: + return "custom_buckets_histogram_samples_v2" +case CustomBucketsFloatHistogramSamplesV2: + return "custom_buckets_float_histogram_samples_v2" +``` + +Note: `SamplesV2` uses "samples-v2" (hyphen) not "samples_v2". This is a known minor inconsistency. The decision is to use underscores for the new histogram V2 types to stay consistent with the histogram V1 naming style. + +### Pattern 4: Decoder.Type() validity switch + +The single case clause at line 229 lists all known valid types. New types are appended to this list: + +```go +// Source: tsdb/record/record.go line 229 (current) +case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples: + +// After change: +case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples, HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2: +``` + +### Anti-Patterns to Avoid + +- **Renumbering existing constants:** Never. WAL format is on-disk. Renumbering would corrupt existing WALs. +- **Using iota:** The existing block uses explicit values for a reason. Keep it that way. +- **Adding ST field after T:** The decision locks ST before T, matching RefSample. Don't swap the order. +- **Naming inconsistency:** Don't mix hyphen and underscore in V2 histogram type strings. Use underscores throughout. +- **Forgetting Decoder.Type():** Adding constants and String() cases but not updating `Decoder.Type()` means the decoder will classify new record types as Unknown. That will cause silent data loss in Phase 3. + +## Don't Hand-Roll + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| Compile check | Custom validator | `go build` | Compiler catches all type and field errors | +| Vet check | Custom linter | `go vet` | Already the project's stated acceptance criterion | + +There is nothing to hand-roll in this phase. The work is declaration-only. + +## Common Pitfalls + +### Pitfall 1: Forgetting to update Decoder.Type() + +**What goes wrong:** New constants compile fine, String() works, but `Decoder.Type()` still returns `Unknown` for types 12-15. Phase 3 decoder functions that switch on the type will never match, and records will be silently discarded or error-out with "invalid record type". + +**Why it happens:** It's easy to focus on the visible constants block and String() switch and overlook the separate validity switch in `Decoder.Type()` at line 224. + +**How to avoid:** The CONTEXT.md explicitly calls out line 229 as a touch point. Treat it as a checklist item in the task. + +**Warning signs:** `dec.Type(encodedRecord)` returns `Unknown` for any of the four new type bytes (12-15). + +### Pitfall 2: ST field position wrong + +**What goes wrong:** If `T, ST int64` is written instead of `ST, T int64`, the struct memory layout differs from what Phase 2 encoders and Phase 3 decoders will assume, and any code that initializes the struct positionally (not by name) will silently assign values to the wrong fields. + +**Why it happens:** Natural tendency to write `T` first since it's already there. + +**How to avoid:** The decision explicitly states ST comes before T, matching RefSample at line 168. Cross-check the final diff. + +**Warning signs:** Positional struct literal compilation errors if any callers use positional syntax (rare in this codebase but possible). + +### Pitfall 3: TODO comment left in place + +**What goes wrong:** The build passes, but the TODO comment remains. The task goal includes removing the TODOs. + +**Why it happens:** Editing the field and forgetting to also edit the comment above it. + +**How to avoid:** Each struct has a comment block directly above it. `RefHistogramSample` at line 188-189 and `RefFloatHistogramSample` at line 196-197. Both contain `// TODO(owilliams): Add support for ST.`. Remove both. + +### Pitfall 4: String() naming inconsistency + +**What goes wrong:** Using "histogram-samples-v2" (hyphens) instead of "histogram_samples_v2" (underscores) for new V2 histogram type strings. + +**Why it happens:** `SamplesV2` uses "samples-v2" with a hyphen, which could be mistakenly treated as the V2 naming pattern. + +**How to avoid:** The CONTEXT.md specifics section explicitly documents this: histogram V2 types should use underscores with v2 suffix. The hyphen in SamplesV2 is an acknowledged inconsistency. + +## Code Examples + +### Before and after: RefHistogramSample + +```go +// BEFORE (lines 188-194) +// RefHistogramSample is a histogram. +// TODO(owilliams): Add support for ST. +type RefHistogramSample struct { + Ref chunks.HeadSeriesRef + T int64 + H *histogram.Histogram +} + +// AFTER +// RefHistogramSample is a histogram. +type RefHistogramSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + H *histogram.Histogram +} +``` + +### Before and after: RefFloatHistogramSample + +```go +// BEFORE (lines 196-202) +// RefFloatHistogramSample is a float histogram. +// TODO(owilliams): Add support for ST. +type RefFloatHistogramSample struct { + Ref chunks.HeadSeriesRef + T int64 + FH *histogram.FloatHistogram +} + +// AFTER +// RefFloatHistogramSample is a float histogram. +type RefFloatHistogramSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + FH *histogram.FloatHistogram +} +``` + +### New constants block (append after SamplesV2 = 11 at line 62) + +```go +// HistogramSamplesV2 is an enhanced histogram record that supports start time per sample. +HistogramSamplesV2 Type = 12 +// FloatHistogramSamplesV2 is an enhanced float histogram record that supports start time per sample. +FloatHistogramSamplesV2 Type = 13 +// CustomBucketsHistogramSamplesV2 is an enhanced custom-buckets histogram record that supports start time per sample. +CustomBucketsHistogramSamplesV2 Type = 14 +// CustomBucketsFloatHistogramSamplesV2 is an enhanced custom-buckets float histogram record that supports start time per sample. +CustomBucketsFloatHistogramSamplesV2 Type = 15 +``` + +### New String() cases (insert in func (rt Type) String(), after existing histogram cases) + +```go +case HistogramSamplesV2: + return "histogram_samples_v2" +case FloatHistogramSamplesV2: + return "float_histogram_samples_v2" +case CustomBucketsHistogramSamplesV2: + return "custom_buckets_histogram_samples_v2" +case CustomBucketsFloatHistogramSamplesV2: + return "custom_buckets_float_histogram_samples_v2" +``` + +### Updated Decoder.Type() case (line 229) + +```go +// Source: tsdb/record/record.go line 228-230 +case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, + HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples, + HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2: + return t +``` + +## State of the Art + +| Old Approach | Current Approach | Impact | +|--------------|------------------|--------| +| RefHistogramSample with T only | RefHistogramSample with ST, T (this phase) | Enables start-time propagation in later phases | +| Types 1-11 only | Types 1-15 (this phase) | New record types needed for V2 encoding in Phase 2 | + +No deprecated patterns introduced. This phase is purely additive. + +## Open Questions + +None. The CONTEXT.md resolves all ambiguities. The code is fully visible and the patterns are unambiguous. + +## Sources + +### Primary (HIGH confidence) +- `tsdb/record/record.go` (read in full) - all patterns, line numbers, and existing types verified directly from source +- `tsdb/record/record_test.go` (read partial) - test framework and patterns confirmed + +### Secondary (MEDIUM confidence) +- `.planning/phases/01-struct-and-type-definitions/01-CONTEXT.md` - user decisions verified against source code + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH - verified by reading the actual source file +- Architecture: HIGH - patterns extracted directly from live source, not docs +- Pitfalls: HIGH - derived from reading actual code structure and call sites + +**Research date:** 2026-03-02 +**Valid until:** 2026-04-01 (stable Go internal package; changes here would come from the same developer) diff --git a/.planning/phases/01-struct-and-type-definitions/01-VERIFICATION.md b/.planning/phases/01-struct-and-type-definitions/01-VERIFICATION.md new file mode 100644 index 0000000000..4293e4648b --- /dev/null +++ b/.planning/phases/01-struct-and-type-definitions/01-VERIFICATION.md @@ -0,0 +1,86 @@ +--- +phase: 01-struct-and-type-definitions +verified: 2026-03-02T21:30:00Z +status: passed +score: 6/6 must-haves verified +re_verification: false +--- + +# Phase 1: Struct and Type Definitions Verification Report + +**Phase Goal:** All types and structs exist so encoding/decoding can reference them. +**Verified:** 2026-03-02T21:30:00Z +**Status:** passed +**Re-verification:** No -- initial verification + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +|-----|-------------------------------------------------------------------------------------------|------------|-------------------------------------------------------------------------------------------| +| 1 | RefHistogramSample has an ST int64 field before T, matching RefSample layout | VERIFIED | Line 207: `ST, T int64` -- ST before T, same pattern as RefSample at line 184 | +| 2 | RefFloatHistogramSample has an ST int64 field before T, matching RefSample layout | VERIFIED | Line 214: `ST, T int64` -- ST before T, same pattern as RefSample at line 184 | +| 3 | Four new V2 record type constants exist with values 12-15 | VERIFIED | Lines 64-70: HistogramSamplesV2=12, FloatHistogramSamplesV2=13, CustomBucketsHistogramSamplesV2=14, CustomBucketsFloatHistogramSamplesV2=15 | +| 4 | Type.String() returns correct snake_case_v2 names for all four new types | VERIFIED | Lines 93-100: returns "histogram_samples_v2", "float_histogram_samples_v2", "custom_buckets_histogram_samples_v2", "custom_buckets_float_histogram_samples_v2" | +| 5 | Decoder.Type() recognizes all four new types as valid (does not return Unknown) | VERIFIED | Lines 243-245: all four V2 constants present in the case clause | +| 6 | TODO comments about ST support are removed from both histogram structs | VERIFIED | grep for "TODO.*owilliams.*ST" returns no matches; both structs have clean doc comments | + +**Score:** 6/6 truths verified + +### Required Artifacts + +| Artifact | Expected | Status | Details | +|---------------------------------|----------------------------------------------------|----------|-----------------------------------------------------------------------------| +| `tsdb/record/record.go` | V2 histogram type constants and ST-enabled structs | VERIFIED | File exists, 1129 lines, substantive. Contains all required changes. | + +**Wiring (Level 3):** This phase is a single-file, single-package change. The artifact is both the definition and the consumer interface. The constants and struct fields are used by the existing encoder and decoder methods in the same file, and will be consumed by Phase 2 (encoders) and Phase 3 (decoders). No orphaned artifact risk. + +### Key Link Verification + +| From | To | Via | Status | Details | +|-------------------------------------------|-------------------------------|------------------------------------------------------|---------|---------------------------------------------------------------------------------------------| +| Constants block (lines 64-70) | Decoder.Type() switch | All four V2 constants in the case list | WIRED | Lines 245: `HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2` present in switch | +| Constants block (lines 64-70) | Type.String() switch | All four V2 constants have case clauses | WIRED | Lines 93-100: all four have distinct return strings | + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +|-------------|-------------|-----------------------------------------------------------|-----------|-------------------------------------------------------------------------------------| +| STRUCT-01 | 01-01-PLAN | RefHistogramSample has ST field (int64) alongside T | SATISFIED | Line 207: `ST, T int64` | +| STRUCT-02 | 01-01-PLAN | RefFloatHistogramSample has ST field (int64) alongside T | SATISFIED | Line 214: `ST, T int64` | +| TYPE-01 | 01-01-PLAN | HistogramSamplesV2 record type constant added | SATISFIED | Line 64: `HistogramSamplesV2 Type = 12` | +| TYPE-02 | 01-01-PLAN | FloatHistogramSamplesV2 record type constant added | SATISFIED | Line 66: `FloatHistogramSamplesV2 Type = 13` | +| TYPE-03 | 01-01-PLAN | CustomBucketsHistogramSamplesV2 record type constant added | SATISFIED | Line 68: `CustomBucketsHistogramSamplesV2 Type = 14` | +| TYPE-04 | 01-01-PLAN | CustomBucketsFloatHistogramSamplesV2 record type constant added | SATISFIED | Line 70: `CustomBucketsFloatHistogramSamplesV2 Type = 15` | +| TYPE-05 | 01-01-PLAN | Type.String() returns correct names for new types | SATISFIED | Lines 93-100: four snake_case_v2 return strings | + +All 7 Phase 1 requirements accounted for. No orphaned requirements (ENC-*, DEC-*, TEST-* are correctly mapped to Phases 2, 3, and 4). + +### Anti-Patterns Found + +No anti-patterns detected. + +- No TODO/FIXME/PLACEHOLDER comments on the modified structs. +- No empty implementations or stub returns in the new code paths. +- `go build ./tsdb/record/...` passes. +- `go vet ./tsdb/record/...` passes. + +### Human Verification Required + +None. All changes are structural declarations (constants, struct fields, switch cases) that are fully verifiable statically. + +### Gaps Summary + +No gaps. Phase goal is fully achieved. + +Both histogram sample structs now carry `ST, T int64` fields in the correct order, matching the `RefSample` layout that encoders and decoders already use as a pattern. All four V2 type constants (12-15) are declared, covered in `Type.String()`, and recognized by `Decoder.Type()`. The codebase will compile cleanly for Phase 2 (encoders) and Phase 3 (decoders) to reference these definitions. + +Commits verified: +- `91daad4f8` -- feat(01-01): add ST field to RefHistogramSample and RefFloatHistogramSample +- `0c33819e5` -- feat(01-01): add V2 record type constants and update String/Type switches + +--- + +_Verified: 2026-03-02T21:30:00Z_ +_Verifier: Claude (gsd-verifier)_ diff --git a/.planning/phases/02-v2-encoders/02-01-PLAN.md b/.planning/phases/02-v2-encoders/02-01-PLAN.md new file mode 100644 index 0000000000..4053f005d4 --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-01-PLAN.md @@ -0,0 +1,299 @@ +--- +phase: 02-v2-encoders +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - tsdb/record/record.go +autonomous: true +requirements: [ENC-01, ENC-03, ENC-05] + +must_haves: + truths: + - "Encoder.HistogramSamples() dispatches to V2 when EnableSTStorage is true" + - "Encoder.CustomBucketsHistogramSamples() dispatches to V2 when EnableSTStorage is true" + - "V2 int-histogram encoding writes varint ref/T/ST for first sample and dRef/dT/STmarker for subsequent samples" + - "Custom-bucket filtering in V2 matches V1 behavior exactly" + artifacts: + - path: "tsdb/record/record.go" + provides: "histogramSamplesV1, histogramSamplesV2, customBucketsHistogramSamplesV1, customBucketsHistogramSamplesV2 private methods; updated HistogramSamples and CustomBucketsHistogramSamples public dispatch methods" + contains: "func (*Encoder) histogramSamplesV2" + key_links: + - from: "Encoder.HistogramSamples()" + to: "histogramSamplesV2" + via: "if e.EnableSTStorage dispatch" + pattern: "e\\.EnableSTStorage" + - from: "Encoder.CustomBucketsHistogramSamples()" + to: "customBucketsHistogramSamplesV2" + via: "if e.EnableSTStorage dispatch" + pattern: "e\\.EnableSTStorage" + - from: "histogramSamplesV2" + to: "EncodeHistogram" + via: "EncodeHistogram(&buf, h.H) call after ref/T/ST encoding" + pattern: "EncodeHistogram\\(&buf" +--- + + +Add V2 encoding for int-histogram record types (HistogramSamples and CustomBucketsHistogramSamples). + +Purpose: Enable histogram WAL records to carry start time (ST) using the varint-based V2 wire format with noST/sameST/explicitST marker scheme. Int-histogram variants first, float-histogram in Plan 02. + +Output: Four new private methods (two V1 extractions, two V2 implementations) and two updated public dispatch methods in record.go. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/01-struct-and-type-definitions/01-01-SUMMARY.md + + + + +From tsdb/record/record.go (lines 63-70): +```go +HistogramSamplesV2 Type = 12 +CustomBucketsHistogramSamplesV2 Type = 14 +``` + +From tsdb/record/record.go (lines 205-216): +```go +type RefHistogramSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + H *histogram.Histogram +} +``` + +From tsdb/record/record.go (lines 825-831): +```go +const ( + noST byte = iota + sameST + explicitST +) +``` + +From tsdb/record/record.go (line 746): +```go +type Encoder struct { + EnableSTStorage bool +} +``` + +From tsdb/record/record.go (line 989): +```go +func EncodeHistogram(buf *encoding.Encbuf, h *histogram.Histogram) +``` + +V2 dispatch pattern (from Samples at line 794): +```go +func (e *Encoder) Samples(samples []RefSample, b []byte) []byte { + if e.EnableSTStorage { + return e.samplesV2(samples, b) + } + return e.samplesV1(samples, b) +} +``` + +samplesV2 loop pattern (line 836) -- the exact template for V2 encoding: +```go +func (*Encoder) samplesV2(samples []RefSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(SamplesV2)) + if len(samples) == 0 { return buf.Get() } + first := samples[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + // ... payload ... + for i := 1; i < len(samples); i++ { + s := samples[i] + prev := samples[i-1] + buf.PutVarint64(int64(s.Ref) - int64(prev.Ref)) // delta to prev ref + buf.PutVarint64(s.T - first.T) // delta to first T + switch s.ST { + case 0: buf.PutByte(noST) + case prev.ST: buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(s.ST - first.ST) // delta to first ST + } + // ... payload ... + } + return buf.Get() +} +``` + + + + + + + Task 1: Extract V1 bodies and add dispatch for HistogramSamples and CustomBucketsHistogramSamples + tsdb/record/record.go + +Modify the two int-histogram public encoder methods to use named receiver and dispatch: + +1. **HistogramSamples** (currently at line 931): + - Change receiver from `(*Encoder)` to `(e *Encoder)` + - Replace body with dispatch: if `e.EnableSTStorage` call `e.histogramSamplesV2`, else call `e.histogramSamplesV1` + - Extract the original body into a new private method `histogramSamplesV1` with receiver `(*Encoder)`. Keep the body EXACTLY as-is (BE64 encoding, first.Ref delta style, custom-bucket filtering, buf.Reset). The only change is removing the type byte write from V1 body and... actually NO: keep the body completely intact including the type byte. The public method just delegates. + + Wait, correction: the public dispatch method should NOT write the type byte. Each V1/V2 method writes its own type byte. So: + - Public `HistogramSamples`: just dispatches, writes nothing itself + - `histogramSamplesV1`: exact current body (writes `HistogramSamples` type byte) + - `histogramSamplesV2`: writes `HistogramSamplesV2` type byte + +2. **CustomBucketsHistogramSamples** (currently at line 965): + - Same pattern: change receiver to `(e *Encoder)`, dispatch to V1/V2 + - Extract original body to `customBucketsHistogramSamplesV1` with `(*Encoder)` receiver + - Body unchanged + +Place the V1 private methods immediately after the public dispatch method. Leave space after V1 for V2 (Task 2). + +Key constraints per user decisions: +- Public method signatures unchanged (same return types) +- V1 bodies are exact copies of current code, just renamed to private methods + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + HistogramSamples and CustomBucketsHistogramSamples dispatch to V1 by default. Code compiles, vet passes. Existing behavior unchanged (EnableSTStorage defaults to false). + + + + Task 2: Add histogramSamplesV2 and customBucketsHistogramSamplesV2 private methods + tsdb/record/record.go + +Add two new private V2 encoder methods, placed immediately after their V1 counterparts. + +**histogramSamplesV2** -- mirrors samplesV2 structure with custom-bucket filtering: + +```go +func (*Encoder) histogramSamplesV2(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(HistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get(), nil + } + + var customBucketHistograms []RefHistogramSample + + // First sample: full varint values, no deltas, no ST marker. + first := histograms[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + EncodeHistogram(&buf, first.H) + + // Subsequent samples: ref delta to prev, T delta to first, ST marker. + for i := 1; i < len(histograms); i++ { + h := histograms[i] + if h.H.UsesCustomBuckets() { + customBucketHistograms = append(customBucketHistograms, h) + continue + } + prev := histograms[i-1] + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeHistogram(&buf, h.H) + } + + if len(histograms) == len(customBucketHistograms) { + buf.Reset() + } + + return buf.Get(), customBucketHistograms +} +``` + +**customBucketsHistogramSamplesV2** -- same V2 pattern but NO filtering, NO buf.Reset, returns []byte only: + +```go +func (*Encoder) customBucketsHistogramSamplesV2(histograms []RefHistogramSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(CustomBucketsHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get() + } + + first := histograms[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + EncodeHistogram(&buf, first.H) + + for i := 1; i < len(histograms); i++ { + h := histograms[i] + prev := histograms[i-1] + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeHistogram(&buf, h.H) + } + + return buf.Get() +} +``` + +Critical rules (per user decisions): +- V2 uses PutVarint64 for first sample ref/T/ST. NOT BE64. This deliberately breaks from V1. +- Ref deltas are to PREVIOUS ref (int64(h.Ref) - int64(prev.Ref)), NOT to first ref. Match samplesV2. +- T deltas are to FIRST T (h.T - first.T). Match samplesV2. +- ST marker: first sample has no marker (raw varint), subsequent use noST/sameST/explicitST. +- ST delta in explicitST case is to FIRST ST (h.ST - first.ST). Match samplesV2. +- CustomBuckets method has NO UsesCustomBuckets() check and NO buf.Reset(). + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + histogramSamplesV2 and customBucketsHistogramSamplesV2 exist, compile, and are reachable via dispatch when EnableSTStorage is true. V2 wire format uses all-varint encoding with ST marker scheme. + + + + + +cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + + +- go build and go vet pass on tsdb/record/... +- HistogramSamples() dispatches to V2 when EnableSTStorage is true +- CustomBucketsHistogramSamples() dispatches to V2 when EnableSTStorage is true +- V1 behavior is completely unchanged (extracted to private methods, bodies untouched) +- V2 methods use varint encoding, ref-delta-to-prev, T-delta-to-first, ST marker scheme + + + +After completion, create `.planning/phases/02-v2-encoders/02-01-SUMMARY.md` + diff --git a/.planning/phases/02-v2-encoders/02-01-SUMMARY.md b/.planning/phases/02-v2-encoders/02-01-SUMMARY.md new file mode 100644 index 0000000000..d2d4cdbb3f --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-01-SUMMARY.md @@ -0,0 +1,61 @@ +--- +phase: 02-v2-encoders +plan: 01 +subsystem: tsdb/record +tags: [encoder, v2, histogram, wal, start-time] +dependency_graph: + requires: [01-01] + provides: [histogramSamplesV2, customBucketsHistogramSamplesV2, dispatch] + affects: [tsdb/record/record.go] +tech_stack: + added: [] + patterns: [v2-st-marker-scheme, varint-encoding, ref-delta-to-prev] +key_files: + modified: [tsdb/record/record.go] +decisions: + - V2 uses all-varint first-sample encoding (not BE64), deliberately breaking from V1 wire format + - Ref deltas are to previous ref (not first), matching samplesV2 pattern + - T deltas are to first T, ST uses noST/sameST/explicitST marker scheme + - Both tasks implemented in single edit pass; no separate task commits needed +metrics: + duration: "~5 minutes" + completed: "2026-03-02T21:40:44Z" + tasks_completed: 2 + files_modified: 1 +--- + +# Phase 02 Plan 01: V2 Int-Histogram Encoder Summary + +Added V2 encoding for int-histogram WAL record types with full ST (start-time) support using the varint-based V2 wire format and noST/sameST/explicitST marker scheme. + +## What Was Built + +Four private methods added + two public dispatch methods updated in `tsdb/record/record.go`: + +- `histogramSamplesV1`: exact extraction of previous `HistogramSamples` body (BE64, delta-to-first-ref, no ST) +- `histogramSamplesV2`: new V2 encoding (varint, delta-to-prev-ref, delta-to-first-T, ST markers, custom-bucket filtering) +- `customBucketsHistogramSamplesV1`: exact extraction of previous `CustomBucketsHistogramSamples` body +- `customBucketsHistogramSamplesV2`: new V2 encoding (same pattern, no filtering, no buf.Reset) +- `HistogramSamples`: now dispatches via `e.EnableSTStorage` +- `CustomBucketsHistogramSamples`: now dispatches via `e.EnableSTStorage` + +## Commits + +| Task | Description | Hash | +|------|-------------|------| +| 1+2 | Add V2 encoding for int-histogram WAL record types | a3d49b0ac | + +## Deviations from Plan + +Tasks 1 and 2 were combined into a single edit and commit. The plan described them as separate tasks but since task 2's code was fully specified in the plan and no intermediate verification was needed between them, they were implemented together. Build and vet passed on the combined result. + +## Self-Check: PASSED + +- `tsdb/record/record.go` modified: confirmed +- commit `a3d49b0ac` exists: confirmed +- `go build ./tsdb/record/...` passes +- `go vet ./tsdb/record/...` passes +- `HistogramSamples` dispatches to `histogramSamplesV2` when `EnableSTStorage` is true +- `CustomBucketsHistogramSamples` dispatches to `customBucketsHistogramSamplesV2` when `EnableSTStorage` is true +- V1 bodies are exact copies of original code (wire format unchanged) +- V2 uses varint encoding, ref-delta-to-prev, T-delta-to-first, ST marker scheme diff --git a/.planning/phases/02-v2-encoders/02-02-PLAN.md b/.planning/phases/02-v2-encoders/02-02-PLAN.md new file mode 100644 index 0000000000..c0b413f53d --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-02-PLAN.md @@ -0,0 +1,267 @@ +--- +phase: 02-v2-encoders +plan: 02 +type: execute +wave: 2 +depends_on: ["02-01"] +files_modified: + - tsdb/record/record.go +autonomous: true +requirements: [ENC-02, ENC-04, ENC-05] + +must_haves: + truths: + - "Encoder.FloatHistogramSamples() dispatches to V2 when EnableSTStorage is true" + - "Encoder.CustomBucketsFloatHistogramSamples() dispatches to V2 when EnableSTStorage is true" + - "V2 float-histogram encoding writes varint ref/T/ST for first sample and dRef/dT/STmarker for subsequent samples" + - "Custom-bucket filtering in V2 float-histogram matches V1 behavior exactly" + artifacts: + - path: "tsdb/record/record.go" + provides: "floatHistogramSamplesV1, floatHistogramSamplesV2, customBucketsFloatHistogramSamplesV1, customBucketsFloatHistogramSamplesV2 private methods; updated FloatHistogramSamples and CustomBucketsFloatHistogramSamples public dispatch methods" + contains: "func (*Encoder) floatHistogramSamplesV2" + key_links: + - from: "Encoder.FloatHistogramSamples()" + to: "floatHistogramSamplesV2" + via: "if e.EnableSTStorage dispatch" + pattern: "e\\.EnableSTStorage" + - from: "Encoder.CustomBucketsFloatHistogramSamples()" + to: "customBucketsFloatHistogramSamplesV2" + via: "if e.EnableSTStorage dispatch" + pattern: "e\\.EnableSTStorage" + - from: "floatHistogramSamplesV2" + to: "EncodeFloatHistogram" + via: "EncodeFloatHistogram(&buf, h.FH) call after ref/T/ST encoding" + pattern: "EncodeFloatHistogram\\(&buf" +--- + + +Add V2 encoding for float-histogram record types (FloatHistogramSamples and CustomBucketsFloatHistogramSamples). + +Purpose: Complete the histogram V2 encoder set. Same ST marker scheme as Plan 01 but using RefFloatHistogramSample and EncodeFloatHistogram. After this plan, all four histogram encoder methods support V2. + +Output: Four new private methods (two V1 extractions, two V2 implementations) and two updated public dispatch methods in record.go. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/02-v2-encoders/02-01-SUMMARY.md + + + + +From tsdb/record/record.go (lines 65-69): +```go +FloatHistogramSamplesV2 Type = 13 +CustomBucketsFloatHistogramSamplesV2 Type = 15 +``` + +From tsdb/record/record.go: +```go +type RefFloatHistogramSample struct { + Ref chunks.HeadSeriesRef + ST, T int64 + FH *histogram.FloatHistogram +} +``` + +From tsdb/record/record.go (line 1089): +```go +func EncodeFloatHistogram(buf *encoding.Encbuf, h *histogram.FloatHistogram) +``` + +ST marker constants (same as Plan 01): +```go +const ( + noST byte = iota + sameST + explicitST +) +``` + +Pattern established by Plan 01 (int-histogram dispatch): +```go +func (e *Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + if e.EnableSTStorage { + return e.histogramSamplesV2(histograms, b) + } + return e.histogramSamplesV1(histograms, b) +} +``` + +Current float-histogram V1 code to extract (line 1030): +```go +func (*Encoder) FloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) ([]byte, []RefFloatHistogramSample) { + // ... uses BE64 for first ref/T, range loop with first-ref deltas, + // custom-bucket filtering via h.FH.UsesCustomBuckets(), buf.Reset() edge case +} +``` + +Current CustomBucketsFloatHistogramSamples (line 1065): +```go +func (*Encoder) CustomBucketsFloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) []byte { + // ... uses BE64 for first ref/T, range loop with first-ref deltas, no filtering +} +``` + + + + + + + Task 1: Extract V1 bodies and add dispatch for FloatHistogramSamples and CustomBucketsFloatHistogramSamples + tsdb/record/record.go + +Modify the two float-histogram public encoder methods. Exact same pattern used in Plan 01 for int-histograms: + +1. **FloatHistogramSamples** (currently at ~line 1030): + - Change receiver from `(*Encoder)` to `(e *Encoder)` + - Replace body with dispatch: if `e.EnableSTStorage` call `e.floatHistogramSamplesV2`, else call `e.floatHistogramSamplesV1` + - Extract original body into `floatHistogramSamplesV1` with `(*Encoder)` receiver, body exactly as-is + +2. **CustomBucketsFloatHistogramSamples** (currently at ~line 1065): + - Same pattern: change receiver to `(e *Encoder)`, dispatch to V1/V2 + - Extract original body to `customBucketsFloatHistogramSamplesV1` with `(*Encoder)` receiver + +Place V1 private methods immediately after their public dispatch counterparts. + +Note: Line numbers will have shifted from Plan 01. Use function names to locate, not line numbers. + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + FloatHistogramSamples and CustomBucketsFloatHistogramSamples dispatch to V1 by default. Code compiles, vet passes. Existing behavior unchanged. + + + + Task 2: Add floatHistogramSamplesV2 and customBucketsFloatHistogramSamplesV2 private methods + tsdb/record/record.go + +Add two new private V2 encoder methods. These are structurally identical to Plan 01's int-histogram V2 methods, but use `RefFloatHistogramSample`, access `h.FH` instead of `h.H`, call `EncodeFloatHistogram` instead of `EncodeHistogram`, and use `h.FH.UsesCustomBuckets()` for filtering. + +**floatHistogramSamplesV2:** + +```go +func (*Encoder) floatHistogramSamplesV2(histograms []RefFloatHistogramSample, b []byte) ([]byte, []RefFloatHistogramSample) { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(FloatHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get(), nil + } + + var customBucketsFloatHistograms []RefFloatHistogramSample + + first := histograms[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + EncodeFloatHistogram(&buf, first.FH) + + for i := 1; i < len(histograms); i++ { + h := histograms[i] + if h.FH.UsesCustomBuckets() { + customBucketsFloatHistograms = append(customBucketsFloatHistograms, h) + continue + } + prev := histograms[i-1] + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeFloatHistogram(&buf, h.FH) + } + + if len(histograms) == len(customBucketsFloatHistograms) { + buf.Reset() + } + + return buf.Get(), customBucketsFloatHistograms +} +``` + +**customBucketsFloatHistogramSamplesV2** -- NO filtering, NO buf.Reset: + +```go +func (*Encoder) customBucketsFloatHistogramSamplesV2(histograms []RefFloatHistogramSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(CustomBucketsFloatHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get() + } + + first := histograms[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + EncodeFloatHistogram(&buf, first.FH) + + for i := 1; i < len(histograms); i++ { + h := histograms[i] + prev := histograms[i-1] + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeFloatHistogram(&buf, h.FH) + } + + return buf.Get() +} +``` + +Same critical rules as Plan 01: +- V2 uses PutVarint64 for first sample ref/T/ST. NOT BE64. +- Ref deltas to PREVIOUS ref. T deltas to FIRST T. ST deltas to FIRST ST. +- First sample: no ST marker. Subsequent: noST/sameST/explicitST. +- CustomBuckets: NO UsesCustomBuckets() check, NO buf.Reset(). + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + floatHistogramSamplesV2 and customBucketsFloatHistogramSamplesV2 exist, compile, and are reachable via dispatch when EnableSTStorage is true. All four histogram V2 encoders are now complete. + + + + + +cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + + +- go build and go vet pass on tsdb/record/... +- FloatHistogramSamples() dispatches to V2 when EnableSTStorage is true +- CustomBucketsFloatHistogramSamples() dispatches to V2 when EnableSTStorage is true +- V1 behavior completely unchanged +- V2 methods use varint encoding, ref-delta-to-prev, T-delta-to-first, ST marker scheme +- All four public histogram encoder methods now have V1/V2 dispatch (combined with Plan 01) + + + +After completion, create `.planning/phases/02-v2-encoders/02-02-SUMMARY.md` + diff --git a/.planning/phases/02-v2-encoders/02-02-SUMMARY.md b/.planning/phases/02-v2-encoders/02-02-SUMMARY.md new file mode 100644 index 0000000000..e0f9b43c1a --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-02-SUMMARY.md @@ -0,0 +1,60 @@ +--- +phase: 02-v2-encoders +plan: 02 +subsystem: tsdb/record +tags: [encoder, v2, float-histogram, wal, start-time] +dependency_graph: + requires: [02-01] + provides: [floatHistogramSamplesV2, customBucketsFloatHistogramSamplesV2, dispatch] + affects: [tsdb/record/record.go] +tech_stack: + added: [] + patterns: [v2-st-marker-scheme, varint-encoding, ref-delta-to-prev] +key_files: + modified: [tsdb/record/record.go] +decisions: + - V2 float-histogram encoder uses all-varint first-sample (not BE64), matching Plan 01 int-histogram V2 pattern + - Ref deltas to previous ref, T deltas to first T, ST uses noST/sameST/explicitST marker scheme + - Tasks 1 and 2 combined into single edit and commit (V2 code fully specified, no intermediate verification needed) +metrics: + duration: "~5 minutes" + completed: "2026-03-02" + tasks_completed: 2 + files_modified: 1 +--- + +# Phase 02 Plan 02: V2 Float-Histogram Encoder Summary + +Added V2 encoding for float-histogram WAL record types with full ST (start-time) support using varint-based V2 wire format and noST/sameST/explicitST marker scheme. All four histogram encoder methods now have V1/V2 dispatch. + +## What Was Built + +Four private methods added + two public dispatch methods updated in `tsdb/record/record.go`: + +- `floatHistogramSamplesV1`: exact extraction of previous `FloatHistogramSamples` body (BE64, delta-to-first-ref, no ST) +- `floatHistogramSamplesV2`: new V2 encoding (varint, delta-to-prev-ref, delta-to-first-T, ST markers, custom-bucket filtering) +- `customBucketsFloatHistogramSamplesV1`: exact extraction of previous `CustomBucketsFloatHistogramSamples` body +- `customBucketsFloatHistogramSamplesV2`: new V2 encoding (same pattern, no filtering, no buf.Reset) +- `FloatHistogramSamples`: now dispatches via `e.EnableSTStorage` +- `CustomBucketsFloatHistogramSamples`: now dispatches via `e.EnableSTStorage` + +## Commits + +| Task | Description | Hash | +|------|-------------|------| +| 1+2 | Add V2 encoding for float-histogram WAL record types | 116b10e2a | + +## Deviations from Plan + +Tasks 1 and 2 were combined into a single edit and commit. The V2 code was fully specified in the plan and no intermediate verification was needed between them. Build and vet passed on the combined result. + +## Self-Check: PASSED + +- `tsdb/record/record.go` modified: confirmed +- commit `116b10e2a` exists: confirmed +- `go build ./tsdb/record/...` passes +- `go vet ./tsdb/record/...` passes +- `FloatHistogramSamples` dispatches to `floatHistogramSamplesV2` when `EnableSTStorage` is true +- `CustomBucketsFloatHistogramSamples` dispatches to `customBucketsFloatHistogramSamplesV2` when `EnableSTStorage` is true +- V1 bodies are exact copies of original code (wire format unchanged) +- V2 uses varint encoding, ref-delta-to-prev, T-delta-to-first, ST marker scheme diff --git a/.planning/phases/02-v2-encoders/02-CONTEXT.md b/.planning/phases/02-v2-encoders/02-CONTEXT.md new file mode 100644 index 0000000000..b9604a0020 --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-CONTEXT.md @@ -0,0 +1,92 @@ +# Phase 2: V2 Encoders - Context + +**Gathered:** 2026-03-02 +**Status:** Ready for planning + + +## Phase Boundary + +Add V2 encoder methods for all four histogram record types, gated on EnableSTStorage. Each V2 method uses the ST marker scheme (noST/sameST/explicitST). Public encoder methods dispatch to V2 when EnableSTStorage is true. No decoder changes (Phase 3). + + + + +## Implementation Decisions + +### V2 wire format +- Use varint throughout for first sample (matching samplesV2 pattern), not BE64 +- First sample: varint(ref) + varint(T) + varint(ST) + histogram payload +- Subsequent samples: varint(dRef) + varint(dT) + STmarker(1 byte) + [varint(dST)] + histogram payload +- This is a deliberate break from V1 histogram encoding which uses BE64 for base ref/time + +### Ref delta style +- Delta to previous ref (matching samplesV2), not delta to first ref +- Each sample's ref is encoded as delta from the immediately preceding sample +- T deltas remain against first T (matching samplesV2 convention) + +### ST marker scheme +- Reuse the existing noST/sameST/explicitST constants (already defined) +- First sample: ST encoded directly as varint (no marker needed) +- Subsequent samples: 1-byte marker, then optional varint delta to first ST +- Exact same logic as samplesV2's ST handling + +### Custom bucket filtering +- V2 public methods keep the same filter-and-return API contract as V1 +- HistogramSamples() with EnableSTStorage still filters out custom-bucket histograms and returns them +- FloatHistogramSamples() same pattern +- Caller encodes returned custom-bucket histograms with the corresponding CustomBuckets V2 method + +### Method signatures and dispatch +- Public methods (HistogramSamples, FloatHistogramSamples, etc.) gain EnableSTStorage gate +- HistogramSamples() and FloatHistogramSamples() return ([]byte, []RefHistogramSample/RefFloatHistogramSample) in both V1 and V2 +- CustomBuckets methods return []byte in both V1 and V2 +- Private V2 methods use (*Encoder) receiver (matching samplesV2 pattern) + +### Claude's Discretion +- Whether to extract a shared ST-encoding helper or inline the marker logic in each V2 method +- Exact comment wording on new methods +- Whether to change the Encoder struct comment to mention histogram V2 + + + + +## Existing Code Insights + +### Reusable Assets +- `samplesV2` at line 836: direct pattern to mirror for the V2 histogram encoding loop +- `noST/sameST/explicitST` constants at line 825: reuse directly +- `EncodeHistogram` at line 989: called after ref/T/ST encoding, unchanged +- `EncodeFloatHistogram` at line 1089: same, for float variants +- `Encoder.EnableSTStorage` at line 746: the gate flag, already exists + +### Established Patterns +- `Encoder.Samples()` at line 794: the V1/V2 dispatch pattern to follow +- V1 histogram encoders at lines 931-984: the existing API contract (filter custom buckets, return them) +- Private method naming: `samplesV1`, `samplesV2` (lowercase, version suffix) + +### Integration Points +- Phase 3 decoders will need to match this exact wire format +- No caller changes needed: public method signatures unchanged, EnableSTStorage flag is the only control + + + + +## Specific Ideas + +- The samplesV2 encoder uses `int64(s.Ref) - int64(prev.Ref)` for ref deltas. V2 histograms should use the same cast pattern. +- T deltas in samplesV2 are `s.T - first.T` (delta to first, not previous). V2 histograms must match this. +- The empty-slice early return should write just the type byte then return (matching both samplesV2 and V1 histogram patterns). + + + + +## Deferred Ideas + +None. Discussion stayed within phase scope. + + + +--- + +*Phase: 02-v2-encoders* +*Context gathered: 2026-03-02* diff --git a/.planning/phases/02-v2-encoders/02-RESEARCH.md b/.planning/phases/02-v2-encoders/02-RESEARCH.md new file mode 100644 index 0000000000..0f9e75f052 --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-RESEARCH.md @@ -0,0 +1,430 @@ +# Phase 2: V2 Encoders - Research + +**Researched:** 2026-03-02 +**Domain:** Go WAL record encoding, `tsdb/record/record.go` +**Confidence:** HIGH + +--- + + +## User Constraints (from CONTEXT.md) + +### Locked Decisions + +**V2 wire format** +- Use varint throughout for first sample (matching samplesV2 pattern), not BE64 +- First sample: varint(ref) + varint(T) + varint(ST) + histogram payload +- Subsequent samples: varint(dRef) + varint(dT) + STmarker(1 byte) + [varint(dST)] + histogram payload +- This is a deliberate break from V1 histogram encoding which uses BE64 for base ref/time + +**Ref delta style** +- Delta to previous ref (matching samplesV2), not delta to first ref +- Each sample's ref is encoded as delta from the immediately preceding sample +- T deltas remain against first T (matching samplesV2 convention) + +**ST marker scheme** +- Reuse the existing noST/sameST/explicitST constants (already defined) +- First sample: ST encoded directly as varint (no marker needed) +- Subsequent samples: 1-byte marker, then optional varint delta to first ST +- Exact same logic as samplesV2's ST handling + +**Custom bucket filtering** +- V2 public methods keep the same filter-and-return API contract as V1 +- HistogramSamples() with EnableSTStorage still filters out custom-bucket histograms and returns them +- FloatHistogramSamples() same pattern +- Caller encodes returned custom-bucket histograms with the corresponding CustomBuckets V2 method + +**Method signatures and dispatch** +- Public methods (HistogramSamples, FloatHistogramSamples, etc.) gain EnableSTStorage gate +- HistogramSamples() and FloatHistogramSamples() return ([]byte, []RefHistogramSample/RefFloatHistogramSample) in both V1 and V2 +- CustomBuckets methods return []byte in both V1 and V2 +- Private V2 methods use (*Encoder) receiver (matching samplesV2 pattern) + +### Claude's Discretion +- Whether to extract a shared ST-encoding helper or inline the marker logic in each V2 method +- Exact comment wording on new methods +- Whether to change the Encoder struct comment to mention histogram V2 + +### Deferred Ideas (OUT OF SCOPE) + +None. Discussion stayed within phase scope. + + +--- + + +## Phase Requirements + +| ID | Description | Research Support | +|----|-------------|-----------------| +| ENC-01 | Encoder.HistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | Public method receiver must change from `(*Encoder)` to `(e *Encoder)` to read the flag; call private `histogramSamplesV2` | +| ENC-02 | Encoder.FloatHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | Same receiver change; call private `floatHistogramSamplesV2` | +| ENC-03 | Encoder.CustomBucketsHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | Same receiver change; call private `customBucketsHistogramSamplesV2` | +| ENC-04 | Encoder.CustomBucketsFloatHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | Same receiver change; call private `customBucketsFloatHistogramSamplesV2` | +| ENC-05 | V2 histogram encoding uses noST/sameST/explicitST marker scheme | Constants already exist at line 825; wire format confirmed from samplesV2 loop | + + +--- + +## Summary + +Phase 1 is complete: `RefHistogramSample` and `RefFloatHistogramSample` both have `ST, T int64` fields, and the four V2 type constants (12-15) are defined with correct `String()` returns. The Decoder's `Type()` method already recognizes all four new types. + +Phase 2 is a pure encoder addition. The entire pattern is already demonstrated in the codebase by `samplesV2` (line 836). The work is mechanical: write four private V2 methods that mirror `samplesV2`'s structure but call `EncodeHistogram`/`EncodeFloatHistogram` instead of writing a float value, then add dispatch gates to the four public methods. + +The only structural change to existing code is that four public methods need their receiver changed from `(*Encoder)` to `(e *Encoder)` so they can read `e.EnableSTStorage`. Private V2 methods can stay as `(*Encoder)` because they do not read the flag (the flag check is in the public caller). + +**Primary recommendation:** Mirror `samplesV2` exactly for all four V2 private methods. Change the four public histogram method receivers to named `(e *Encoder)`. Add a two-branch `if e.EnableSTStorage` guard in each public method. + +--- + +## Standard Stack + +### Core +| Element | Location | Purpose | +|---------|----------|---------| +| `encoding.Encbuf` | `tsdb/encoding` | Buffer with typed Put methods — all encoding uses this | +| `buf.PutByte()` | Encbuf | Writes the record type byte | +| `buf.PutVarint64()` | Encbuf | Writes signed varint — used for ref, T, ST, dRef, dT, dST | +| `buf.PutBE64int64()` | Encbuf | Big-endian int64 — used in V1 only, NOT in V2 | +| `buf.PutBE64()` | Encbuf | Big-endian uint64 — used in V1 only, NOT in V2 | +| `buf.Reset()` | Encbuf | Resets buffer to empty — used when all samples were custom-bucket filtered | +| `buf.Get()` | Encbuf | Returns the accumulated byte slice | +| `EncodeHistogram()` | record.go:989 | Encodes `*histogram.Histogram` payload into buf | +| `EncodeFloatHistogram()` | record.go:1089 | Encodes `*histogram.FloatHistogram` payload into buf | +| `noST`, `sameST`, `explicitST` | record.go:828-830 | Byte constants for ST marker scheme | + +### No New Dependencies +No new imports are needed. All tools are already imported. + +--- + +## Architecture Patterns + +### Pattern 1: samplesV2 — the exact template to follow + +```go +// Source: record.go:836 +func (*Encoder) samplesV2(samples []RefSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(SamplesV2)) + + if len(samples) == 0 { + return buf.Get() + } + + // First sample: full varint values (no deltas, no marker) + first := samples[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + buf.PutBE64(math.Float64bits(first.V)) // <-- replace with EncodeHistogram/EncodeFloatHistogram + + // Subsequent samples: deltas + ST marker + for i := 1; i < len(samples); i++ { + s := samples[i] + prev := samples[i-1] + + buf.PutVarint64(int64(s.Ref) - int64(prev.Ref)) // delta to prev ref + buf.PutVarint64(s.T - first.T) // delta to first T + + switch s.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(s.ST - first.ST) // delta to first ST + } + buf.PutBE64(math.Float64bits(s.V)) // <-- replace with EncodeHistogram/EncodeFloatHistogram + } + return buf.Get() +} +``` + +### Pattern 2: V1 HistogramSamples — the filtering contract to preserve + +```go +// Source: record.go:931 +func (*Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(HistogramSamples)) + + if len(histograms) == 0 { + return buf.Get(), nil + } + var customBucketHistograms []RefHistogramSample + + first := histograms[0] + buf.PutBE64(uint64(first.Ref)) // V1 uses BE64 for base + buf.PutBE64int64(first.T) + + for _, h := range histograms { + if h.H.UsesCustomBuckets() { + customBucketHistograms = append(customBucketHistograms, h) + continue // skip, accumulate for caller + } + buf.PutVarint64(int64(h.Ref) - int64(first.Ref)) + buf.PutVarint64(h.T - first.T) + EncodeHistogram(&buf, h.H) + } + + // If ALL were custom buckets, reset the buffer (don't write type-byte-only record) + if len(histograms) == len(customBucketHistograms) { + buf.Reset() + } + + return buf.Get(), customBucketHistograms +} +``` + +### Pattern 3: Public dispatch gate — how to add EnableSTStorage + +```go +// Source: record.go:794 — Samples() dispatch pattern +func (e *Encoder) Samples(samples []RefSample, b []byte) []byte { + if e.EnableSTStorage { + return e.samplesV2(samples, b) + } + return e.samplesV1(samples, b) +} +``` + +Applied to HistogramSamples (currently `(*Encoder)`, must become `(e *Encoder)`): + +```go +func (e *Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + if e.EnableSTStorage { + return e.histogramSamplesV2(histograms, b) + } + return e.histogramSamplesV1(histograms, b) +} +``` + +### Recommended Project Structure + +No structural changes. All new code goes in `tsdb/record/record.go`: + +- Rename existing public bodies to `histogramSamplesV1`, `floatHistogramSamplesV1`, etc. +- Add four private V2 methods immediately after the corresponding V1 methods. +- Update four public methods to named receiver + dispatch gate. + +### Anti-Patterns to Avoid + +- **Using BE64 for first sample in V2.** V1 uses `PutBE64` / `PutBE64int64` for the base ref and T. V2 uses `PutVarint64` for everything. Mixing them breaks the decoder. +- **Ref delta to first in V2.** V1 histogram uses `h.Ref - first.Ref` throughout. V2 uses `s.Ref - prev.Ref` (delta to previous). Match `samplesV2`, not V1 histogram. +- **Writing ST marker for the first sample.** The first sample writes `PutVarint64(first.ST)` directly — no marker byte. Markers only appear for index 1+. +- **Forgetting `buf.Reset()` in V2 filtering methods.** When all histograms are custom-bucket, the buffer has only the type byte. Reset it so the caller gets an empty (not malformed) slice. +- **Not renaming existing V1 bodies.** The public methods will call V1 bodies; extract them as `histogramSamplesV1` first, then add the dispatch, to avoid duplicating logic. + +--- + +## Don't Hand-Roll + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| ST marker encoding | Custom bit-packing | noST/sameST/explicitST + PutByte + PutVarint64 | Already designed and tested in samplesV2 | +| Histogram payload encoding | Custom field serialization | EncodeHistogram / EncodeFloatHistogram | These functions handle schema, spans, buckets, custom values correctly | +| Buffer accumulation | Manual []byte appending | encoding.Encbuf | Handles growth, alignment, and all Put variants | + +--- + +## Common Pitfalls + +### Pitfall 1: Receiver change breaks compilation silently if overlooked + +**What goes wrong:** The four public methods are currently `(*Encoder)`. If you add the V2 dispatch body while forgetting to rename the receiver to `(e *Encoder)`, the compiler accepts it but `e.EnableSTStorage` is inaccessible. You'd be forced to write `e.EnableSTStorage` which the compiler will catch — so this won't silently break, but it is easy to forget. + +**How to avoid:** Change all four public method receivers at once as the first edit, before writing dispatch logic. + +### Pitfall 2: First-sample encoding diverges from V1 + +**What goes wrong:** V1 histogram encodes `first.Ref` with `PutBE64` and `first.T` with `PutBE64int64`. If you copy the V1 loop header into a V2 method without converting to `PutVarint64`, the decoder (Phase 3) will fail silently or read garbage. + +**How to avoid:** V2 first-sample header is always three varint64 calls: `PutVarint64(int64(first.Ref))`, `PutVarint64(first.T)`, `PutVarint64(first.ST)`. Then `EncodeHistogram`. + +### Pitfall 3: T delta convention differs between V1 and samplesV2 + +**What goes wrong:** V1 histogram uses `h.T - first.T` throughout the loop. samplesV2 also uses `s.T - first.T`. These agree. But if you accidentally write `s.T - prev.T` for T (following the ref-delta-to-prev pattern), the decoder will reconstruct wrong timestamps. + +**How to avoid:** T is always delta to `first.T`. Only `Ref` is delta to `prev.Ref`. + +### Pitfall 4: buf.Reset() edge case in V2 filtering methods + +**What goes wrong:** If every histogram in the input slice uses custom buckets, the V2 method writes one type byte and then filters everything. Returning that single-byte buffer would produce a record with just a type byte and no samples — technically parseable but wasteful and potentially confusing for callers. V1 resets the buffer in this case. + +**How to avoid:** After the loop, check `if len(histograms) == len(customBucketHistograms) { buf.Reset() }` — exact copy of V1. + +### Pitfall 5: CustomBuckets V2 methods must NOT filter + +**What goes wrong:** `CustomBucketsHistogramSamples` and its float variant do NOT filter — they assume all input already IS custom-bucket histograms. Their V2 counterparts must follow the same assumption and must NOT add filtering logic. + +**How to avoid:** The CustomBuckets V2 methods have no `if h.H.UsesCustomBuckets()` check. They also have no `buf.Reset()` call. They just write type byte + first sample + loop. Return type is `[]byte`, not `([]byte, []RefHistogramSample)`. + +--- + +## Code Examples + +### histogramSamplesV2 — complete implementation + +```go +// Source: derived from samplesV2 (record.go:836) + HistogramSamples (record.go:931) +func (*Encoder) histogramSamplesV2(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(HistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get(), nil + } + + var customBucketHistograms []RefHistogramSample + + first := histograms[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + EncodeHistogram(&buf, first.H) + + for i := 1; i < len(histograms); i++ { + h := histograms[i] + if h.H.UsesCustomBuckets() { + customBucketHistograms = append(customBucketHistograms, h) + continue + } + prev := histograms[i-1] + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeHistogram(&buf, h.H) + } + + if len(histograms) == len(customBucketHistograms) { + buf.Reset() + } + + return buf.Get(), customBucketHistograms +} +``` + +**Note on first-sample filtering:** The V1 method processes `first` before the loop and does not check `first.H.UsesCustomBuckets()` — if the first histogram is a custom-bucket type it gets encoded into the V1 record (this is a pre-existing V1 quirk). The V2 method should handle this consistently: write first's header fields before the filtering loop begins, then filter inside the loop. If only the non-first samples are custom buckets, first is already written. This matches V1 behavior exactly. + +### customBucketsHistogramSamplesV2 — complete implementation + +```go +// Source: derived from CustomBucketsHistogramSamples (record.go:965) + samplesV2 pattern +func (*Encoder) customBucketsHistogramSamplesV2(histograms []RefHistogramSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(CustomBucketsHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get() + } + + first := histograms[0] + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + EncodeHistogram(&buf, first.H) + + for i := 1; i < len(histograms); i++ { + h := histograms[i] + prev := histograms[i-1] + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeHistogram(&buf, h.H) + } + + return buf.Get() +} +``` + +### Public method dispatch — HistogramSamples (same pattern for all four) + +```go +// Change receiver from (*Encoder) to (e *Encoder) and extract V1 body +func (e *Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + if e.EnableSTStorage { + return e.histogramSamplesV2(histograms, b) + } + return e.histogramSamplesV1(histograms, b) +} + +// Rename original body to histogramSamplesV1 +func (*Encoder) histogramSamplesV1(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + // ... original body unchanged ... +} +``` + +--- + +## State of the Art + +| Old Approach | Current Approach | Impact | +|--------------|------------------|--------| +| V1: BE64 base ref+T, varint deltas | V2: all-varint (base and deltas) | Smaller records for typical ref/T magnitudes | +| V1: No ST field on histograms | V2: ST varint for first, marker byte for subsequent | ST propagated through WAL without overhead on trivial cases | +| Public methods as `(*Encoder)` | Public dispatch methods as `(e *Encoder)` | Enables reading EnableSTStorage flag | + +--- + +## Open Questions + +1. **First-sample custom-bucket edge case** + - What we know: V1 writes first.Ref and first.T before the filtering loop, so if histograms[0] is a custom-bucket type, it still gets a partial record written (header fields only, no payload filtered). + - What's unclear: Whether the V2 method should pre-check `first.H.UsesCustomBuckets()` and skip the header write, or follow V1 exactly. + - Recommendation: Follow V1's exact behavior (write header for first, filter in loop). Changing this would diverge from V1 semantics and the scope of this phase is V2 encoding only, not fixing V1 quirks. + +2. **Inlining vs. shared ST helper** + - What we know: The ST marker switch appears identically in all four V2 methods. + - What's unclear: Whether a small helper reduces maintenance burden enough to justify an unexported function. + - Recommendation (Claude's discretion): Inline in all four methods. The switch is 5 lines, the helper would cost an extra function signature and a `buf *encoding.Encbuf` parameter pass. Not worth it for 4 call sites. + +--- + +## Sources + +### Primary (HIGH confidence) +- `tsdb/record/record.go` lines 794-873 — `Samples()` dispatch and `samplesV2` implementation, the exact template for all V2 histogram methods +- `tsdb/record/record.go` lines 825-831 — `noST`, `sameST`, `explicitST` constants +- `tsdb/record/record.go` lines 931-987 — V1 `HistogramSamples` and `CustomBucketsHistogramSamples`, the filtering contract +- `tsdb/record/record.go` lines 1030-1087 — V1 `FloatHistogramSamples` and `CustomBucketsFloatHistogramSamples` +- `tsdb/record/record.go` lines 989-1128 — `EncodeHistogram` and `EncodeFloatHistogram` payload encoders +- `tsdb/record/record.go` lines 63-70 — Phase 1 complete: V2 type constants 12-15 confirmed present +- `tsdb/record/record.go` lines 205-216 — Phase 1 complete: `ST, T int64` fields confirmed on both sample structs + +### Secondary (MEDIUM confidence) +- `.planning/phases/02-v2-encoders/02-CONTEXT.md` — user decisions on wire format, delta conventions, ST marker scheme + +--- + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH — all tooling is existing code in the same file +- Architecture: HIGH — the exact template (`samplesV2`) exists and is directly verifiable +- Pitfalls: HIGH — derived from direct code reading, not speculation + +**Research date:** 2026-03-02 +**Valid until:** N/A — this is internal code, no external dependency versioning diff --git a/.planning/phases/02-v2-encoders/02-VERIFICATION.md b/.planning/phases/02-v2-encoders/02-VERIFICATION.md new file mode 100644 index 0000000000..673c202b79 --- /dev/null +++ b/.planning/phases/02-v2-encoders/02-VERIFICATION.md @@ -0,0 +1,86 @@ +--- +phase: 02-v2-encoders +verified: 2026-03-02T22:00:00Z +status: passed +score: 8/8 must-haves verified +--- + +# Phase 02: V2 Encoders Verification Report + +**Phase Goal:** Encoder can produce V2 histogram records with ST when EnableSTStorage is true. +**Verified:** 2026-03-02T22:00:00Z +**Status:** PASSED +**Re-verification:** No — initial verification + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +|---|-------|--------|----------| +| 1 | Encoder.HistogramSamples() dispatches to V2 when EnableSTStorage is true | VERIFIED | Line 931-936: `if e.EnableSTStorage { return e.histogramSamplesV2(histograms, b) }` | +| 2 | Encoder.CustomBucketsHistogramSamples() dispatches to V2 when EnableSTStorage is true | VERIFIED | Line 1021-1026: `if e.EnableSTStorage { return e.customBucketsHistogramSamplesV2(histograms, b) }` | +| 3 | V2 int-histogram encoding writes varint ref/T/ST for first sample and dRef/dT/STmarker for subsequent samples | VERIFIED | Lines 984-1010: PutVarint64 for first.Ref/T/ST, delta-to-prev-ref, delta-to-first-T, noST/sameST/explicitST markers | +| 4 | Custom-bucket filtering in V2 matches V1 behavior exactly | VERIFIED | histogramSamplesV2 at lines 993-996 filters via UsesCustomBuckets(); customBucketsHistogramSamplesV2 has no filter — matches V1 pattern | +| 5 | Encoder.FloatHistogramSamples() dispatches to V2 when EnableSTStorage is true | VERIFIED | Line 1131-1136: `if e.EnableSTStorage { return e.floatHistogramSamplesV2(histograms, b) }` | +| 6 | Encoder.CustomBucketsFloatHistogramSamples() dispatches to V2 when EnableSTStorage is true | VERIFIED | Line 1220-1225: `if e.EnableSTStorage { return e.customBucketsFloatHistogramSamplesV2(histograms, b) }` | +| 7 | V2 float-histogram encoding writes varint ref/T/ST for first sample and dRef/dT/STmarker for subsequent samples | VERIFIED | Lines 1184-1210: PutVarint64 for first.Ref/T/ST, EncodeFloatHistogram, delta-to-prev-ref, delta-to-first-T, ST markers | +| 8 | Custom-bucket filtering in V2 float-histogram matches V1 behavior exactly | VERIFIED | floatHistogramSamplesV2 filters via UsesCustomBuckets(); customBucketsFloatHistogramSamplesV2 has no filter | + +**Score:** 8/8 truths verified + +### Required Artifacts + +| Artifact | Expected | Status | Details | +|----------|----------|--------|---------| +| `tsdb/record/record.go` | 8 new private methods + 4 updated dispatch methods | VERIFIED | All 8 functions present at lines 938, 972, 1028, 1052, 1138, 1173, 1227, 1251 | + +**Artifact depth check:** +- Exists: yes +- Substantive: yes — each V2 method is 20-30 lines with real encoding logic, not stubs +- Wired: yes — all 4 public dispatch methods call their V2 counterpart via `e.EnableSTStorage` guard + +### Key Link Verification + +| From | To | Via | Status | Details | +|------|----|-----|--------|---------| +| HistogramSamples() | histogramSamplesV2 | e.EnableSTStorage check | WIRED | Line 932-933 | +| CustomBucketsHistogramSamples() | customBucketsHistogramSamplesV2 | e.EnableSTStorage check | WIRED | Line 1022-1023 | +| histogramSamplesV2 | EncodeHistogram | EncodeHistogram(&buf, h.H) | WIRED | Lines 987, 1010 | +| FloatHistogramSamples() | floatHistogramSamplesV2 | e.EnableSTStorage check | WIRED | Line 1132-1133 | +| CustomBucketsFloatHistogramSamples() | customBucketsFloatHistogramSamplesV2 | e.EnableSTStorage check | WIRED | Line 1221-1222 | +| floatHistogramSamplesV2 | EncodeFloatHistogram | EncodeFloatHistogram(&buf, h.FH) | WIRED | Lines 1187, 1210 | + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +|-------------|------------|-------------|--------|----------| +| ENC-01 | 02-01 | Encoder.HistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | SATISFIED | Line 931-936 in record.go | +| ENC-02 | 02-02 | Encoder.FloatHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | SATISFIED | Line 1131-1136 in record.go | +| ENC-03 | 02-01 | Encoder.CustomBucketsHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | SATISFIED | Line 1021-1026 in record.go | +| ENC-04 | 02-02 | Encoder.CustomBucketsFloatHistogramSamples() gates on EnableSTStorage, dispatches to V2 when enabled | SATISFIED | Line 1220-1225 in record.go | +| ENC-05 | 02-01, 02-02 | V2 histogram encoding uses noST/sameST/explicitST marker scheme | SATISFIED | All four V2 methods use noST/sameST/explicitST at lines 1003-1008, 1077-1082, 1200-1205, 1274-1279 | + +All 5 requirement IDs from REQUIREMENTS.md are satisfied. No orphaned requirements. + +### Anti-Patterns Found + +None. No TODOs, FIXMEs, placeholder returns, or empty handlers found in the modified file sections. + +### Build / Vet + +`go build ./tsdb/record/...` — PASS +`go vet ./tsdb/record/...` — PASS + +### Human Verification Required + +None. All behaviors are mechanically verifiable via code inspection. + +## Summary + +Phase 02 goal is fully achieved. All four public histogram encoder methods (HistogramSamples, CustomBucketsHistogramSamples, FloatHistogramSamples, CustomBucketsFloatHistogramSamples) dispatch to V2 implementations when `EnableSTStorage` is true. All V2 methods correctly implement the varint wire format with the noST/sameST/explicitST ST marker scheme. V1 paths are intact as private extractions. Build and vet pass. All 5 ENC requirements satisfied. + +--- + +_Verified: 2026-03-02T22:00:00Z_ +_Verifier: Claude (gsd-verifier)_ diff --git a/.planning/phases/03-v2-decoders/03-01-PLAN.md b/.planning/phases/03-v2-decoders/03-01-PLAN.md new file mode 100644 index 0000000000..7435532250 --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-01-PLAN.md @@ -0,0 +1,230 @@ +--- +phase: 03-v2-decoders +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: [tsdb/record/record.go] +autonomous: true +requirements: [DEC-01, DEC-03, DEC-04] + +must_haves: + truths: + - "Decoder.HistogramSamples() accepts V2 record types (HistogramSamplesV2, CustomBucketsHistogramSamplesV2) and decodes them correctly" + - "V2 int-histogram decoding reads ST marker bytes and reconstructs ST values using noST/sameST/explicitST scheme" + - "V1 int-histogram records still decode with ST=0 (backward compat, zero value)" + artifacts: + - path: "tsdb/record/record.go" + provides: "histogramSamplesV2 private method + updated HistogramSamples dispatch" + contains: "func (d *Decoder) histogramSamplesV2" + key_links: + - from: "Decoder.HistogramSamples()" + to: "histogramSamplesV2" + via: "switch on type byte" + pattern: "case HistogramSamplesV2, CustomBucketsHistogramSamplesV2" + - from: "histogramSamplesV2" + to: "DecodeHistogram" + via: "function call after ref/T/ST decode" + pattern: "DecodeHistogram\\(&dec, rh\\.H\\)" +--- + + +Add V2 int-histogram decoder: private `histogramSamplesV2` method and updated `HistogramSamples()` dispatch. + +Purpose: Enable decoding of V2 int-histogram WAL records (with ST support) while preserving V1 backward compatibility. +Output: Updated `tsdb/record/record.go` with working int-histogram V2 decode path. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/03-v2-decoders/03-CONTEXT.md +@.planning/phases/03-v2-decoders/03-RESEARCH.md +@.planning/phases/02-v2-encoders/02-01-SUMMARY.md + + + + +From tsdb/record/record.go (dispatch pattern to follow, line 336): +```go +func (d *Decoder) Samples(rec []byte, samples []RefSample) ([]RefSample, error) { + dec := encoding.Decbuf{B: rec} + switch typ := dec.Byte(); Type(typ) { + case Samples: + return d.samplesV1(&dec, samples) + case SamplesV2: + return d.samplesV2(&dec, samples) + default: + return nil, fmt.Errorf("invalid record type %v, expected Samples(2) or SamplesV2(11)", typ) + } +} +``` + +From tsdb/record/record.go (samplesV2 decoder, lines 384-434 - the pattern to mirror): +```go +func (*Decoder) samplesV2(dec *encoding.Decbuf, samples []RefSample) ([]RefSample, error) { + if dec.Len() == 0 { + return samples, nil + } + var firstT, firstST int64 + for len(dec.B) > 0 && dec.Err() == nil { + var prev RefSample + var ref, t, ST int64 + if len(samples) == 0 { + ref = dec.Varint64() + firstT = dec.Varint64() + t = firstT + ST = dec.Varint64() + firstST = ST + } else { + prev = samples[len(samples)-1] + ref = int64(prev.Ref) + dec.Varint64() + t = firstT + dec.Varint64() + stMarker := dec.Byte() + switch stMarker { + case noST: + case sameST: + ST = prev.ST + default: + ST = firstST + dec.Varint64() + } + } + // ... then value decode ... + } +} +``` + +From tsdb/record/record.go (current V1 HistogramSamples, lines 529-577): +```go +func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) ([]RefHistogramSample, error) { + dec := encoding.Decbuf{B: rec} + t := Type(dec.Byte()) + if t != HistogramSamples && t != CustomBucketsHistogramSamples { + return nil, errors.New("invalid record type") + } + if dec.Len() == 0 { + return histograms, nil + } + var ( + baseRef = dec.Be64() + baseTime = dec.Be64int64() + ) + for len(dec.B) > 0 && dec.Err() == nil { + dref := dec.Varint64() + dtime := dec.Varint64() + rh := RefHistogramSample{ + Ref: chunks.HeadSeriesRef(baseRef + uint64(dref)), + T: baseTime + dtime, + H: &histogram.Histogram{}, + } + DecodeHistogram(&dec, rh.H) + // ... schema validation ... + histograms = append(histograms, rh) + } + // ... error checks ... +} +``` + +ST marker constants (already defined): +```go +const ( + noST byte = 0 + sameST byte = 1 + // default case = explicitST (delta from firstST) +) +``` + +V2 type constants (from Phase 1): +```go +HistogramSamplesV2 Type = 12 +CustomBucketsHistogramSamplesV2 Type = 14 +``` + + + + + + + Task 1: Extract V1 int-histogram decoder into private method and add V2 decoder + tsdb/record/record.go + +Two changes to `Decoder` in tsdb/record/record.go: + +1. **Extract V1 body into `histogramSamplesV1`:** Create a private method `func (d *Decoder) histogramSamplesV1(dec *encoding.Decbuf, histograms []RefHistogramSample) ([]RefHistogramSample, error)` containing the exact current body of `HistogramSamples` starting from the `if dec.Len() == 0` check through the return. This is a pure mechanical extraction with zero logic changes. The V1 body reads BE64 baseRef/baseTime then loops with varint dref/dtime. Preserve the schema validation (IsKnownSchema + ReduceResolution) exactly. + +2. **Create `histogramSamplesV2`:** Create a private method `func (d *Decoder) histogramSamplesV2(dec *encoding.Decbuf, histograms []RefHistogramSample) ([]RefHistogramSample, error)` modeled on `samplesV2` (lines 384-434) but adapted for histograms: + - Early return if `dec.Len() == 0` + - Track `firstT` and `firstST` as loop-outer variables + - First iteration (when `len(histograms) == 0`): read absolute `ref = dec.Varint64()`, `firstT = dec.Varint64()`, `t = firstT`, `ST = dec.Varint64()`, `firstST = ST` + - Subsequent iterations: `prev = histograms[len(histograms)-1]`, `ref = int64(prev.Ref) + dec.Varint64()`, `t = firstT + dec.Varint64()`, then read stMarker byte with switch: `noST` (ST stays zero), `sameST` (ST = prev.ST), `default` (ST = firstST + dec.Varint64()) + - Build `RefHistogramSample{Ref: chunks.HeadSeriesRef(ref), ST: ST, T: t, H: &histogram.Histogram{}}` + - Call `DecodeHistogram(&dec, rh.H)` (note: pass `&dec` not `dec`, matching V1 pattern at line 552) + - Preserve schema validation: `if !histogram.IsKnownSchema(rh.H.Schema)` warn-and-continue, then `ReduceResolution` check. Copy this block verbatim from the V1 body. + - Append to histograms slice + - After loop: check `dec.Err()` and `len(dec.B) > 0`, same error format as V1 + +CRITICAL: Do NOT read BE64 in the V2 path. V2 wire format is all-varint. Do NOT use delta-to-first for ref (use delta-to-prev). T uses delta-to-first. Do NOT forget schema validation in the V2 path. + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + histogramSamplesV1 and histogramSamplesV2 private methods exist. V1 is a pure extraction of the previous inline body. V2 mirrors samplesV2 with histogram payload decode and schema validation. + + + + Task 2: Update HistogramSamples() to dispatch V1/V2 via switch + tsdb/record/record.go + +Replace the body of `Decoder.HistogramSamples()` (currently at line 529) with a switch dispatch matching the `Decoder.Samples()` pattern (line 336): + +```go +func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) ([]RefHistogramSample, error) { + dec := encoding.Decbuf{B: rec} + switch typ := Type(dec.Byte()); typ { + case HistogramSamples, CustomBucketsHistogramSamples: + return d.histogramSamplesV1(&dec, histograms) + case HistogramSamplesV2, CustomBucketsHistogramSamplesV2: + return d.histogramSamplesV2(&dec, histograms) + default: + return nil, fmt.Errorf("invalid record type %v", typ) + } +} +``` + +The old body (type guard + inline V1 logic) is entirely replaced by this 10-line dispatch. All logic now lives in the private methods from Task 1. + +Note: The error message format changes from `errors.New("invalid record type")` to `fmt.Errorf("invalid record type %v", typ)` which is an improvement (includes the actual type value for debugging), matching the pattern in `Decoder.Samples()`. + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + HistogramSamples() dispatches to V1 for types HistogramSamples/CustomBucketsHistogramSamples and V2 for HistogramSamplesV2/CustomBucketsHistogramSamplesV2. V1 path unchanged (ST defaults to zero). Build and vet pass. + + + + + +- `go build ./tsdb/record/...` compiles without errors +- `go vet ./tsdb/record/...` passes +- `Decoder.HistogramSamples()` has switch dispatch on type byte +- `histogramSamplesV2` method exists with all-varint decode, ST marker scheme, schema validation +- V1 path is pure mechanical extraction (no logic changes) + + + +- HistogramSamples() accepts V1 types (HistogramSamples, CustomBucketsHistogramSamples) and V2 types (HistogramSamplesV2, CustomBucketsHistogramSamplesV2) +- V2 decoder reads varint ref/T/ST for first sample, delta-ref/delta-T/ST-marker for subsequent samples +- Schema validation preserved in V2 path (IsKnownSchema + ReduceResolution) +- V1 records decode identically to before (ST=0, zero value default) +- Code compiles and passes vet + + + +After completion, create `.planning/phases/03-v2-decoders/03-01-SUMMARY.md` + diff --git a/.planning/phases/03-v2-decoders/03-01-SUMMARY.md b/.planning/phases/03-v2-decoders/03-01-SUMMARY.md new file mode 100644 index 0000000000..704691d320 --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-01-SUMMARY.md @@ -0,0 +1,97 @@ +--- +phase: 03-v2-decoders +plan: 01 +subsystem: tsdb/record +tags: [decoder, v2, histogram, wal, start-time, varint] + +requires: + - phase: 02-v2-encoders + provides: V2 wire format (all-varint, ST markers) and type constants + - phase: 01-struct-and-type-definitions + provides: ST field on RefHistogramSample, V2 type constants +provides: + - histogramSamplesV1 private decoder method (extracted from HistogramSamples) + - histogramSamplesV2 private decoder method (all-varint, ST marker scheme) + - HistogramSamples() V1/V2 switch dispatch +affects: [03-02, 04-tests] + +tech-stack: + added: [] + patterns: [v2-histogram-decode-dispatch, varint-decoding, st-marker-reconstruction] + +key-files: + created: [] + modified: [tsdb/record/record.go] + +key-decisions: + - "Extracted V1 body into histogramSamplesV1 private method for clean switch dispatch (Claude's discretion)" + - "V2 decoder mirrors samplesV2 exactly: all-varint, ref-delta-to-prev, T-delta-to-first, ST marker scheme" + +patterns-established: + - "V2 histogram decoder dispatch: switch on type byte, V1 and V2 cases, matching Decoder.Samples() pattern" + +requirements-completed: [DEC-01, DEC-03, DEC-04] + +duration: 2min +completed: 2026-03-02 +--- + +# Phase 03 Plan 01: V2 Int-Histogram Decoder Summary + +**V2 int-histogram decoder with all-varint wire format, ST marker reconstruction, and V1/V2 switch dispatch in HistogramSamples()** + +## Performance + +- **Duration:** ~2 min +- **Started:** 2026-03-02T22:03:41Z +- **Completed:** 2026-03-02T22:05:12Z +- **Tasks:** 2 +- **Files modified:** 1 + +## Accomplishments +- Extracted V1 int-histogram decode logic into `histogramSamplesV1` private method (pure mechanical extraction, zero logic changes) +- Added `histogramSamplesV2` private method with all-varint decode, ST marker reconstruction (noST/sameST/explicitST), and schema validation +- Updated `HistogramSamples()` public method to switch-dispatch V1 and V2 record types + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Extract V1 into private method and add V2 decoder** - `4c9118810` (feat) +2. **Task 2: Update HistogramSamples() to dispatch V1/V2 via switch** - `060c6f13e` (feat) + +## Files Created/Modified +- `tsdb/record/record.go` - Added histogramSamplesV1 (V1 extraction), histogramSamplesV2 (V2 decoder), updated HistogramSamples() dispatch + +## Decisions Made +- Extracted V1 body into a private method (Claude's discretion per CONTEXT.md) for consistency with Decoder.Samples() pattern and cleaner switch dispatch +- Error message in default case now includes the actual type value (`fmt.Errorf("invalid record type %v", typ)`) matching the Decoder.Samples() pattern + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered +None + +## User Setup Required +None - no external service configuration required. + +## Next Phase Readiness +- V2 int-histogram decoder complete, ready for Plan 02 (V2 float-histogram decoder) +- Same pattern applies: extract V1 into floatHistogramSamplesV1, add floatHistogramSamplesV2, update FloatHistogramSamples() dispatch + +## Self-Check: PASSED + +- `03-01-SUMMARY.md` exists: confirmed +- commit `4c9118810` exists: confirmed +- commit `060c6f13e` exists: confirmed +- `go build ./tsdb/record/...` passes +- `go vet ./tsdb/record/...` passes +- `HistogramSamples()` has switch dispatch on type byte +- `histogramSamplesV2` method exists with all-varint decode, ST marker scheme, schema validation +- V1 path is pure mechanical extraction (no logic changes) + +--- +*Phase: 03-v2-decoders* +*Completed: 2026-03-02* diff --git a/.planning/phases/03-v2-decoders/03-02-PLAN.md b/.planning/phases/03-v2-decoders/03-02-PLAN.md new file mode 100644 index 0000000000..b626bbefc2 --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-02-PLAN.md @@ -0,0 +1,210 @@ +--- +phase: 03-v2-decoders +plan: 02 +type: execute +wave: 2 +depends_on: [03-01] +files_modified: [tsdb/record/record.go] +autonomous: true +requirements: [DEC-02, DEC-03, DEC-04] + +must_haves: + truths: + - "Decoder.FloatHistogramSamples() accepts V2 record types (FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2) and decodes them correctly" + - "V2 float-histogram decoding reads ST marker bytes and reconstructs ST values using noST/sameST/explicitST scheme" + - "V1 float-histogram records still decode with ST=0 (backward compat, zero value)" + artifacts: + - path: "tsdb/record/record.go" + provides: "floatHistogramSamplesV2 private method + updated FloatHistogramSamples dispatch" + contains: "func (d *Decoder) floatHistogramSamplesV2" + key_links: + - from: "Decoder.FloatHistogramSamples()" + to: "floatHistogramSamplesV2" + via: "switch on type byte" + pattern: "case FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2" + - from: "floatHistogramSamplesV2" + to: "DecodeFloatHistogram" + via: "function call after ref/T/ST decode" + pattern: "DecodeFloatHistogram\\(&dec, rh\\.FH\\)" +--- + + +Add V2 float-histogram decoder: private `floatHistogramSamplesV2` method and updated `FloatHistogramSamples()` dispatch. + +Purpose: Enable decoding of V2 float-histogram WAL records (with ST support) while preserving V1 backward compatibility. Completes all four histogram decoder V2 paths. +Output: Updated `tsdb/record/record.go` with working float-histogram V2 decode path. All DEC requirements satisfied. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/03-v2-decoders/03-CONTEXT.md +@.planning/phases/03-v2-decoders/03-RESEARCH.md +@.planning/phases/03-v2-decoders/03-01-SUMMARY.md + + + + + +From Plan 01 (histogramSamplesV2 pattern - mirror this for float): +```go +func (d *Decoder) histogramSamplesV2(dec *encoding.Decbuf, histograms []RefHistogramSample) ([]RefHistogramSample, error) { + if dec.Len() == 0 { + return histograms, nil + } + var firstT, firstST int64 + for len(dec.B) > 0 && dec.Err() == nil { + var ref, t, ST int64 + if len(histograms) == 0 { + ref = dec.Varint64() + firstT = dec.Varint64() + t = firstT + ST = dec.Varint64() + firstST = ST + } else { + prev := histograms[len(histograms)-1] + ref = int64(prev.Ref) + dec.Varint64() + t = firstT + dec.Varint64() + stMarker := dec.Byte() + switch stMarker { + case noST: + case sameST: + ST = prev.ST + default: + ST = firstST + dec.Varint64() + } + } + rh := RefHistogramSample{Ref: chunks.HeadSeriesRef(ref), ST: ST, T: t, H: &histogram.Histogram{}} + DecodeHistogram(&dec, rh.H) + // schema validation... + histograms = append(histograms, rh) + } + // error checks... +} +``` + +From tsdb/record/record.go (current V1 FloatHistogramSamples, lines 635-683): +```go +func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error) { + dec := encoding.Decbuf{B: rec} + t := Type(dec.Byte()) + if t != FloatHistogramSamples && t != CustomBucketsFloatHistogramSamples { + return nil, errors.New("invalid record type") + } + if dec.Len() == 0 { + return histograms, nil + } + var ( + baseRef = dec.Be64() + baseTime = dec.Be64int64() + ) + for len(dec.B) > 0 && dec.Err() == nil { + dref := dec.Varint64() + dtime := dec.Varint64() + rh := RefFloatHistogramSample{ + Ref: chunks.HeadSeriesRef(baseRef + uint64(dref)), + T: baseTime + dtime, + FH: &histogram.FloatHistogram{}, + } + DecodeFloatHistogram(&dec, rh.FH) + // ... schema validation ... + histograms = append(histograms, rh) + } + // ... error checks ... +} +``` + +V2 type constants (from Phase 1): +```go +FloatHistogramSamplesV2 Type = 13 +CustomBucketsFloatHistogramSamplesV2 Type = 15 +``` + + + + + + + Task 1: Extract V1 float-histogram decoder into private method and add V2 decoder + tsdb/record/record.go + +Two changes to `Decoder` in tsdb/record/record.go: + +1. **Extract V1 body into `floatHistogramSamplesV1`:** Create a private method `func (d *Decoder) floatHistogramSamplesV1(dec *encoding.Decbuf, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error)` containing the exact current body of `FloatHistogramSamples` starting from the `if dec.Len() == 0` check through the return. Pure mechanical extraction, zero logic changes. The V1 body reads BE64 baseRef/baseTime then loops with varint dref/dtime. Preserve the schema validation (IsKnownSchema + ReduceResolution on `rh.FH`) exactly. + +2. **Create `floatHistogramSamplesV2`:** Create a private method `func (d *Decoder) floatHistogramSamplesV2(dec *encoding.Decbuf, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error)` identical in structure to `histogramSamplesV2` from Plan 01 but adapted for float histograms: + - Early return if `dec.Len() == 0` + - Track `firstT` and `firstST` as loop-outer variables + - First iteration (when `len(histograms) == 0`): read absolute `ref = dec.Varint64()`, `firstT = dec.Varint64()`, `t = firstT`, `ST = dec.Varint64()`, `firstST = ST` + - Subsequent iterations: `prev = histograms[len(histograms)-1]`, `ref = int64(prev.Ref) + dec.Varint64()`, `t = firstT + dec.Varint64()`, then read stMarker byte with switch: `noST` (ST stays zero), `sameST` (ST = prev.ST), `default` (ST = firstST + dec.Varint64()) + - Build `RefFloatHistogramSample{Ref: chunks.HeadSeriesRef(ref), ST: ST, T: t, FH: &histogram.FloatHistogram{}}` + - Call `DecodeFloatHistogram(&dec, rh.FH)` (note: `&dec`, and `rh.FH` not `rh.H`) + - Preserve schema validation: `if !histogram.IsKnownSchema(rh.FH.Schema)` warn-and-continue, then `rh.FH.ReduceResolution` check. Copy this block verbatim from the V1 float-histogram body (note: accesses `rh.FH.Schema` not `rh.H.Schema`). + - Append to histograms slice + - After loop: check `dec.Err()` and `len(dec.B) > 0`, same error format as V1 + +CRITICAL: Same guardrails as Plan 01. No BE64 in V2 path. Delta-to-prev for ref, delta-to-first for T. Schema validation must be present. Use `rh.FH` everywhere (not `rh.H`). + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + floatHistogramSamplesV1 and floatHistogramSamplesV2 private methods exist. V1 is a pure extraction of the previous inline body. V2 mirrors histogramSamplesV2 with float-histogram payload decode and schema validation. + + + + Task 2: Update FloatHistogramSamples() to dispatch V1/V2 via switch + tsdb/record/record.go + +Replace the body of `Decoder.FloatHistogramSamples()` (currently at line 635) with a switch dispatch matching the pattern established by Plan 01's `HistogramSamples()`: + +```go +func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error) { + dec := encoding.Decbuf{B: rec} + switch typ := Type(dec.Byte()); typ { + case FloatHistogramSamples, CustomBucketsFloatHistogramSamples: + return d.floatHistogramSamplesV1(&dec, histograms) + case FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2: + return d.floatHistogramSamplesV2(&dec, histograms) + default: + return nil, fmt.Errorf("invalid record type %v", typ) + } +} +``` + +The old body (type guard + inline V1 logic) is entirely replaced by this dispatch. All logic now lives in the private methods from Task 1. + + + cd /home/owilliams/src/grafana/prometheus && go build ./tsdb/record/... && go vet ./tsdb/record/... + + FloatHistogramSamples() dispatches to V1 for types FloatHistogramSamples/CustomBucketsFloatHistogramSamples and V2 for FloatHistogramSamplesV2/CustomBucketsFloatHistogramSamplesV2. V1 path unchanged (ST defaults to zero). Build and vet pass. + + + + + +- `go build ./tsdb/record/...` compiles without errors +- `go vet ./tsdb/record/...` passes +- `Decoder.FloatHistogramSamples()` has switch dispatch on type byte +- `floatHistogramSamplesV2` method exists with all-varint decode, ST marker scheme, schema validation +- V1 path is pure mechanical extraction (no logic changes) +- All four histogram decoder methods (int V1, int V2, float V1, float V2) exist + + + +- FloatHistogramSamples() accepts V1 types (FloatHistogramSamples, CustomBucketsFloatHistogramSamples) and V2 types (FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2) +- V2 decoder reads varint ref/T/ST for first sample, delta-ref/delta-T/ST-marker for subsequent samples +- Schema validation preserved in V2 path (IsKnownSchema + ReduceResolution on FH) +- V1 records decode identically to before (ST=0, zero value default) +- Code compiles and passes vet +- All DEC-01 through DEC-04 requirements are satisfied across Plans 01 and 02 + + + +After completion, create `.planning/phases/03-v2-decoders/03-02-SUMMARY.md` + diff --git a/.planning/phases/03-v2-decoders/03-02-SUMMARY.md b/.planning/phases/03-v2-decoders/03-02-SUMMARY.md new file mode 100644 index 0000000000..1057973412 --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-02-SUMMARY.md @@ -0,0 +1,102 @@ +--- +phase: 03-v2-decoders +plan: 02 +subsystem: tsdb/record +tags: [decoder, v2, float-histogram, wal, start-time, varint] + +requires: + - phase: 02-v2-encoders + provides: V2 wire format (all-varint, ST markers) and type constants + - phase: 01-struct-and-type-definitions + provides: ST field on RefFloatHistogramSample, V2 type constants + - phase: 03-v2-decoders-plan-01 + provides: histogramSamplesV2 pattern and V1/V2 dispatch pattern +provides: + - floatHistogramSamplesV1 private decoder method (extracted from FloatHistogramSamples) + - floatHistogramSamplesV2 private decoder method (all-varint, ST marker scheme) + - FloatHistogramSamples() V1/V2 switch dispatch +affects: [04-tests] + +tech-stack: + added: [] + patterns: [v2-float-histogram-decode-dispatch, varint-decoding, st-marker-reconstruction] + +key-files: + created: [] + modified: [tsdb/record/record.go] + +key-decisions: + - "Extracted V1 body into floatHistogramSamplesV1 private method, mirroring Plan 01 pattern" + - "V2 decoder mirrors histogramSamplesV2 exactly: all-varint, ref-delta-to-prev, T-delta-to-first, ST marker scheme, adapted for FloatHistogram payload" + +patterns-established: + - "All four histogram decoder methods (int V1/V2, float V1/V2) now follow consistent private-method extraction and switch dispatch" + +requirements-completed: [DEC-02, DEC-03, DEC-04] + +duration: 2min +completed: 2026-03-02 +--- + +# Phase 03 Plan 02: V2 Float-Histogram Decoder Summary + +**V2 float-histogram decoder with all-varint wire format, ST marker reconstruction, and V1/V2 switch dispatch in FloatHistogramSamples()** + +## Performance + +- **Duration:** ~2 min +- **Started:** 2026-03-02T22:07:21Z +- **Completed:** 2026-03-02T22:08:49Z +- **Tasks:** 2 +- **Files modified:** 1 + +## Accomplishments +- Extracted V1 float-histogram decode logic into `floatHistogramSamplesV1` private method (pure mechanical extraction, zero logic changes) +- Added `floatHistogramSamplesV2` private method with all-varint decode, ST marker reconstruction (noST/sameST/explicitST), and schema validation +- Updated `FloatHistogramSamples()` public method to switch-dispatch V1 and V2 record types +- All four histogram decoder V2 paths now complete (int V1/V2, float V1/V2) + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Extract V1 float-histogram decoder into private method and add V2 decoder** - `c1e5db311` (feat) +2. **Task 2: Update FloatHistogramSamples() to dispatch V1/V2 via switch** - `02214cd2d` (feat) + +## Files Created/Modified +- `tsdb/record/record.go` - Added floatHistogramSamplesV1 (V1 extraction), floatHistogramSamplesV2 (V2 decoder), updated FloatHistogramSamples() dispatch + +## Decisions Made +- Extracted V1 body into a private method (Claude's discretion per CONTEXT.md) for consistency with HistogramSamples() pattern from Plan 01 +- Error message in default case includes actual type value (`fmt.Errorf("invalid record type %v", typ)`) matching the HistogramSamples() and Samples() dispatch pattern + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered +None + +## User Setup Required +None - no external service configuration required. + +## Next Phase Readiness +- All V2 decoder paths complete (samples, int-histogram, float-histogram) +- Ready for Phase 4 (round-trip encode/decode tests) +- DEC-01 through DEC-04 requirements satisfied across Plans 01 and 02 + +## Self-Check: PASSED + +- `03-02-SUMMARY.md` exists: confirmed +- commit `c1e5db311` exists: confirmed +- commit `02214cd2d` exists: confirmed +- `go build ./tsdb/record/...` passes +- `go vet ./tsdb/record/...` passes +- `FloatHistogramSamples()` has switch dispatch on type byte +- `floatHistogramSamplesV1` method exists (V1 extraction) +- `floatHistogramSamplesV2` method exists with all-varint decode, ST marker scheme, schema validation +- All four histogram decoder methods (int V1/V2, float V1/V2) present + +--- +*Phase: 03-v2-decoders* +*Completed: 2026-03-02* diff --git a/.planning/phases/03-v2-decoders/03-CONTEXT.md b/.planning/phases/03-v2-decoders/03-CONTEXT.md new file mode 100644 index 0000000000..ac0d91afbd --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-CONTEXT.md @@ -0,0 +1,89 @@ +# Phase 3: V2 Decoders - Context + +**Gathered:** 2026-03-02 +**Status:** Ready for planning + + +## Phase Boundary + +Add V2 decoder methods for all four histogram record types. Decoder.HistogramSamples() and Decoder.FloatHistogramSamples() must accept both V1 and V2 record types. V1 records decode with ST=0 (backward compat). No encoder changes. + + + + +## Implementation Decisions + +### V2 wire format (locked from Phase 2) +- First sample: varint(ref) + varint(T) + varint(ST) + histogram payload +- Subsequent samples: varint(dRef from prev) + varint(dT from first) + STmarker(1 byte) + [varint(dST from first)] + histogram payload +- Decoder must mirror this exactly + +### Dispatch pattern +- Decoder.HistogramSamples() already reads the type byte and switches on it +- Add V2 types to the switch: HistogramSamplesV2 and CustomBucketsHistogramSamplesV2 dispatch to a new histogramSamplesV2 private method +- Same for FloatHistogramSamples(): add FloatHistogramSamplesV2 and CustomBucketsFloatHistogramSamplesV2 +- V1 path unchanged (ST defaults to zero value in the struct) + +### Ref reconstruction +- Delta to previous ref: `ref = int64(prev.Ref) + dec.Varint64()` (matching samplesV2 decoder) +- First sample: `ref = dec.Varint64()` (absolute, no delta) + +### ST reconstruction +- First sample: `ST = dec.Varint64()` (absolute) +- Subsequent: read marker byte, then noST (ST=0), sameST (ST=prev.ST), explicitST (ST=firstST+delta) +- Exact same logic as samplesV2 decoder + +### Schema validation +- V2 decoder must preserve the existing schema validation from V1: IsKnownSchema check, ReduceResolution for high-resolution histograms +- Same warn-and-skip behavior for unknown schemas + +### Backward compatibility +- V1 records decode with ST=0 (zero value, no change to V1 path) +- New decoders accept both V1 and V2 type bytes + +### Claude's Discretion +- Whether to extract V1 decoder bodies into private methods (like encoders did) or leave inline +- Exact error message wording for V2 decode failures + + + + +## Existing Code Insights + +### Reusable Assets +- `samplesV2` decoder at line 384: direct pattern to mirror +- `DecodeHistogram` function: called after ref/T/ST decoding, unchanged +- `DecodeFloatHistogram` function: same for float variants +- Existing `Decoder.HistogramSamples()` at line 529: add V2 dispatch here +- Existing `Decoder.FloatHistogramSamples()`: add V2 dispatch here + +### Established Patterns +- `Decoder.Samples()` at line 320: the V1/V2 dispatch pattern (switch on type byte) +- V1 histogram decoder reads Be64 for baseRef/baseTime, then varint deltas +- Schema validation + ReduceResolution logic must be preserved in V2 path + +### Integration Points +- Phase 4 tests will round-trip encode/decode to verify correctness +- No caller changes needed: public method signatures unchanged + + + + +## Specific Ideas + +- The samplesV2 decoder tracks `firstT` and `firstST` as running state outside the loop. V2 histogram decoders need the same. +- The "skipped custom-bucket" edge case in the encoder means the decoder's `prev` ref tracks the last decoded sample, not the last encoded one. This is fine because the decoder only sees non-skipped samples in its byte stream. + + + + +## Deferred Ideas + +None. Discussion stayed within phase scope. + + + +--- + +*Phase: 03-v2-decoders* +*Context gathered: 2026-03-02* diff --git a/.planning/phases/03-v2-decoders/03-RESEARCH.md b/.planning/phases/03-v2-decoders/03-RESEARCH.md new file mode 100644 index 0000000000..ef35b84aff --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-RESEARCH.md @@ -0,0 +1,270 @@ +# Phase 3: V2 Decoders - Research + +**Researched:** 2026-03-02 +**Domain:** Go WAL record decoding, tsdb/record/record.go +**Confidence:** HIGH + +## Summary + +Phase 3 adds V2 decoder support to `Decoder.HistogramSamples()` and `Decoder.FloatHistogramSamples()`. The wire format is already locked by Phase 2. The implementation is a direct mirror of `samplesV2` (lines 384-434), adapted for histograms instead of scalar samples. + +The existing V1 decoder bodies read a BE64 baseRef/baseTime then loop over varint deltas. V2 replaces that with all-varint first sample, ref-delta-to-prev, T-delta-to-first, and a 1-byte ST marker for subsequent samples. The schema validation and `ReduceResolution` logic from V1 must be preserved unchanged in the V2 path. + +**Primary recommendation:** Add `histogramSamplesV2` and `floatHistogramSamplesV2` private methods modeled exactly on `samplesV2`. Update the two public methods to switch on type byte (matching `Decoder.Samples()` dispatch at line 336). Leave V1 paths completely untouched. + + +## User Constraints (from CONTEXT.md) + +### Locked Decisions + +- V2 wire format mirrors encoder exactly: first sample is varint(ref) + varint(T) + varint(ST) + histogram payload; subsequent samples are varint(dRef from prev) + varint(dT from first) + STmarker(1 byte) + [varint(dST from first)] + histogram payload +- Dispatch pattern: add V2 types to switch in HistogramSamples() and FloatHistogramSamples(); V2 types dispatch to new private methods +- Ref reconstruction: first sample absolute varint, subsequent samples delta to prev ref +- ST reconstruction: first sample absolute varint, subsequent samples use noST/sameST/explicitST marker; explicitST = firstST + delta +- Schema validation preserved: IsKnownSchema check + ReduceResolution for high-res histograms, same warn-and-skip behavior +- V1 path unchanged; V1 records decode with ST=0 + +### Claude's Discretion + +- Whether to extract V1 decoder bodies into private methods (like encoders did) or leave inline +- Exact error message wording for V2 decode failures + +### Deferred Ideas (OUT OF SCOPE) + +None. Discussion stayed within phase scope. + + + +## Phase Requirements + +| ID | Description | Research Support | +|----|-------------|-----------------| +| DEC-01 | Decoder.HistogramSamples() accepts both V1 and V2 record types | Switch on type byte (pattern from Decoder.Samples() line 336); add HistogramSamplesV2 and CustomBucketsHistogramSamplesV2 cases | +| DEC-02 | Decoder.FloatHistogramSamples() accepts both V1 and V2 record types | Same switch pattern; add FloatHistogramSamplesV2 and CustomBucketsFloatHistogramSamplesV2 cases | +| DEC-03 | V2 histogram decoding correctly reads ST marker bytes and reconstructs ST values | histogramSamplesV2 / floatHistogramSamplesV2 private methods mirror samplesV2 (lines 384-434); track firstT, firstST outside loop | +| DEC-04 | V1 records decoded with ST=0 (backward compat) | ST field zero-value in struct; V1 path never writes ST, so zero value is automatic | + + +## Standard Stack + +### Core + +| Library | Version | Purpose | Why Standard | +|---------|---------|---------|--------------| +| `encoding.Decbuf` | in-repo | Binary decode buffer | Same type used by all existing decoders | +| `histogram.Histogram` / `histogram.FloatHistogram` | in-repo | Histogram structs | Target types for DecodeHistogram/DecodeFloatHistogram | +| `chunks.HeadSeriesRef` | in-repo | Ref type cast | Required for Ref field assignment | + +No new dependencies. Everything is already imported. + +## Architecture Patterns + +### Recommended Structure + +Two new private methods, two updated public methods. No new files. + +``` +tsdb/record/record.go + Decoder.HistogramSamples() — updated: switch on type byte + Decoder.histogramSamplesV2() — NEW private method + Decoder.FloatHistogramSamples() — updated: switch on type byte + Decoder.floatHistogramSamplesV2() — NEW private method +``` + +### Pattern 1: Public Method Dispatch (mirror Decoder.Samples()) + +**What:** Switch on the type byte immediately after reading it. Dispatch to private method. Return error for unknown types. + +**When to use:** Always — this is the established pattern in this file. + +```go +// Source: record.go line 336 (Decoder.Samples) +func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) ([]RefHistogramSample, error) { + dec := encoding.Decbuf{B: rec} + switch typ := Type(dec.Byte()); typ { + case HistogramSamples, CustomBucketsHistogramSamples: + return d.histogramSamplesV1(&dec, histograms) + case HistogramSamplesV2, CustomBucketsHistogramSamplesV2: + return d.histogramSamplesV2(&dec, histograms) + default: + return nil, fmt.Errorf("invalid record type %v", typ) + } +} +``` + +Note: The existing V1 body uses `if t != A && t != B` guard. Refactoring to a switch (Claude's discretion) is cleaner and matches `Decoder.Samples()`. Either approach is valid; switch is preferred for readability. + +### Pattern 2: V2 Private Decoder (mirror samplesV2) + +**What:** Track `firstT` and `firstST` outside the loop. First iteration reads absolute ref/T/ST. Subsequent iterations read delta-ref, delta-T, then ST marker byte. + +**When to use:** histogramSamplesV2 and floatHistogramSamplesV2 both follow this pattern. + +```go +// Source: record.go lines 384-434 (samplesV2), adapted for histograms +func (d *Decoder) histogramSamplesV2(dec *encoding.Decbuf, histograms []RefHistogramSample) ([]RefHistogramSample, error) { + if dec.Len() == 0 { + return histograms, nil + } + var firstT, firstST int64 + for len(dec.B) > 0 && dec.Err() == nil { + var ref, t, ST int64 + + if len(histograms) == 0 { + ref = dec.Varint64() + firstT = dec.Varint64() + t = firstT + ST = dec.Varint64() + firstST = ST + } else { + prev := histograms[len(histograms)-1] + ref = int64(prev.Ref) + dec.Varint64() + t = firstT + dec.Varint64() + stMarker := dec.Byte() + switch stMarker { + case noST: + // ST stays zero + case sameST: + ST = prev.ST + default: + ST = firstST + dec.Varint64() + } + } + + rh := RefHistogramSample{ + Ref: chunks.HeadSeriesRef(ref), + ST: ST, + T: t, + H: &histogram.Histogram{}, + } + DecodeHistogram(dec, rh.H) + + // Schema validation — same as V1 path, must be preserved + if !histogram.IsKnownSchema(rh.H.Schema) { + d.logger.Warn("skipping histogram with unknown schema in WAL record", "schema", rh.H.Schema, "timestamp", rh.T) + continue + } + if rh.H.Schema > histogram.ExponentialSchemaMax && rh.H.Schema <= histogram.ExponentialSchemaMaxReserved { + if err := rh.H.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { + return nil, fmt.Errorf("error reducing resolution of histogram #%d: %w", len(histograms)+1, err) + } + } + histograms = append(histograms, rh) + } + if dec.Err() != nil { + return nil, fmt.Errorf("decode error after %d histograms: %w", len(histograms), dec.Err()) + } + if len(dec.B) > 0 { + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) + } + return histograms, nil +} +``` + +`floatHistogramSamplesV2` is identical except it uses `RefFloatHistogramSample`, `rh.FH`, and `DecodeFloatHistogram`. + +### Anti-Patterns to Avoid + +- **Reading BE64 baseRef/baseTime in V2 path:** V2 wire format is all-varint. Reading BE64 would corrupt the decode. +- **Delta-to-first for ref:** V2 uses delta-to-prev for ref (unlike T which is delta-to-first). Getting these mixed up produces wrong refs for all samples after the first. +- **Forgetting schema validation in V2 path:** The warn-and-skip logic for unknown schemas must exist in V2 just as in V1. Omitting it would silently accept records that V1 would reject. +- **Using `prev` before histograms is non-empty:** `histograms[len(histograms)-1]` panics on empty slice. Guard with `if len(histograms) == 0` (same guard as samplesV2). + +## Don't Hand-Roll + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| Binary decode | Custom byte parsing | `encoding.Decbuf` | Handles error propagation, bounds checking | +| Histogram payload decode | Manual field reads | `DecodeHistogram` / `DecodeFloatHistogram` | Already correct, handles custom buckets | +| Schema check | Custom schema logic | `histogram.IsKnownSchema`, `histogram.ExponentialSchemaMax` | Shared constants, must stay in sync | + +## Common Pitfalls + +### Pitfall 1: ST marker default case +**What goes wrong:** The `default` case of the ST marker switch in samplesV2 means "explicitST". The byte value itself is not one of the named constants — it's any value that is neither `noST` nor `sameST`. Reading the varint delta only in the default case is correct. +**Why it happens:** Engineers expect a three-way switch with explicit constant matching and forget the default IS the third case. +**How to avoid:** Copy the switch structure verbatim from samplesV2 (line 408-415). Do not add a named `explicitST` case. + +### Pitfall 2: prev.ST vs firstST +**What goes wrong:** `sameST` sets `ST = prev.ST` (the last decoded sample's ST). `explicitST` sets `ST = firstST + delta` (delta from the first sample's ST). Swapping these produces wrong ST values. +**How to avoid:** Keep `firstST` as a loop-outer variable set once on first iteration. + +### Pitfall 3: skipped-custom-bucket effect on prev +**What goes wrong:** When schema validation causes a `continue` (skipping appending), `histograms[len(histograms)-1]` on the next iteration still refers to the last successfully decoded sample, not the skipped one. +**Why it happens:** This is correct behavior — the decoder's `prev` tracks the last appended sample, matching what the encoder encoded. No fix needed, just awareness. + +### Pitfall 4: V1 path modification +**What goes wrong:** Accidentally modifying the V1 path while refactoring (e.g., extracting into a private method and introducing a bug). +**How to avoid:** If extracting V1 into `histogramSamplesV1`, do it as a pure mechanical extraction with zero logic changes. Run existing tests to confirm. + +## Code Examples + +### Existing dispatch pattern (Decoder.Samples, line 336) +```go +// Source: tsdb/record/record.go:336 +func (d *Decoder) Samples(rec []byte, samples []RefSample) ([]RefSample, error) { + dec := encoding.Decbuf{B: rec} + switch typ := dec.Byte(); Type(typ) { + case Samples: + return d.samplesV1(&dec, samples) + case SamplesV2: + return d.samplesV2(&dec, samples) + default: + return nil, fmt.Errorf("invalid record type %v, expected Samples(2) or SamplesV2(11)", typ) + } +} +``` + +### ST marker decode (samplesV2, lines 408-415) +```go +// Source: tsdb/record/record.go:408 +stMarker := dec.Byte() +switch stMarker { +case noST: +case sameST: + ST = prev.ST +default: + ST = firstST + dec.Varint64() +} +``` + +### Current V1 HistogramSamples type guard (line 532) +```go +// Source: tsdb/record/record.go:532 — this gets replaced by the switch +if t != HistogramSamples && t != CustomBucketsHistogramSamples { + return nil, errors.New("invalid record type") +} +``` + +## State of the Art + +| Old Approach | Current Approach | When Changed | Impact | +|--------------|------------------|--------------|--------| +| BE64 baseRef + BE64 baseTime header | All-varint, no separate header | Phase 2 (V2 encoders) | Decoder must NOT read BE64 for V2 records | +| No ST field | ST field with noST/sameST/explicitST marker | Phase 1+2 | Decoder reads marker byte on every non-first sample | + +## Open Questions + +1. **Extract V1 into private method or leave inline?** + - What we know: Encoders extracted V1/V2 into private methods. Decoders currently have V1 inline. + - What's unclear: Whether the planner wants consistency with encoder style. + - Recommendation: Claude's discretion per CONTEXT.md. Extracting is cleaner (switch reads naturally) but is optional. Mark as discretionary in plan. + +## Sources + +### Primary (HIGH confidence) +- `tsdb/record/record.go` lines 336-434 (Decoder.Samples, samplesV1, samplesV2) — direct pattern source +- `tsdb/record/record.go` lines 529-683 (HistogramSamples, FloatHistogramSamples) — exact code being modified +- `.planning/phases/03-v2-decoders/03-CONTEXT.md` — locked decisions +- `.planning/phases/02-v2-encoders/02-CONTEXT.md` — encoder wire format reference +- `.planning/REQUIREMENTS.md` — DEC-01 through DEC-04 + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH — all in-repo, no external deps +- Architecture: HIGH — exact pattern exists in samplesV2, direct mirror +- Pitfalls: HIGH — derived from reading actual code and encoder decisions + +**Research date:** 2026-03-02 +**Valid until:** Until record.go is modified (stable internal code) diff --git a/.planning/phases/03-v2-decoders/03-VERIFICATION.md b/.planning/phases/03-v2-decoders/03-VERIFICATION.md new file mode 100644 index 0000000000..ce1e2d1383 --- /dev/null +++ b/.planning/phases/03-v2-decoders/03-VERIFICATION.md @@ -0,0 +1,101 @@ +--- +phase: 03-v2-decoders +verified: 2026-03-02T22:30:00Z +status: passed +score: 6/6 must-haves verified +re_verification: false +--- + +# Phase 03: V2 Decoders Verification Report + +**Phase Goal:** Decoder reads both V1 and V2 histogram records correctly. +**Verified:** 2026-03-02T22:30:00Z +**Status:** passed +**Re-verification:** No -- initial verification + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +|---|-------|--------|----------| +| 1 | Decoder.HistogramSamples() accepts V2 record types (HistogramSamplesV2, CustomBucketsHistogramSamplesV2) and decodes them correctly | VERIFIED | record.go:534 switch case dispatches to histogramSamplesV2 | +| 2 | V2 int-histogram decoding reads ST marker bytes and reconstructs ST values using noST/sameST/explicitST scheme | VERIFIED | record.go:606-613 reads stMarker byte and switches noST/sameST/default | +| 3 | V1 int-histogram records still decode with ST=0 (backward compat, zero value) | VERIFIED | histogramSamplesV1 (line 542) never sets ST; zero value preserved | +| 4 | Decoder.FloatHistogramSamples() accepts V2 record types (FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2) and decodes them correctly | VERIFIED | record.go:710 switch case dispatches to floatHistogramSamplesV2 | +| 5 | V2 float-histogram decoding reads ST marker bytes and reconstructs ST values using noST/sameST/explicitST scheme | VERIFIED | record.go:782-789 reads stMarker byte and switches noST/sameST/default | +| 6 | V1 float-histogram records still decode with ST=0 (backward compat, zero value) | VERIFIED | floatHistogramSamplesV1 (line 718) never sets ST; zero value preserved | + +**Score:** 6/6 truths verified + +### Required Artifacts + +| Artifact | Expected | Status | Details | +|----------|----------|--------|---------| +| `tsdb/record/record.go` histogramSamplesV2 | V2 int-histogram decoder with all-varint, ST markers, schema validation | VERIFIED | Line 588, 59 lines, full implementation with varint decode, ST marker switch, DecodeHistogram call, IsKnownSchema + ReduceResolution validation | +| `tsdb/record/record.go` floatHistogramSamplesV2 | V2 float-histogram decoder with all-varint, ST markers, schema validation | VERIFIED | Line 764, 59 lines, full implementation with varint decode, ST marker switch, DecodeFloatHistogram call, IsKnownSchema + ReduceResolution validation | +| `tsdb/record/record.go` histogramSamplesV1 | Extracted V1 int-histogram decoder (pure extraction) | VERIFIED | Line 542, 43 lines, reads BE64 baseRef/baseTime, varint deltas, schema validation preserved | +| `tsdb/record/record.go` floatHistogramSamplesV1 | Extracted V1 float-histogram decoder (pure extraction) | VERIFIED | Line 718, 43 lines, reads BE64 baseRef/baseTime, varint deltas, schema validation preserved | +| `tsdb/record/record.go` HistogramSamples dispatch | Switch on type byte dispatching V1/V2 | VERIFIED | Line 529, 10-line switch matching Decoder.Samples() pattern | +| `tsdb/record/record.go` FloatHistogramSamples dispatch | Switch on type byte dispatching V1/V2 | VERIFIED | Line 705, 10-line switch matching Decoder.Samples() pattern | + +### Key Link Verification + +| From | To | Via | Status | Details | +|------|----|-----|--------|---------| +| HistogramSamples() | histogramSamplesV2 | switch on type byte | WIRED | Line 534: `case HistogramSamplesV2, CustomBucketsHistogramSamplesV2` dispatches to `d.histogramSamplesV2(&dec, histograms)` | +| histogramSamplesV2 | DecodeHistogram | function call after ref/T/ST decode | WIRED | Line 622: `DecodeHistogram(dec, rh.H)` called inside V2 loop | +| FloatHistogramSamples() | floatHistogramSamplesV2 | switch on type byte | WIRED | Line 710: `case FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2` dispatches to `d.floatHistogramSamplesV2(&dec, histograms)` | +| floatHistogramSamplesV2 | DecodeFloatHistogram | function call after ref/T/ST decode | WIRED | Line 798: `DecodeFloatHistogram(dec, rh.FH)` called inside V2 loop | + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +|-------------|------------|-------------|--------|----------| +| DEC-01 | 03-01 | Decoder.HistogramSamples() accepts both V1 and V2 record types | SATISFIED | Switch at line 531 handles V1 types (HistogramSamples, CustomBucketsHistogramSamples) and V2 types (HistogramSamplesV2, CustomBucketsHistogramSamplesV2) | +| DEC-02 | 03-02 | Decoder.FloatHistogramSamples() accepts both V1 and V2 record types | SATISFIED | Switch at line 707 handles V1 types (FloatHistogramSamples, CustomBucketsFloatHistogramSamples) and V2 types (FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2) | +| DEC-03 | 03-01, 03-02 | V2 histogram decoding correctly reads ST marker bytes and reconstructs ST values | SATISFIED | Both histogramSamplesV2 (line 606-613) and floatHistogramSamplesV2 (line 782-789) implement correct noST/sameST/explicitST marker reconstruction | +| DEC-04 | 03-01, 03-02 | V1 records decoded with ST=0 (backward compat) | SATISFIED | V1 private methods (histogramSamplesV1, floatHistogramSamplesV1) never set ST; struct zero value (int64 = 0) provides backward compat | + +No orphaned requirements. All four DEC-* requirements from REQUIREMENTS.md traceability table map to Phase 3 and are covered by plans 03-01 and 03-02. + +### Anti-Patterns Found + +| File | Line | Pattern | Severity | Impact | +|------|------|---------|----------|--------| +| record.go | 181 | TODO(beorn7) | Info | Pre-existing, not from this phase | +| record.go | 230 | FIXME remove t | Info | Pre-existing, not from this phase | +| record.go | 923 | TODO: reconsider | Info | Pre-existing, not from this phase | + +No blocker or warning anti-patterns introduced by Phase 3. + +### Build and Vet + +- `go build ./tsdb/record/...` -- PASSES +- `go vet ./tsdb/record/...` -- PASSES + +### Test Suite Note + +`TestRecord_Type` fails (expects V1 type 0x7 but gets V2 type 0xc). This is a pre-existing issue caused by Phase 2 encoder changes. The test creates an `Encoder{EnableSTStorage: true}` at line 549, then calls `enc.HistogramSamples()` at line 600 which now emits V2 records. The test assertion at line 602 was not updated. This is out of Phase 3 scope. Phase 4 (TEST requirements) will address it. + +### Commit Verification + +| Commit | Message | Exists | +|--------|---------|--------| +| 4c9118810 | feat(03-01): add V1 and V2 int-histogram decoder private methods | Yes | +| 060c6f13e | feat(03-01): update HistogramSamples() to dispatch V1/V2 via switch | Yes | +| c1e5db311 | feat(03-02): add floatHistogramSamplesV1 and floatHistogramSamplesV2 decoder methods | Yes | +| 02214cd2d | feat(03-02): update FloatHistogramSamples() to dispatch V1/V2 via switch | Yes | + +### Human Verification Required + +None. All decoder behavior is verifiable through code inspection and build/vet checks. Round-trip correctness will be confirmed by Phase 4 tests. + +### Gaps Summary + +No gaps found. All six observable truths verified. All four DEC requirements satisfied. All artifacts are substantive (not stubs), correctly wired, and the code compiles and passes vet. The phase goal "Decoder reads both V1 and V2 histogram records correctly" is achieved. + +--- + +_Verified: 2026-03-02T22:30:00Z_ +_Verifier: Claude (gsd-verifier)_ diff --git a/.planning/phases/04-tests/04-01-PLAN.md b/.planning/phases/04-tests/04-01-PLAN.md new file mode 100644 index 0000000000..93e9e60a30 --- /dev/null +++ b/.planning/phases/04-tests/04-01-PLAN.md @@ -0,0 +1,272 @@ +--- +phase: 04-tests +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - tsdb/record/record_test.go +autonomous: true +requirements: + - TEST-01 + - TEST-02 + - TEST-03 + - TEST-04 + - TEST-05 + - TEST-06 + +must_haves: + truths: + - "V2 int-histogram round-trip produces identical samples for all ST scenarios (no-ST, constant-ST, delta-ST, same-ST)" + - "V2 float-histogram round-trip produces identical samples for all ST scenarios" + - "V2 custom-bucket histograms (int and float) round-trip correctly through separate encode/decode path" + - "V2 gauge histograms (int and float) round-trip correctly" + - "V1-encoded histograms decode with ST=0 on all samples (backward compat)" + artifacts: + - path: "tsdb/record/record_test.go" + provides: "V2 histogram round-trip tests and backward compat tests" + contains: "EnableSTStorage: true" + key_links: + - from: "tsdb/record/record_test.go" + to: "Encoder.HistogramSamples / Decoder.HistogramSamples" + via: "V2 encode/decode round-trip with ST field" + pattern: "EnableSTStorage.*true" + - from: "tsdb/record/record_test.go" + to: "Encoder.FloatHistogramSamples / Decoder.FloatHistogramSamples" + via: "V2 float encode/decode round-trip with ST field" + pattern: "FloatHistogramSamples" + - from: "tsdb/record/record_test.go" + to: "Encoder{} (V1)" + via: "Backward compat: V1 encode, decode, assert ST=0" + pattern: "Encoder\\{\\}" +--- + + +Add V2 histogram round-trip tests covering all ST scenarios, custom-bucket variants, gauge variants, and backward compatibility to TestRecord_EncodeDecode. + +Purpose: Verify that V2 histogram encoders and decoders (built in Phases 2-3) correctly round-trip all ST marker paths (noST, sameST, explicitST) for int-histograms, float-histograms, and custom-bucket variants. Also verify V1 backward compatibility. +Output: Extended TestRecord_EncodeDecode function with V2 histogram test blocks. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/04-tests/04-RESEARCH.md + +Source file: +@tsdb/record/record_test.go +@tsdb/record/record.go + + + + + + Task 1: Add V2 int-histogram and float-histogram round-trip tests with all ST scenarios + tsdb/record/record_test.go + +Add new V2 histogram test blocks inside `TestRecord_EncodeDecode`, inserted BEFORE the gauge mutation block at line 248. This positioning is critical because the gauge block mutates `histograms[i].H.CounterResetHint` in-place, which would corrupt subsequent test data. + +Use `enc = Encoder{EnableSTStorage: true}` (reassign the existing `enc` variable, matching how line 91 does it for V2 samples). + +Create four ST scenario blocks for int-histograms. Each block: +1. Defines a `[]RefHistogramSample` slice using the histogram pointers from the existing `histograms` slice (indices 0 and 1 are standard schema=1, index 2 is custom-bucket schema=-53). +2. Calls `enc.HistogramSamples(slice, nil)` which returns `(histSamples, customBucketsSlice)`. +3. If the slice includes the custom-bucket histogram (index 2), also calls `enc.CustomBucketsHistogramSamples(customBucketsSlice, nil)`. +4. Decodes both records, appends, and `require.Equal` against the input. + +**ST Scenario 1 - No ST (TEST-01):** +All samples have ST=0 (zero-value default). Use all 3 histograms (indices 0, 1, 2) so this also exercises custom-bucket V2 path. +```go +// V2 int-histogram round-trip: no ST. +enc = Encoder{EnableSTStorage: true} +histsV2NoST := []RefHistogramSample{ + {Ref: 56, T: 1234, H: histograms[0].H}, + {Ref: 42, T: 5678, H: histograms[1].H}, + {Ref: 67, T: 5678, H: histograms[2].H}, +} +histSamplesV2, customBucketsV2 := enc.HistogramSamples(histsV2NoST, nil) +customBucketsHistSamplesV2 := enc.CustomBucketsHistogramSamples(customBucketsV2, nil) +decHistsV2, err := dec.HistogramSamples(histSamplesV2, nil) +require.NoError(t, err) +decCustomBucketsV2, err := dec.HistogramSamples(customBucketsHistSamplesV2, nil) +require.NoError(t, err) +decHistsV2 = append(decHistsV2, decCustomBucketsV2...) +require.Equal(t, histsV2NoST, decHistsV2) +``` + +**ST Scenario 2 - Constant ST (TEST-02):** +All samples have identical non-zero ST (exercises sameST marker path for samples after the first). +```go +// V2 int-histogram round-trip: constant ST. +histsV2ConstST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1000, H: histograms[1].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, +} +// ... same encode/decode/append/equal pattern +``` + +**ST Scenario 3 - Varying/Delta ST (TEST-03):** +Each sample has a different ST (exercises explicitST marker path). +```go +// V2 int-histogram round-trip: varying ST. +histsV2VarST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1234, H: histograms[1].H}, + {Ref: 67, T: 9012, ST: 5678, H: histograms[2].H}, +} +``` + +**ST Scenario 4 - Same ST across samples (mirrors samplesWithConstST from line 127):** +Samples with same ST but different T values. +```go +// V2 int-histogram round-trip: same ST across samples. +histsV2SameST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 900, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 900, H: histograms[1].H}, + {Ref: 67, T: 9012, ST: 900, H: histograms[2].H}, +} +``` + +Then, for each of the four int-histogram scenario slices, derive the corresponding float-histogram slice (TEST-04) using the established pattern from lines 231-238: +```go +floatHistsV2NoST := make([]RefFloatHistogramSample, len(histsV2NoST)) +for i, h := range histsV2NoST { + floatHistsV2NoST[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } +} +``` +IMPORTANT: Include `ST: h.ST` in the float conversion loop. The existing line-232 pattern does NOT copy ST because it was written before ST existed on histogram structs. + +Encode float histograms with `enc.FloatHistogramSamples()` and `enc.CustomBucketsFloatHistogramSamples()`, decode, append, require.Equal. Same pattern as lines 239-246. + +For all four scenarios, TEST-05 (custom-bucket V2) is exercised automatically because the third histogram in the slice has schema=-53. + +Use `t.Run` subtests for each scenario to improve failure isolation. Group as: +- `t.Run("V2 int-histogram no ST", func(t *testing.T) { ... })` +- `t.Run("V2 int-histogram constant ST", func(t *testing.T) { ... })` +- etc. +- `t.Run("V2 float-histogram no ST", func(t *testing.T) { ... })` +- etc. + +**IMPORTANT ORDERING:** All V2 test blocks must be inserted BEFORE the gauge mutation at line 248 (`histograms[i].H.CounterResetHint = histogram.GaugeType`). This is because the gauge block mutates the shared `histograms[i].H` pointer, which would corrupt the test data for V2 blocks added afterward. + + +```bash +go build ./tsdb/record/... +go test ./tsdb/record/ -run "TestRecord_EncodeDecode/V2_(int|float)-histogram" -count=1 -v +``` + + All 8 V2 histogram round-trip subtests (4 int-histogram ST scenarios + 4 float-histogram ST scenarios) pass. Each scenario includes custom-bucket histograms (TEST-05). + + + + Task 2: Add V2 gauge variant round-trip tests and backward compatibility test + tsdb/record/record_test.go + +After the existing gauge block (which ends around line 274), add V2 gauge histogram round-trip tests and the backward compatibility assertion. + +**V2 Gauge Variants:** +At this point in the function, `histograms[i].H.CounterResetHint` has been set to `histogram.GaugeType` and `floatHistograms[i].FH.CounterResetHint` has been set to `histogram.GaugeType`. Re-use these for V2 gauge round-trips. + +Add two blocks (or subtests): + +1. **V2 gauge int-histograms:** Use `enc = Encoder{EnableSTStorage: true}` (it should already be set from Task 1, but reassign for clarity). Create gauge histogram samples with ST values (pick one ST scenario, e.g. constant ST). Encode with `HistogramSamples` + `CustomBucketsHistogramSamples`, decode, append, require.Equal. +```go +// V2 gauge int-histogram round-trip. +enc = Encoder{EnableSTStorage: true} +gaugeHistsV2 := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1000, H: histograms[1].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, +} +// encode/decode/append/equal pattern +``` + +2. **V2 gauge float-histograms:** Same pattern with `floatHistograms` (already mutated to gauge). Derive from `gaugeHistsV2` or use the already-mutated `floatHistograms` slice. +```go +// V2 gauge float-histogram round-trip. +gaugeFloatHistsV2 := make([]RefFloatHistogramSample, len(gaugeHistsV2)) +for i, h := range gaugeHistsV2 { + gaugeFloatHistsV2[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } +} +// encode/decode/append/equal pattern +``` + +**Backward Compatibility (TEST-06):** +After the V2 gauge blocks, add a backward compat test that: +1. Creates a FRESH V1 encoder: `encV1 := Encoder{}` (NOT the `enc` variable which is V2). +2. Encodes the `histograms` slice (still gauge-mutated, that's fine — backward compat is about ST, not counter reset hint). +3. Decodes and asserts `ST == 0` on every decoded sample. +4. Does the same for float histograms. + +```go +// Backward compat: V1-encoded histograms decode with ST=0. +encV1 := Encoder{} +v1HistSamples, v1CustomBucketsHists := encV1.HistogramSamples(histograms, nil) +v1CustomBucketsHistSamples := encV1.CustomBucketsHistogramSamples(v1CustomBucketsHists, nil) +decV1Hists, err := dec.HistogramSamples(v1HistSamples, nil) +require.NoError(t, err) +decV1CustomBuckets, err := dec.HistogramSamples(v1CustomBucketsHistSamples, nil) +require.NoError(t, err) +for _, h := range append(decV1Hists, decV1CustomBuckets...) { + require.Equal(t, int64(0), h.ST, "V1 histogram records must decode with ST=0") +} + +// Same for float histograms. +v1FloatHistSamples, v1CustomBucketsFloatHists := encV1.FloatHistogramSamples(floatHistograms, nil) +v1CustomBucketsFloatHistSamples := encV1.CustomBucketsFloatHistogramSamples(v1CustomBucketsFloatHists, nil) +decV1FloatHists, err := dec.FloatHistogramSamples(v1FloatHistSamples, nil) +require.NoError(t, err) +decV1CustomBucketsFloatHists, err := dec.FloatHistogramSamples(v1CustomBucketsFloatHistSamples, nil) +require.NoError(t, err) +for _, h := range append(decV1FloatHists, decV1CustomBucketsFloatHists...) { + require.Equal(t, int64(0), h.ST, "V1 float histogram records must decode with ST=0") +} +``` + + +```bash +go test ./tsdb/record/ -run TestRecord_EncodeDecode -count=1 -v +``` + + V2 gauge histogram round-trips pass. V1 backward compat test confirms all decoded histogram samples have ST=0. Full TestRecord_EncodeDecode passes. + + + + + +```bash +go test ./tsdb/record/ -run TestRecord_EncodeDecode -count=1 -v +go vet ./tsdb/record/... +``` +All subtests pass. No vet warnings. + + + +- TestRecord_EncodeDecode passes with V2 int-histogram round-trips for 4 ST scenarios +- TestRecord_EncodeDecode passes with V2 float-histogram round-trips for 4 ST scenarios +- Custom-bucket V2 histograms (schema=-53) round-trip through separate encode/decode path in every scenario +- V2 gauge variant round-trips pass for both int and float histograms +- V1-encoded histograms decode with ST=0 (backward compat confirmed) +- go vet passes + + + +After completion, create `.planning/phases/04-tests/04-01-SUMMARY.md` + diff --git a/.planning/phases/04-tests/04-01-SUMMARY.md b/.planning/phases/04-tests/04-01-SUMMARY.md new file mode 100644 index 0000000000..75329e7828 --- /dev/null +++ b/.planning/phases/04-tests/04-01-SUMMARY.md @@ -0,0 +1,100 @@ +--- +phase: 04-tests +plan: 01 +subsystem: testing +tags: [go-test, histogram, wal, v2-encoding, round-trip, backward-compat] + +# Dependency graph +requires: + - phase: 02-v2-encoders + provides: V2 histogram/float-histogram encoder methods with ST marker scheme + - phase: 03-v2-decoders + provides: V2 histogram/float-histogram decoder methods matching encoder wire format +provides: + - V2 histogram round-trip tests for all 4 ST scenarios (no ST, constant, varying, same) + - V2 float-histogram round-trip tests for all 4 ST scenarios + - V2 gauge histogram round-trip tests (int + float) + - V1 backward compatibility verification (ST=0 on all decoded samples) +affects: [04-02-PLAN] + +# Tech tracking +tech-stack: + added: [] + patterns: [t.Run subtests for ST scenario isolation, ToFloat derivation with ST propagation] + +key-files: + created: [] + modified: [tsdb/record/record_test.go] + +key-decisions: + - "Used t.Run subtests for each ST scenario and gauge/backward-compat block for failure isolation" + - "Float histogram test data derived from int-histogram slices via ToFloat(nil) with explicit ST copy" + - "V2 test blocks inserted before gauge mutation (line 248) to avoid shared pointer corruption" + +patterns-established: + - "V2 histogram test pattern: build RefHistogramSample slice with ST, encode via HistogramSamples + CustomBucketsHistogramSamples, decode both, append, require.Equal" + - "Float histogram derivation must include ST field copy (existing V1 pattern at line 232 omits it)" + +requirements-completed: [TEST-01, TEST-02, TEST-03, TEST-04, TEST-05, TEST-06] + +# Metrics +duration: 2min +completed: 2026-03-03 +--- + +# Phase 4 Plan 1: V2 Histogram Round-Trip Tests Summary + +**V2 histogram encode/decode round-trips for all ST marker paths (noST, sameST, explicitST) plus gauge variants and V1 backward compat** + +## Performance + +- **Duration:** 2 min +- **Started:** 2026-03-03T14:14:46Z +- **Completed:** 2026-03-03T14:16:38Z +- **Tasks:** 2 +- **Files modified:** 1 + +## Accomplishments +- 8 V2 histogram round-trip subtests covering 4 ST scenarios x 2 types (int + float), each including custom-bucket histograms +- 2 V2 gauge histogram round-trip subtests (int + float) with constant ST +- 2 V1 backward compatibility subtests confirming all decoded histograms have ST=0 +- All 12 subtests pass; go vet clean + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: V2 int-histogram and float-histogram round-trip tests** - `021b0d9e4` (test) +2. **Task 2: V2 gauge variants and backward compat** - `b05e0d328` (test) + +## Files Created/Modified +- `tsdb/record/record_test.go` - Added 12 subtests to TestRecord_EncodeDecode for V2 histogram round-trips, gauge variants, and V1 backward compat + +## Decisions Made +- Used `t.Run` subtests for each scenario (plan allowed Claude's discretion; chose subtests for failure isolation) +- Float histogram test data derived from int-histogram slices with explicit `ST: h.ST` copy in conversion loop +- All V2 test blocks placed before the gauge mutation block to avoid shared histogram pointer corruption + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered +None + +## User Setup Required +None - no external service configuration required. + +## Next Phase Readiness +- TestRecord_EncodeDecode now fully covers V2 histogram encode/decode for all ST marker paths +- Plan 04-02 (TestRecord_Type V2 type assertions) can proceed independently + +## Self-Check: PASSED + +- FOUND: tsdb/record/record_test.go +- FOUND: 021b0d9e4 (Task 1 commit) +- FOUND: b05e0d328 (Task 2 commit) + +--- +*Phase: 04-tests* +*Completed: 2026-03-03* diff --git a/.planning/phases/04-tests/04-02-PLAN.md b/.planning/phases/04-tests/04-02-PLAN.md new file mode 100644 index 0000000000..1c0e77cd47 --- /dev/null +++ b/.planning/phases/04-tests/04-02-PLAN.md @@ -0,0 +1,134 @@ +--- +phase: 04-tests +plan: 02 +type: execute +wave: 2 +depends_on: + - "04-01" +files_modified: + - tsdb/record/record_test.go +autonomous: true +requirements: + - TEST-07 + +must_haves: + truths: + - "dec.Type() returns HistogramSamplesV2 for V2-encoded int-histogram records" + - "dec.Type() returns CustomBucketsHistogramSamplesV2 for V2-encoded custom-bucket int-histogram records" + - "dec.Type() returns FloatHistogramSamplesV2 for V2-encoded float-histogram records" + - "dec.Type() returns CustomBucketsFloatHistogramSamplesV2 for V2-encoded custom-bucket float-histogram records" + artifacts: + - path: "tsdb/record/record_test.go" + provides: "V2 histogram type recognition assertions in TestRecord_Type" + contains: "HistogramSamplesV2" + key_links: + - from: "tsdb/record/record_test.go" + to: "Decoder.Type()" + via: "Type assertion for V2 histogram record types" + pattern: "HistogramSamplesV2|FloatHistogramSamplesV2|CustomBuckets.*V2" +--- + + +Add V2 histogram type recognition assertions to TestRecord_Type for all four V2 histogram record types. + +Purpose: Verify that Decoder.Type() correctly identifies V2-encoded histogram records (HistogramSamplesV2, CustomBucketsHistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2). +Output: Extended TestRecord_Type function with V2 type assertions. + + + +@/home/owilliams/.claude/get-shit-done/workflows/execute-plan.md +@/home/owilliams/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/04-tests/04-01-SUMMARY.md + +Source file: +@tsdb/record/record_test.go +@tsdb/record/record.go + + + + + + Task 1: Add V2 histogram type assertions to TestRecord_Type + tsdb/record/record_test.go + +In `TestRecord_Type`, add V2 histogram type assertions after the existing V1 histogram type assertions (which end around line 605 with `require.Equal(t, CustomBucketsHistogramSamples, recordType)`). + +The `enc` variable was already reassigned to `Encoder{EnableSTStorage: true}` at line 549 for the V2 samples test. However, it is cleaner to reassign it again explicitly for the histogram V2 block. This avoids hidden coupling to code above. + +The `histograms` slice is already defined in `TestRecord_Type` at line 566 with 2 entries: one standard (schema=1) and one custom-bucket (schema=-53). + +**V2 int-histogram type assertions:** +```go +// V2 histogram type recognition. +enc = Encoder{EnableSTStorage: true} +hists, customBucketsHistograms = enc.HistogramSamples(histograms, nil) +recordType = dec.Type(hists) +require.Equal(t, HistogramSamplesV2, recordType) +customBucketsHists = enc.CustomBucketsHistogramSamples(customBucketsHistograms, nil) +recordType = dec.Type(customBucketsHists) +require.Equal(t, CustomBucketsHistogramSamplesV2, recordType) +``` + +Note: Reuse the existing local variables `hists`, `customBucketsHistograms`, `customBucketsHists` that are already declared (lines 600-604). If the executor finds these are `:=` assigned (not `=` assigned), use `=` to avoid redeclaration. + +**V2 float-histogram type assertions:** +Construct float histograms from the existing `histograms` slice, then encode and check types. +```go +// V2 float-histogram type recognition. +floatHistograms := make([]RefFloatHistogramSample, len(histograms)) +for i, h := range histograms { + floatHistograms[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + FH: h.H.ToFloat(nil), + } +} +floatHists, customBucketsFloatHistograms := enc.FloatHistogramSamples(floatHistograms, nil) +recordType = dec.Type(floatHists) +require.Equal(t, FloatHistogramSamplesV2, recordType) +customBucketsFloatHists := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatHistograms, nil) +recordType = dec.Type(customBucketsFloatHists) +require.Equal(t, CustomBucketsFloatHistogramSamplesV2, recordType) +``` + +Place all of this between the existing V1 histogram type block (ending at line 605) and the `Unknown` type assertions (lines 607-611). This keeps the logical flow: V1 types, then V2 types, then Unknown. + +**PITFALL:** The existing code at lines 600-604 uses `:=` for `hists` and `customBucketsHistograms`. The V2 block should use `=` since these variables are already declared. Check the actual code — if they are already `=` (not `:=`), the V2 block matches naturally. + + +```bash +go test ./tsdb/record/ -run TestRecord_Type -count=1 -v +go test ./tsdb/record/ -count=1 +``` + + TestRecord_Type passes with all four V2 histogram type assertions (HistogramSamplesV2, CustomBucketsHistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2). Full test suite passes. + + + + + +```bash +go test ./tsdb/record/ -run TestRecord_Type -count=1 -v +go test ./tsdb/record/ -count=1 +go vet ./tsdb/record/... +``` +TestRecord_Type passes. Full package test suite passes. No vet warnings. + + + +- dec.Type() returns HistogramSamplesV2 for V2-encoded int-histogram records +- dec.Type() returns CustomBucketsHistogramSamplesV2 for V2-encoded custom-bucket int-histogram records +- dec.Type() returns FloatHistogramSamplesV2 for V2-encoded float-histogram records +- dec.Type() returns CustomBucketsFloatHistogramSamplesV2 for V2-encoded custom-bucket float-histogram records +- Full `go test ./tsdb/record/ -count=1` passes + + + +After completion, create `.planning/phases/04-tests/04-02-SUMMARY.md` + diff --git a/.planning/phases/04-tests/04-02-SUMMARY.md b/.planning/phases/04-tests/04-02-SUMMARY.md new file mode 100644 index 0000000000..867806ab91 --- /dev/null +++ b/.planning/phases/04-tests/04-02-SUMMARY.md @@ -0,0 +1,101 @@ +--- +phase: 04-tests +plan: 02 +subsystem: testing +tags: [wal, histogram, v2, decoder-type, record-type] + +# Dependency graph +requires: + - phase: 04-tests-plan-01 + provides: "V2 histogram round-trip test infrastructure" + - phase: 03-v2-decoders + provides: "Decoder.Type() dispatch for V2 histogram record types" +provides: + - "V2 histogram type recognition assertions for all four V2 histogram record types" +affects: [] + +# Tech tracking +tech-stack: + added: [] + patterns: ["V2 type recognition testing via enc/dec.Type() round-trip"] + +key-files: + created: [] + modified: + - "tsdb/record/record_test.go" + +key-decisions: + - "Reset enc to zero-value Encoder{} before V1 histogram block to fix pre-existing bug where EnableSTStorage leaked from SamplesV2 test" + +patterns-established: + - "V1/V2 encoder reset: explicitly set enc before each type-recognition block to avoid cross-contamination" + +requirements-completed: [TEST-07] + +# Metrics +duration: 1min +completed: 2026-03-03 +--- + +# Phase 4 Plan 02: TestRecord_Type V2 Assertions Summary + +**V2 histogram type recognition assertions for all four V2 types (int, custom-bucket int, float, custom-bucket float) plus fix for pre-existing V1 encoder leak bug** + +## Performance + +- **Duration:** 1 min +- **Started:** 2026-03-03T14:19:40Z +- **Completed:** 2026-03-03T14:20:23Z +- **Tasks:** 1 +- **Files modified:** 1 + +## Accomplishments +- Added V2 int-histogram type assertions (HistogramSamplesV2, CustomBucketsHistogramSamplesV2) +- Added V2 float-histogram type assertions (FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2) +- Fixed pre-existing bug where EnableSTStorage leaked into V1 histogram assertions + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Add V2 histogram type assertions to TestRecord_Type** - `2523b681e` (test) + +**Plan metadata:** (pending) + +## Files Created/Modified +- `tsdb/record/record_test.go` - Added V2 histogram type recognition assertions and fixed V1 encoder leak + +## Decisions Made +- Reset `enc` to zero-value `Encoder{}` before V1 histogram block. The previous code left `EnableSTStorage: true` from the SamplesV2 test (line 789), causing V1 histogram assertions to encode V2 records. This was a pre-existing bug that caused TestRecord_Type to fail. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 1 - Bug] Fixed pre-existing EnableSTStorage leak in TestRecord_Type** +- **Found during:** Task 1 (Add V2 histogram type assertions) +- **Issue:** `enc` was set to `Encoder{EnableSTStorage: true}` at line 789 for SamplesV2 tests but never reset before V1 histogram assertions at line 840. This caused `HistogramSamples()` to dispatch to V2, making the V1 type assertion fail (expected 0x7, got 0xc). +- **Fix:** Added `enc = Encoder{}` before the V1 histogram type recognition block. +- **Files modified:** tsdb/record/record_test.go +- **Verification:** TestRecord_Type passes, full package suite passes, go vet clean. +- **Committed in:** 2523b681e (Task 1 commit) + +--- + +**Total deviations:** 1 auto-fixed (1 bug) +**Impact on plan:** Bug fix necessary for test correctness. No scope creep. + +## Issues Encountered +None beyond the pre-existing bug documented above. + +## User Setup Required +None - no external service configuration required. + +## Next Phase Readiness +- All four V2 histogram types verified via Decoder.Type() recognition +- Phase 4 testing plan complete (both plan 01 and plan 02 done) +- Project milestone complete: histogram ST WAL records fully implemented and tested + +--- +*Phase: 04-tests* +*Completed: 2026-03-03* diff --git a/.planning/phases/04-tests/04-CONTEXT.md b/.planning/phases/04-tests/04-CONTEXT.md new file mode 100644 index 0000000000..5330ef35b4 --- /dev/null +++ b/.planning/phases/04-tests/04-CONTEXT.md @@ -0,0 +1,82 @@ +# Phase 4: Tests - Context + +**Gathered:** 2026-03-02 +**Status:** Ready for planning + + +## Phase Boundary + +Full test coverage for V2 histogram encoding/decoding. Round-trip tests for all ST scenarios, backward compat verified, Type() recognition for new types. All in `tsdb/record/record_test.go`. + + + + +## Implementation Decisions + +### Test organization +- Add V2 histogram round-trip tests as new sections within the existing `TestRecord_EncodeDecode` function, matching how V2 float samples were added inline (lines 91-134) +- `TestRecord_Type` gets new assertions for V2 histogram types (extend existing function, don't create new one) +- Schema validation and corrupted record tests already loop over `enableSTStorage: true/false` so they exercise V2 paths. No changes needed there. + +### ST scenario coverage +- Mirror all 4 float sample ST patterns for histograms (no ST, constant ST, varying/delta ST, same-ST-across-samples) +- Each scenario tests both int-histogram and float-histogram V2 +- Each scenario includes custom-bucket variants +- Encoder uses `EnableSTStorage: true` for all V2 tests + +### Backward compatibility +- Encode histograms with V1 encoder (default, no EnableSTStorage), decode, verify ST=0 on all decoded samples +- This confirms V1 records remain readable and the zero-value ST is backward-compatible + +### TestRecord_Type additions +- Encode histograms with `EnableSTStorage: true`, verify `dec.Type()` returns `HistogramSamplesV2` and `CustomBucketsHistogramSamplesV2` +- Same for float histogram V2 types +- Uses the existing `histograms` test data slice already defined in `TestRecord_Type` + +### Claude's Discretion +- Exact histogram test data values (can reuse existing `histograms` slice from line 166) +- Whether to use subtests (`t.Run`) for each ST scenario within the monolithic test +- Exact ST values in test data + + + + +## Existing Code Insights + +### Reusable Assets +- `histograms` slice at line 166: 3 samples (standard schema=1, negative buckets, custom buckets schema=-53). Already filters into standard + custom-bucket groups. +- `floatHistograms` construction at line 231: converts from int histograms via `h.H.ToFloat(nil)`. Same pattern for V2 float tests. +- Float sample ST test patterns at lines 79-134: direct template for histogram ST scenarios. +- `enc.HistogramSamples()` returns `(histSamples, customBucketsHistograms)` pattern already used at line 222. + +### Established Patterns +- Round-trip: `enc.HistogramSamples(input, nil)` -> `dec.HistogramSamples(encoded, nil)` -> `require.Equal(t, input, decoded)` +- Custom buckets: filter return from `HistogramSamples()`, encode separately with `CustomBucketsHistogramSamples()`, decode, append, compare +- Gauge variant: mutate `CounterResetHint` on same data, re-encode, round-trip again (lines 248-274) +- `Encoder{EnableSTStorage: true}` creates V2 encoder (line 91) + +### Integration Points +- `TestRecord_Type` at line 536: add V2 type assertions after existing V1 histogram assertions (line 605) +- `TestRecord_EncodeDecode` at line 275: add V2 histogram sections before the closing brace + + + + +## Specific Ideas + +- The existing V1 histogram test data includes a custom-bucket histogram (schema=-53 with CustomValues). When testing V2, the same data naturally exercises the custom-bucket filtering and separate encoding path. +- The gauge variant re-test (lines 248-274) should also be done for V2, since gauge histograms encode differently (no counter reset hint assumptions). + + + + +## Deferred Ideas + +None. Discussion stayed within phase scope. + + + +--- + +*Phase: 04-tests* +*Context gathered: 2026-03-02* diff --git a/.planning/phases/04-tests/04-RESEARCH.md b/.planning/phases/04-tests/04-RESEARCH.md new file mode 100644 index 0000000000..f92cdb35e0 --- /dev/null +++ b/.planning/phases/04-tests/04-RESEARCH.md @@ -0,0 +1,332 @@ +# Phase 4: Tests - Research + +**Researched:** 2026-03-03 +**Domain:** Go test patterns for WAL record encode/decode round-trips +**Confidence:** HIGH + + +## User Constraints (from CONTEXT.md) + +### Locked Decisions + +**Test organization** +- Add V2 histogram round-trip tests as new sections within the existing `TestRecord_EncodeDecode` function, matching how V2 float samples were added inline (lines 91-134). +- `TestRecord_Type` gets new assertions for V2 histogram types (extend existing function, do not create a new one). +- Schema validation and corrupted record tests already loop over `enableSTStorage: true/false` so they exercise V2 paths. No changes needed there. + +**ST scenario coverage** +- Mirror all 4 float sample ST patterns for histograms (no ST, constant ST, varying/delta ST, same-ST-across-samples). +- Each scenario tests both int-histogram and float-histogram V2. +- Each scenario includes custom-bucket variants. +- Encoder uses `EnableSTStorage: true` for all V2 tests. + +**Backward compatibility** +- Encode histograms with V1 encoder (default, no EnableSTStorage), decode, verify ST=0 on all decoded samples. +- This confirms V1 records remain readable and the zero-value ST is backward-compatible. + +**TestRecord_Type additions** +- Encode histograms with `EnableSTStorage: true`, verify `dec.Type()` returns `HistogramSamplesV2` and `CustomBucketsHistogramSamplesV2`. +- Same for float histogram V2 types. +- Uses the existing `histograms` test data slice already defined in `TestRecord_Type`. + +### Claude's Discretion +- Exact histogram test data values (can reuse existing `histograms` slice from line 166). +- Whether to use subtests (`t.Run`) for each ST scenario within the monolithic test. +- Exact ST values in test data. + +### Deferred Ideas (OUT OF SCOPE) +None. Discussion stayed within phase scope. + + + +## Phase Requirements + +| ID | Description | Research Support | +|----|-------------|-----------------| +| TEST-01 | Round-trip encode/decode for histogram V2 with no ST | Use `histograms` slice (line 166), V2 encoder (EnableSTStorage: true), all ST fields zero; decode and require.Equal | +| TEST-02 | Round-trip encode/decode for histogram V2 with constant ST | Same data with identical non-zero ST on all samples; sameST marker path | +| TEST-03 | Round-trip encode/decode for histogram V2 with varying ST | Different non-zero ST per sample; explicitST marker path | +| TEST-04 | Round-trip encode/decode for float histogram V2 (same ST scenarios) | Derive floatHistograms from histograms via h.H.ToFloat(nil); same 4 scenarios | +| TEST-05 | Round-trip encode/decode for custom buckets variants V2 | Third histogram in slice (schema=-53, CustomValues set) naturally splits into custom-buckets path; encode with CustomBucketsHistogramSamplesV2 / CustomBucketsFloatHistogramSamplesV2 | +| TEST-06 | V1 records still decode correctly (backward compat test) | V1 encoder (zero-value Encoder{}), encode histograms, decode, assert ST==0 on all results | +| TEST-07 | Type() correctly identifies new record types | enc with EnableSTStorage:true; assert dec.Type returns HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2 | + + +## Summary + +This phase is entirely within `tsdb/record/record_test.go`. The implementation work (encoders and decoders for all four V2 histogram record types) is complete in Phases 1-3. Phase 4 only adds tests — no production code changes. + +The test patterns are already fully established in the file. The V2 float-sample tests (lines 91-134) are the direct template for V2 histogram tests. The existing V1 histogram round-trip block (lines 166-274) is the source of reusable test data and shows exactly how custom-bucket splitting works. + +There are no unknown APIs, no new dependencies, and no architectural decisions left. Every encoder and decoder method being tested already exists and compiles. + +**Primary recommendation:** Extend `TestRecord_EncodeDecode` with four ST-scenario blocks (no-ST, constant-ST, delta-ST, same-ST) for int-histograms + float-histograms + custom-buckets, then extend `TestRecord_Type` with V2 type assertions. Treat the existing V2 sample tests as the line-by-line model. + +## Standard Stack + +### Core + +| Library | Version | Purpose | Why Standard | +|---------|---------|---------|--------------| +| `testing` | stdlib | Go test framework | Required | +| `github.com/stretchr/testify/require` | already imported | Assertion helpers | Already used throughout the file | + +No new imports are needed. The test file already imports everything required: `testing`, `require`, `histogram`, `labels`, `promslog`, `testutil`, `rand`. + +### Supporting + +None. This is a pure test extension of an existing file. + +### Alternatives Considered + +| Instead of | Could Use | Tradeoff | +|------------|-----------|----------| +| Inline sections in `TestRecord_EncodeDecode` | Separate top-level test functions | Locked decision: inline, matching existing V2 sample pattern | +| `t.Run` subtests for ST scenarios | Flat inline blocks | Claude's discretion — subtests improve failure isolation, planner should decide | + +## Architecture Patterns + +### Pattern 1: V2 Histogram Round-Trip (noST case) + +**What:** Encode with V2 encoder, decode, assert equality. +**When to use:** Every scenario block. + +```go +// Source: record_test.go lines 91-102 (V2 sample pattern) +enc = Encoder{EnableSTStorage: true} +histogramsNoST := []RefHistogramSample{ + {Ref: 56, T: 1234, H: histograms[0].H}, + {Ref: 42, T: 5678, H: histograms[1].H}, +} +histSamples, customBucketsHistograms := enc.HistogramSamples(histogramsNoST, nil) +require.Equal(t, HistogramSamplesV2, dec.Type(histSamples)) +decHistograms, err := dec.HistogramSamples(histSamples, nil) +require.NoError(t, err) +require.Equal(t, histogramsNoST, decHistograms) +``` + +### Pattern 2: Custom Buckets Split-and-Encode + +**What:** `HistogramSamples()` returns `(mainRecord, customBucketsSlice)`. Custom-bucket samples go through a separate encoder call. +**When to use:** Every block that includes the custom-bucket histogram (schema=-53). + +```go +// Source: record_test.go lines 222-229 +histSamples, customBucketsHistograms := enc.HistogramSamples(histograms, nil) +customBucketsHistSamples := enc.CustomBucketsHistogramSamples(customBucketsHistograms, nil) +decHistograms, err := dec.HistogramSamples(histSamples, nil) +require.NoError(t, err) +decCustomBuckets, err := dec.HistogramSamples(customBucketsHistSamples, nil) +require.NoError(t, err) +decHistograms = append(decHistograms, decCustomBuckets...) +require.Equal(t, histograms, decHistograms) +``` + +### Pattern 3: Float Histogram Derivation + +**What:** Derive float histograms from int histograms via `h.H.ToFloat(nil)`. +**When to use:** Float histogram V2 tests (TEST-04). + +```go +// Source: record_test.go lines 231-238 +floatHistograms := make([]RefFloatHistogramSample, len(histogramsWithST)) +for i, h := range histogramsWithST { + floatHistograms[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } +} +``` + +### Pattern 4: Backward Compat Assertion + +**What:** Encode with V1 encoder (zero-value `Encoder{}`), decode, verify `ST == 0` on every result. + +```go +// V1 encoder: no EnableSTStorage +encV1 := Encoder{} +histSamples, customBucketsHistograms := encV1.HistogramSamples(histograms, nil) +customBucketsHistSamples := encV1.CustomBucketsHistogramSamples(customBucketsHistograms, nil) +decHistograms, err := dec.HistogramSamples(histSamples, nil) +require.NoError(t, err) +decCustomBuckets, err := dec.HistogramSamples(customBucketsHistSamples, nil) +require.NoError(t, err) +all := append(decHistograms, decCustomBuckets...) +for _, h := range all { + require.Equal(t, int64(0), h.ST, "V1 records must decode with ST=0") +} +``` + +### Pattern 5: TestRecord_Type V2 Assertions + +**What:** After the existing V1 histogram type assertions (line 605), add V2 type checks. + +```go +// V2 type assertions — append after existing histogram type block +enc = Encoder{EnableSTStorage: true} +hists, customBucketsHistograms := enc.HistogramSamples(histograms, nil) +recordType = dec.Type(hists) +require.Equal(t, HistogramSamplesV2, recordType) +customBucketsHists := enc.CustomBucketsHistogramSamples(customBucketsHistograms, nil) +recordType = dec.Type(customBucketsHists) +require.Equal(t, CustomBucketsHistogramSamplesV2, recordType) + +// Float histogram V2 types +floatHistogramsLocal := make([]RefFloatHistogramSample, len(histograms)) +for i, h := range histograms { + floatHistogramsLocal[i] = RefFloatHistogramSample{Ref: h.Ref, T: h.T, FH: h.H.ToFloat(nil)} +} +floatHists, customBucketsFloatHistograms := enc.FloatHistogramSamples(floatHistogramsLocal, nil) +recordType = dec.Type(floatHists) +require.Equal(t, FloatHistogramSamplesV2, recordType) +customBucketsFloatHists := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatHistograms, nil) +recordType = dec.Type(customBucketsFloatHists) +require.Equal(t, CustomBucketsFloatHistogramSamplesV2, recordType) +``` + +### Anti-Patterns to Avoid + +- **Mutating the shared `histograms` slice before V2 tests:** The existing V1 test at line 248 mutates `histograms[i].H.CounterResetHint = histogram.GaugeType`. V2 test blocks added after line 274 will see already-mutated data. Either use fresh data or take a copy before adding V2 test blocks. +- **Testing custom-bucket type without non-custom samples:** If all 3 histograms in the slice use custom buckets, `HistogramSamples()` resets its buffer and returns empty. The test data has 2 standard + 1 custom, which is fine. +- **Forgetting ST in the float histogram copy:** The `ToFloat(nil)` pattern from line 232 does not copy ST, because the existing test data has no ST. For V2 float tests with ST, the conversion loop must also set `ST: h.ST`. + +## Don't Hand-Roll + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| Histogram payload encoding | Custom byte packing | `EncodeHistogram` / `DecodeHistogram` (already in record.go) | These handle all schema variants including custom buckets | +| Test data construction | Novel histogram structs | Reuse `histograms` slice from line 166, set ST field | Already covers standard, negative-buckets, and custom-bucket cases | + +## Common Pitfalls + +### Pitfall 1: Gauge mutation aliasing + +**What goes wrong:** The gauge test block (lines 248-274) mutates `histograms[i].H.CounterResetHint` in-place. Any V2 int-histogram test blocks appended after line 274 will use gauge-type histograms, not counter-type. + +**Why it happens:** Go slices of structs with pointer fields (`*histogram.Histogram`) share the underlying `Histogram` objects when assigned by value. Mutation through `histograms[i].H.CounterResetHint` affects the shared object. + +**How to avoid:** Define V2 test data slices fresh (using `RefHistogramSample{Ref: ..., T: ..., ST: ..., H: histograms[i].H}` where `histograms[i]` refers to the original line-166 slice — but that is already mutated). Safest: define V2 test slices before line 248, or build fresh `histogram.Histogram` values inline for V2 blocks, or reset `CounterResetHint` before the V2 blocks. + +**Warning signs:** Tests pass with `CounterResetHint = histogram.GaugeType` in the decoded histograms when you expected `histogram.UnknownCounterReset`. + +### Pitfall 2: Custom-bucket histogram in the first position + +**What goes wrong:** `histogramSamplesV2` writes the first histogram unconditionally (full varint, no marker). If `histograms[0]` is a custom-bucket histogram, it gets encoded into the main record (not split out), then the decoder returns it via `HistogramSamples()`. But subsequent custom-bucket items are split. This creates an inconsistency in what `HistogramSamples()` returns vs what `CustomBucketsHistogramSamples()` returns. + +**Why it happens:** The V2 encoder only checks `h.H.UsesCustomBuckets()` for items at index `i >= 1`. Index 0 is always written to the main buffer. + +**How to avoid:** In test data, place the custom-bucket histogram (schema=-53) last in the slice — matching the existing test data at line 166 (index 2). The standard histograms are at indices 0 and 1. + +**Warning signs:** `dec.HistogramSamples(histSamples, nil)` returns more items than expected, or `customBucketsHistograms` slice returned by encoder has fewer items than expected. + +### Pitfall 3: Backward compat test must use a separate V1 encoder + +**What goes wrong:** Re-using the `enc` variable that was set to `Encoder{EnableSTStorage: true}` earlier in the function will encode V2 records, not V1. + +**How to avoid:** Declare a local `encV1 := Encoder{}` for the backward compat block (TEST-06). + +### Pitfall 4: ST=0 is the noST case, not "zero start time" + +**What goes wrong:** A test that sets ST=0 on all samples and decodes expects to get ST=0 back. That works. But it is NOT testing the noST encoder branch for samples after the first — for subsequent samples, ST=0 encodes as `noST` marker, and decodes leaving ST at the zero value of the struct. The round-trip produces ST=0, which matches. No problem. + +**Why it matters:** The noST scenario (TEST-01) does work correctly when ST=0 on all samples. Just be aware the encoder writes `noST` byte (not an explicit varint) for subsequent samples, so the test implicitly exercises that path without needing special assertions. + +### Pitfall 5: `dec.Type()` call requires the V2 encoder to be set + +**What goes wrong:** `TestRecord_Type` has `var enc Encoder` at line 537 (V1 encoder). After adding V2 type assertions, the pattern in the function reassigns `enc = Encoder{EnableSTStorage: true}` partway through (line 549). The histogram V2 block must come after that reassignment, OR must use a locally declared V2 encoder. + +**How to avoid:** Add V2 histogram type assertions after line 605 and use the `enc` that was already set to `EnableSTStorage: true` at line 549. + +## Code Examples + +### ST scenario test data (INT histogram) + +```go +// Source: mirrors lines 104-134 (float sample V2 ST scenarios) + +// Reusable base histogram pointers (indices 0 and 1 are standard, index 2 is custom) +// Define these before the gauge mutation at line 248. +h0 := histograms[0].H // standard, schema=1 +h1 := histograms[1].H // standard with negatives, schema=1 +hCB := histograms[2].H // custom buckets, schema=-53 + +// No ST (TEST-01 for int histograms) +histsNoST := []RefHistogramSample{ + {Ref: 56, T: 1234, H: h0}, + {Ref: 42, T: 5678, H: h1}, +} + +// Constant ST (TEST-02 for int histograms) +histsConstST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: h0}, + {Ref: 42, T: 5678, ST: 1000, H: h1}, +} + +// Varying ST — delta case (TEST-03 for int histograms) +histsDeltaST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: h0}, + {Ref: 42, T: 5678, ST: 1234, H: h1}, // ST[1] == T[0] +} + +// All three samples including custom-bucket (TEST-05) +histsWithCB := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: h0}, + {Ref: 42, T: 5678, ST: 1000, H: h1}, + {Ref: 67, T: 5678, ST: 1000, H: hCB}, +} +``` + +### Run command + +```bash +go test ./tsdb/record/ -run TestRecord_EncodeDecode -count=1 -v +go test ./tsdb/record/ -run TestRecord_Type -count=1 -v +go test ./tsdb/record/ -count=1 +``` + +Full package test confirmed working: `ok github.com/prometheus/prometheus/tsdb/record 0.006s` + +## State of the Art + +| Old Approach | Current Approach | When Changed | Impact | +|--------------|------------------|--------------|--------| +| V1 histogram encoding (BE64 baseRef/baseTime) | V2 all-varint + ST marker scheme | Phase 2 of this project | V2 tests require `EnableSTStorage: true`, not just non-zero ST fields | +| No ST field on histogram structs | `ST, T int64` on `RefHistogramSample` and `RefFloatHistogramSample` | Phase 1 of this project | Existing test data can gain ST by setting the field | + +## Open Questions + +1. **Should V2 histogram blocks use `t.Run` subtests?** + - What we know: The existing V2 sample blocks (lines 91-134) do NOT use subtests. The schema/corrupted tests DO use subtests. + - What's unclear: Whether using `t.Run` is preferable for readability given the number of scenario blocks. + - Recommendation: Left to planner. Subtests improve failure isolation. Inline (matching existing pattern) keeps the file consistent with the V2 sample section. Either is correct. + +2. **Should the V2 histogram test blocks also test the gauge variant?** + - What we know: The CONTEXT.md specifically calls out that the gauge variant re-test (lines 248-274) should also be done for V2, matching the existing pattern. + - What's unclear: Whether this is captured in an explicit requirement. + - Recommendation: Include it. The CONTEXT.md `` section says "The gauge variant re-test (lines 248-274) should also be done for V2." It is not a separate TEST-XX requirement but is within scope. + +## Validation Architecture + +(Skipped: `nyquist_validation` not set in config.json.) + +## Sources + +### Primary (HIGH confidence) +- Direct code inspection: `/home/owilliams/src/grafana/prometheus/tsdb/record/record_test.go` — full test file read, all line numbers verified +- Direct code inspection: `/home/owilliams/src/grafana/prometheus/tsdb/record/record.go` — all encoder and decoder implementations read and verified +- Live test run: `go test ./tsdb/record/ -run TestRecord_EncodeDecode -count=1` — passes, 0.006s + +### Secondary (MEDIUM confidence) +- Phase 3 implementation summaries in `.planning/phases/03-v2-decoders/` — confirmed all four decoder methods are complete + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH — pure Go stdlib + testify, already imported +- Architecture: HIGH — patterns read directly from the file, line numbers verified +- Pitfalls: HIGH — identified from direct code analysis, not inference + +**Research date:** 2026-03-03 +**Valid until:** Until record.go changes (stable; 90 days reasonable) diff --git a/.planning/phases/04-tests/04-VERIFICATION.md b/.planning/phases/04-tests/04-VERIFICATION.md new file mode 100644 index 0000000000..da03552087 --- /dev/null +++ b/.planning/phases/04-tests/04-VERIFICATION.md @@ -0,0 +1,87 @@ +--- +phase: 04-tests +verified: 2026-03-03T14:30:00Z +status: passed +score: 9/9 must-haves verified +--- + +# Phase 4: Tests Verification Report + +**Phase Goal:** Full test coverage for V2 histogram encoding, backward compat verified. +**Verified:** 2026-03-03T14:30:00Z +**Status:** passed +**Re-verification:** No -- initial verification + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +|---|-------|--------|----------| +| 1 | V2 int-histogram round-trip produces identical samples for all ST scenarios (no-ST, constant-ST, delta-ST, same-ST) | VERIFIED | Lines 251-313: 4 subtests, all PASS. Each uses `Encoder{EnableSTStorage: true}`, encodes via `HistogramSamples` + `CustomBucketsHistogramSamples`, decodes, `require.Equal`. | +| 2 | V2 float-histogram round-trip produces identical samples for all ST scenarios | VERIFIED | Lines 316-414: 4 subtests, all PASS. Float data derived from int histograms with explicit `ST: h.ST` copy. Encodes via `FloatHistogramSamples` + `CustomBucketsFloatHistogramSamples`. | +| 3 | V2 custom-bucket histograms (int and float) round-trip correctly through separate encode/decode path | VERIFIED | Every scenario includes `histograms[2].H` (schema=-53, `CustomValues` set). Encoder splits it, `CustomBucketsHistogramSamples`/`CustomBucketsFloatHistogramSamples` called in every block. | +| 4 | V2 gauge histograms (int and float) round-trip correctly | VERIFIED | Lines 444-486: 2 subtests ("V2 gauge int-histogram", "V2 gauge float-histogram") use data after `CounterResetHint = GaugeType` mutation. Both PASS. | +| 5 | V1-encoded histograms decode with ST=0 on all samples (backward compat) | VERIFIED | Lines 488-514: 2 subtests use `encV1 := Encoder{}`, encode, decode, assert `h.ST == int64(0)` for both int and float histograms including custom buckets. Both PASS. | +| 6 | dec.Type() returns HistogramSamplesV2 for V2-encoded int-histogram records | VERIFIED | Line 853: `require.Equal(t, HistogramSamplesV2, recordType)` | +| 7 | dec.Type() returns CustomBucketsHistogramSamplesV2 for V2-encoded custom-bucket int-histogram records | VERIFIED | Line 856: `require.Equal(t, CustomBucketsHistogramSamplesV2, recordType)` | +| 8 | dec.Type() returns FloatHistogramSamplesV2 for V2-encoded float-histogram records | VERIFIED | Line 869: `require.Equal(t, FloatHistogramSamplesV2, recordType)` | +| 9 | dec.Type() returns CustomBucketsFloatHistogramSamplesV2 for V2-encoded custom-bucket float-histogram records | VERIFIED | Line 872: `require.Equal(t, CustomBucketsFloatHistogramSamplesV2, recordType)` | + +**Score:** 9/9 truths verified + +### Required Artifacts + +| Artifact | Expected | Status | Details | +|----------|----------|--------|---------| +| `tsdb/record/record_test.go` | V2 histogram round-trip tests and backward compat tests | VERIFIED | 12 new subtests in `TestRecord_EncodeDecode` (8 round-trip + 2 gauge + 2 backward compat). 4 new V2 type assertions in `TestRecord_Type`. File is 1079 lines, substantive, compiles, all tests pass. | + +### Key Link Verification + +| From | To | Via | Status | Details | +|------|----|-----|--------|---------| +| `record_test.go` | `Encoder.HistogramSamples / Decoder.HistogramSamples` | V2 encode/decode round-trip with ST field | WIRED | `EnableSTStorage: true` at lines 249, 446, 850. V2 encode+decode called in every int-histogram subtest. | +| `record_test.go` | `Encoder.FloatHistogramSamples / Decoder.FloatHistogramSamples` | V2 float encode/decode round-trip with ST field | WIRED | `FloatHistogramSamples` called at lines 331, 356, 381, 406, 478, 867. V2 float round-trip in every float-histogram subtest. | +| `record_test.go` | `Encoder{} (V1)` | Backward compat: V1 encode, decode, assert ST=0 | WIRED | `encV1 := Encoder{}` at lines 490, 504. V1 encode + decode + ST=0 assertion for both int and float histograms. | +| `record_test.go` | `Decoder.Type()` | Type assertion for V2 histogram record types | WIRED | Lines 852-872: `dec.Type()` called with all 4 V2 record types, `require.Equal` against expected constants. | + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +|-------------|------------|-------------|--------|----------| +| TEST-01 | 04-01 | Round-trip encode/decode for histogram V2 with no ST | SATISFIED | Subtest "V2 int-histogram no ST" (line 251) | +| TEST-02 | 04-01 | Round-trip encode/decode for histogram V2 with constant ST | SATISFIED | Subtest "V2 int-histogram constant ST" (line 267) | +| TEST-03 | 04-01 | Round-trip encode/decode for histogram V2 with varying ST | SATISFIED | Subtest "V2 int-histogram varying ST" (line 283) | +| TEST-04 | 04-01 | Round-trip encode/decode for float histogram V2 (same ST scenarios) | SATISFIED | 4 float subtests (lines 316-414) | +| TEST-05 | 04-01 | Round-trip encode/decode for custom buckets variants V2 | SATISFIED | Every scenario includes schema=-53 histogram, calls CustomBuckets* encode/decode | +| TEST-06 | 04-01 | V1 records still decode correctly (backward compat test) | SATISFIED | 2 backward compat subtests (lines 488-514), assert ST=0 | +| TEST-07 | 04-02 | Type() correctly identifies new record types | SATISFIED | 4 type assertions in TestRecord_Type (lines 849-872) | + +No orphaned requirements. All 7 TEST-* IDs mapped to Phase 4 in REQUIREMENTS.md are claimed by plans and implemented. + +### Anti-Patterns Found + +| File | Line | Pattern | Severity | Impact | +|------|------|---------|----------|--------| +| (none) | -- | -- | -- | -- | + +No TODOs, FIXMEs, placeholders, empty implementations, or stub patterns found in modified file. + +### Human Verification Required + +None. All verification is automated (test execution + code inspection). The phase is purely about test code, and all tests pass. + +### Gaps Summary + +No gaps found. All 9 must-have truths verified. All 7 requirements satisfied. All key links wired. All commits exist. Full package test suite passes. `go vet` clean. + +**Test execution results:** +- `TestRecord_EncodeDecode`: PASS (12 subtests, 0.005s) +- `TestRecord_Type`: PASS (0.004s) +- Full package `go test ./tsdb/record/ -count=1`: PASS (0.056s) +- `go vet ./tsdb/record/...`: Clean + +--- + +_Verified: 2026-03-03T14:30:00Z_ +_Verifier: Claude (gsd-verifier)_ diff --git a/tsdb/record/bench_test.go b/tsdb/record/bench_test.go index 1420fffc46..fdbe65ad9d 100644 --- a/tsdb/record/bench_test.go +++ b/tsdb/record/bench_test.go @@ -120,88 +120,323 @@ var ( testrecord.WorstCase1000, testrecord.WorstCase1000WithSTSamples, } - UseV2 = true + versions = []struct { + name string + enableST bool + }{ + {"V1", false}, + {"V2", true}, + } ) /* - export bench=encode-v2 && go test ./tsdb/record/... \ + go test ./tsdb/record/... \ -run '^$' -bench '^BenchmarkEncode_Samples' \ -benchtime 5s -count 6 -cpu 2 -timeout 999m \ - | tee ${bench}.txt + | tee encode.txt + benchstat -col /version encode.txt */ func BenchmarkEncode_Samples(b *testing.B) { - for _, compr := range compressions { - for _, data := range dataCases { - b.Run(fmt.Sprintf("compr=%v/data=%v", compr, data), func(b *testing.B) { - var ( - samples = testrecord.GenTestRefSamplesCase(b, data) - enc = record.Encoder{EnableSTStorage: UseV2} - buf []byte - cBuf []byte - ) + for _, ver := range versions { + for _, compr := range compressions { + for _, data := range dataCases { + b.Run(fmt.Sprintf("version=%s/compr=%v/data=%v", ver.name, compr, data), func(b *testing.B) { + var ( + samples = testrecord.GenTestRefSamplesCase(b, data) + enc = record.Encoder{EnableSTStorage: ver.enableST} + buf []byte + cBuf []byte + ) - cEnc, err := compression.NewEncoder() - require.NoError(b, err) + cEnc, err := compression.NewEncoder() + require.NoError(b, err) - // Warm up. - buf = enc.Samples(samples, buf[:0]) - cBuf, _, err = cEnc.Encode(compr, buf, cBuf[:0]) - require.NoError(b, err) - - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { + // Warm up. buf = enc.Samples(samples, buf[:0]) - b.ReportMetric(float64(len(buf)), "B/rec") + cBuf, _, err = cEnc.Encode(compr, buf, cBuf[:0]) + require.NoError(b, err) - cBuf, _, _ = cEnc.Encode(compr, buf, cBuf[:0]) - b.ReportMetric(float64(len(cBuf)), "B/compressed-rec") - } - }) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + buf = enc.Samples(samples, buf[:0]) + b.ReportMetric(float64(len(buf)), "B/rec") + + cBuf, _, _ = cEnc.Encode(compr, buf, cBuf[:0]) + b.ReportMetric(float64(len(cBuf)), "B/compressed-rec") + } + }) + } } } } /* - export bench=decode-v2 && go test ./tsdb/record/... \ + go test ./tsdb/record/... \ -run '^$' -bench '^BenchmarkDecode_Samples' \ -benchtime 5s -count 6 -cpu 2 -timeout 999m \ - | tee ${bench}.txt + | tee decode.txt + benchstat -col /version decode.txt */ func BenchmarkDecode_Samples(b *testing.B) { - for _, compr := range compressions { - for _, data := range dataCases { - b.Run(fmt.Sprintf("compr=%v/data=%v", compr, data), func(b *testing.B) { - var ( - samples = testrecord.GenTestRefSamplesCase(b, data) - enc = record.Encoder{EnableSTStorage: UseV2} - dec record.Decoder - cDec = compression.NewDecoder() - cBuf []byte - samplesBuf []record.RefSample - ) + for _, ver := range versions { + for _, compr := range compressions { + for _, data := range dataCases { + b.Run(fmt.Sprintf("version=%s/compr=%v/data=%v", ver.name, compr, data), func(b *testing.B) { + var ( + samples = testrecord.GenTestRefSamplesCase(b, data) + enc = record.Encoder{EnableSTStorage: ver.enableST} + dec record.Decoder + cDec = compression.NewDecoder() + cBuf []byte + samplesBuf []record.RefSample + ) - buf := enc.Samples(samples, nil) + buf := enc.Samples(samples, nil) - cEnc, err := compression.NewEncoder() - require.NoError(b, err) + cEnc, err := compression.NewEncoder() + require.NoError(b, err) - buf, _, err = cEnc.Encode(compr, buf, nil) - require.NoError(b, err) + buf, _, err = cEnc.Encode(compr, buf, nil) + require.NoError(b, err) - // Warm up. - cBuf, err = cDec.Decode(compr, buf, cBuf[:0]) - require.NoError(b, err) - samplesBuf, err = dec.Samples(cBuf, samplesBuf[:0]) - require.NoError(b, err) + // Warm up. + cBuf, err = cDec.Decode(compr, buf, cBuf[:0]) + require.NoError(b, err) + samplesBuf, err = dec.Samples(cBuf, samplesBuf[:0]) + require.NoError(b, err) - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { - cBuf, _ = cDec.Decode(compr, buf, cBuf[:0]) - samplesBuf, _ = dec.Samples(cBuf, samplesBuf[:0]) - } - }) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + cBuf, _ = cDec.Decode(compr, buf, cBuf[:0]) + samplesBuf, _ = dec.Samples(cBuf, samplesBuf[:0]) + } + }) + } + } + } +} + +var ( + histDataCases = testrecord.HistDataCases + histCounts = testrecord.HistCounts +) + +/* + go test ./tsdb/record/... \ + -run '^$' -bench '^BenchmarkEncode_Histograms' \ + -benchtime 5s -count 6 -cpu 2 -timeout 999m \ + | tee encode-hist.txt + benchstat -col /version encode-hist.txt +*/ +func BenchmarkEncode_Histograms(b *testing.B) { + for _, ver := range versions { + for _, compr := range compressions { + for _, hcase := range histDataCases { + for _, stCase := range testrecord.HistSTCases { + for _, count := range histCounts { + b.Run(fmt.Sprintf("version=%s/compr=%v/type=%s/st=%s/n=%d", ver.name, compr, hcase.Name, stCase, count), func(b *testing.B) { + var ( + samples = hcase.Gen(count, stCase) + enc = record.Encoder{EnableSTStorage: ver.enableST} + buf []byte + cBuf []byte + ) + + cEnc, err := compression.NewEncoder() + require.NoError(b, err) + + // Warm up. + if hcase.Name == "nhcb" { + buf = enc.CustomBucketsHistogramSamples(samples, buf[:0]) + } else { + buf, _ = enc.HistogramSamples(samples, buf[:0]) + } + cBuf, _, err = cEnc.Encode(compr, buf, cBuf[:0]) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if hcase.Name == "nhcb" { + buf = enc.CustomBucketsHistogramSamples(samples, buf[:0]) + } else { + buf, _ = enc.HistogramSamples(samples, buf[:0]) + } + b.ReportMetric(float64(len(buf)), "B/rec") + + cBuf, _, _ = cEnc.Encode(compr, buf, cBuf[:0]) + b.ReportMetric(float64(len(cBuf)), "B/compressed-rec") + } + }) + } + } + } + } + } +} + +/* + go test ./tsdb/record/... \ + -run '^$' -bench '^BenchmarkDecode_Histograms' \ + -benchtime 5s -count 6 -cpu 2 -timeout 999m \ + | tee decode-hist.txt + benchstat -col /version decode-hist.txt +*/ +func BenchmarkDecode_Histograms(b *testing.B) { + for _, ver := range versions { + for _, compr := range compressions { + for _, hcase := range histDataCases { + for _, stCase := range testrecord.HistSTCases { + for _, count := range histCounts { + b.Run(fmt.Sprintf("version=%s/compr=%v/type=%s/st=%s/n=%d", ver.name, compr, hcase.Name, stCase, count), func(b *testing.B) { + var ( + samples = hcase.Gen(count, stCase) + enc = record.Encoder{EnableSTStorage: ver.enableST} + dec record.Decoder + cDec = compression.NewDecoder() + cBuf []byte + samplesBuf []record.RefHistogramSample + ) + + var buf []byte + if hcase.Name == "nhcb" { + buf = enc.CustomBucketsHistogramSamples(samples, nil) + } else { + buf, _ = enc.HistogramSamples(samples, nil) + } + + cEnc, err := compression.NewEncoder() + require.NoError(b, err) + buf, _, err = cEnc.Encode(compr, buf, nil) + require.NoError(b, err) + + // Warm up. + cBuf, err = cDec.Decode(compr, buf, cBuf[:0]) + require.NoError(b, err) + samplesBuf, err = dec.HistogramSamples(cBuf, samplesBuf[:0]) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + cBuf, _ = cDec.Decode(compr, buf, cBuf[:0]) + samplesBuf, _ = dec.HistogramSamples(cBuf, samplesBuf[:0]) + } + }) + } + } + } + } + } +} + +/* + go test ./tsdb/record/... \ + -run '^$' -bench '^BenchmarkEncode_FloatHistograms' \ + -benchtime 5s -count 6 -cpu 2 -timeout 999m \ + | tee encode-fhist.txt + benchstat -col /version encode-fhist.txt +*/ +func BenchmarkEncode_FloatHistograms(b *testing.B) { + for _, ver := range versions { + for _, compr := range compressions { + for _, hcase := range histDataCases { + for _, stCase := range testrecord.HistSTCases { + for _, count := range histCounts { + b.Run(fmt.Sprintf("version=%s/compr=%v/type=%s/st=%s/n=%d", ver.name, compr, hcase.Name, stCase, count), func(b *testing.B) { + var ( + samples = testrecord.GenFloatHistograms(hcase.Gen(count, stCase)) + enc = record.Encoder{EnableSTStorage: ver.enableST} + buf []byte + cBuf []byte + ) + + cEnc, err := compression.NewEncoder() + require.NoError(b, err) + + // Warm up. + if hcase.Name == "nhcb" { + buf = enc.CustomBucketsFloatHistogramSamples(samples, buf[:0]) + } else { + buf, _ = enc.FloatHistogramSamples(samples, buf[:0]) + } + cBuf, _, err = cEnc.Encode(compr, buf, cBuf[:0]) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if hcase.Name == "nhcb" { + buf = enc.CustomBucketsFloatHistogramSamples(samples, buf[:0]) + } else { + buf, _ = enc.FloatHistogramSamples(samples, buf[:0]) + } + b.ReportMetric(float64(len(buf)), "B/rec") + + cBuf, _, _ = cEnc.Encode(compr, buf, cBuf[:0]) + b.ReportMetric(float64(len(cBuf)), "B/compressed-rec") + } + }) + } + } + } + } + } +} + +/* + go test ./tsdb/record/... \ + -run '^$' -bench '^BenchmarkDecode_FloatHistograms' \ + -benchtime 5s -count 6 -cpu 2 -timeout 999m \ + | tee decode-fhist.txt + benchstat -col /version decode-fhist.txt +*/ +func BenchmarkDecode_FloatHistograms(b *testing.B) { + for _, ver := range versions { + for _, compr := range compressions { + for _, hcase := range histDataCases { + for _, stCase := range testrecord.HistSTCases { + for _, count := range histCounts { + b.Run(fmt.Sprintf("version=%s/compr=%v/type=%s/st=%s/n=%d", ver.name, compr, hcase.Name, stCase, count), func(b *testing.B) { + var ( + samples = testrecord.GenFloatHistograms(hcase.Gen(count, stCase)) + enc = record.Encoder{EnableSTStorage: ver.enableST} + dec record.Decoder + cDec = compression.NewDecoder() + cBuf []byte + samplesBuf []record.RefFloatHistogramSample + ) + + var buf []byte + if hcase.Name == "nhcb" { + buf = enc.CustomBucketsFloatHistogramSamples(samples, nil) + } else { + buf, _ = enc.FloatHistogramSamples(samples, nil) + } + + cEnc, err := compression.NewEncoder() + require.NoError(b, err) + buf, _, err = cEnc.Encode(compr, buf, nil) + require.NoError(b, err) + + // Warm up. + cBuf, err = cDec.Decode(compr, buf, cBuf[:0]) + require.NoError(b, err) + samplesBuf, err = dec.FloatHistogramSamples(cBuf, samplesBuf[:0]) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + cBuf, _ = cDec.Decode(compr, buf, cBuf[:0]) + samplesBuf, _ = dec.FloatHistogramSamples(cBuf, samplesBuf[:0]) + } + }) + } + } + } } } } diff --git a/tsdb/record/record.go b/tsdb/record/record.go index 2a4f45e490..0839146a57 100644 --- a/tsdb/record/record.go +++ b/tsdb/record/record.go @@ -60,6 +60,14 @@ const ( CustomBucketsFloatHistogramSamples Type = 10 // SamplesV2 is an enhanced sample record with an encoding scheme that allows storing float samples with timestamp and an optional ST per sample. SamplesV2 Type = 11 + // HistogramSamplesV2 is an enhanced histogram record that supports start time per sample. + HistogramSamplesV2 Type = 12 + // FloatHistogramSamplesV2 is an enhanced float histogram record that supports start time per sample. + FloatHistogramSamplesV2 Type = 13 + // CustomBucketsHistogramSamplesV2 is an enhanced custom-buckets histogram record that supports start time per sample. + CustomBucketsHistogramSamplesV2 Type = 14 + // CustomBucketsFloatHistogramSamplesV2 is an enhanced custom-buckets float histogram record that supports start time per sample. + CustomBucketsFloatHistogramSamplesV2 Type = 15 ) func (rt Type) String() string { @@ -69,7 +77,7 @@ func (rt Type) String() string { case Samples: return "samples" case SamplesV2: - return "samples-v2" + return "samples_v2" case Tombstones: return "tombstones" case Exemplars: @@ -82,6 +90,14 @@ func (rt Type) String() string { return "custom_buckets_histogram_samples" case CustomBucketsFloatHistogramSamples: return "custom_buckets_float_histogram_samples" + case HistogramSamplesV2: + return "histogram_samples_v2" + case FloatHistogramSamplesV2: + return "float_histogram_samples_v2" + case CustomBucketsHistogramSamplesV2: + return "custom_buckets_histogram_samples_v2" + case CustomBucketsFloatHistogramSamplesV2: + return "custom_buckets_float_histogram_samples_v2" case MmapMarkers: return "mmapmarkers" case Metadata: @@ -186,19 +202,17 @@ type RefExemplar struct { } // RefHistogramSample is a histogram. -// TODO(owilliams): Add support for ST. type RefHistogramSample struct { - Ref chunks.HeadSeriesRef - T int64 - H *histogram.Histogram + Ref chunks.HeadSeriesRef + ST, T int64 + H *histogram.Histogram } // RefFloatHistogramSample is a float histogram. -// TODO(owilliams): Add support for ST. type RefFloatHistogramSample struct { - Ref chunks.HeadSeriesRef - T int64 - FH *histogram.FloatHistogram + Ref chunks.HeadSeriesRef + ST, T int64 + FH *histogram.FloatHistogram } // RefMmapMarker marks that the all the samples of the given series until now have been m-mapped to disk. @@ -226,7 +240,9 @@ func (*Decoder) Type(rec []byte) Type { return Unknown } switch t := Type(rec[0]); t { - case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples: + case Series, Samples, SamplesV2, Tombstones, Exemplars, MmapMarkers, Metadata, + HistogramSamples, FloatHistogramSamples, CustomBucketsHistogramSamples, CustomBucketsFloatHistogramSamples, + HistogramSamplesV2, FloatHistogramSamplesV2, CustomBucketsHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2: return t } return Unknown @@ -512,10 +528,18 @@ func (*Decoder) MmapMarkers(rec []byte, markers []RefMmapMarker) ([]RefMmapMarke func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) ([]RefHistogramSample, error) { dec := encoding.Decbuf{B: rec} - t := Type(dec.Byte()) - if t != HistogramSamples && t != CustomBucketsHistogramSamples { - return nil, errors.New("invalid record type") + switch typ := Type(dec.Byte()); typ { + case HistogramSamples, CustomBucketsHistogramSamples: + return d.histogramSamplesV1(&dec, histograms) + case HistogramSamplesV2, CustomBucketsHistogramSamplesV2: + return d.histogramSamplesV2(&dec, histograms) + default: + return nil, fmt.Errorf("invalid record type %v", typ) } +} + +// histogramSamplesV1 decodes V1 int-histogram records (BE64 baseRef/baseTime, varint deltas). +func (d *Decoder) histogramSamplesV1(dec *encoding.Decbuf, histograms []RefHistogramSample) ([]RefHistogramSample, error) { if dec.Len() == 0 { return histograms, nil } @@ -533,7 +557,72 @@ func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) H: &histogram.Histogram{}, } - DecodeHistogram(&dec, rh.H) + DecodeHistogram(dec, rh.H) + + if !histogram.IsKnownSchema(rh.H.Schema) { + d.logger.Warn("skipping histogram with unknown schema in WAL record", "schema", rh.H.Schema, "timestamp", rh.T) + continue + } + if rh.H.Schema > histogram.ExponentialSchemaMax && rh.H.Schema <= histogram.ExponentialSchemaMaxReserved { + // This is a very slow path, but it should only happen if the + // record is from a newer Prometheus version that supports higher + // resolution. + if err := rh.H.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { + return nil, fmt.Errorf("error reducing resolution of histogram #%d: %w", len(histograms)+1, err) + } + } + + histograms = append(histograms, rh) + } + + if dec.Err() != nil { + return nil, fmt.Errorf("decode error after %d histograms: %w", len(histograms), dec.Err()) + } + if len(dec.B) > 0 { + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) + } + return histograms, nil +} + +// histogramSamplesV2 decodes V2 int-histogram records (all-varint, ST marker scheme). +func (d *Decoder) histogramSamplesV2(dec *encoding.Decbuf, histograms []RefHistogramSample) ([]RefHistogramSample, error) { + if dec.Len() == 0 { + return histograms, nil + } + firstRef := chunks.HeadSeriesRef(dec.Varint64()) + firstT := dec.Varint64() + firstST := dec.Varint64() + var prev *RefHistogramSample + + for len(dec.B) > 0 && dec.Err() == nil { + if prev == nil || len(histograms) == 0 { + prev = &RefHistogramSample{ + Ref: firstRef, + ST: firstST, + } + } else { + prev = &histograms[len(histograms)-1] + } + + ref := int64(prev.Ref) + dec.Varint64() + t := firstT + dec.Varint64() + stMarker := dec.Byte() + var ST int64 + switch stMarker { + case noST: + case sameST: + ST = prev.ST + default: + ST = firstST + dec.Varint64() + } + + rh := RefHistogramSample{ + Ref: chunks.HeadSeriesRef(ref), + ST: ST, + T: t, + H: &histogram.Histogram{}, + } + DecodeHistogram(dec, rh.H) if !histogram.IsKnownSchema(rh.H.Schema) { d.logger.Warn("skipping histogram with unknown schema in WAL record", "schema", rh.H.Schema, "timestamp", rh.T) @@ -618,10 +707,18 @@ func DecodeHistogram(buf *encoding.Decbuf, h *histogram.Histogram) { func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error) { dec := encoding.Decbuf{B: rec} - t := Type(dec.Byte()) - if t != FloatHistogramSamples && t != CustomBucketsFloatHistogramSamples { - return nil, errors.New("invalid record type") + switch typ := Type(dec.Byte()); typ { + case FloatHistogramSamples, CustomBucketsFloatHistogramSamples: + return d.floatHistogramSamplesV1(&dec, histograms) + case FloatHistogramSamplesV2, CustomBucketsFloatHistogramSamplesV2: + return d.floatHistogramSamplesV2(&dec, histograms) + default: + return nil, fmt.Errorf("invalid record type %v", typ) } +} + +// floatHistogramSamplesV1 decodes V1 float-histogram records (BE64 baseRef/baseTime, varint deltas). +func (d *Decoder) floatHistogramSamplesV1(dec *encoding.Decbuf, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error) { if dec.Len() == 0 { return histograms, nil } @@ -639,7 +736,7 @@ func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogr FH: &histogram.FloatHistogram{}, } - DecodeFloatHistogram(&dec, rh.FH) + DecodeFloatHistogram(dec, rh.FH) if !histogram.IsKnownSchema(rh.FH.Schema) { d.logger.Warn("skipping histogram with unknown schema in WAL record", "schema", rh.FH.Schema, "timestamp", rh.T) @@ -666,6 +763,71 @@ func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogr return histograms, nil } +// floatHistogramSamplesV2 decodes V2 float-histogram records (all-varint, ST marker scheme). +func (d *Decoder) floatHistogramSamplesV2(dec *encoding.Decbuf, histograms []RefFloatHistogramSample) ([]RefFloatHistogramSample, error) { + if dec.Len() == 0 { + return histograms, nil + } + firstRef := chunks.HeadSeriesRef(dec.Varint64()) + firstT := dec.Varint64() + firstST := dec.Varint64() + var prev *RefFloatHistogramSample + + for len(dec.B) > 0 && dec.Err() == nil { + if prev == nil || len(histograms) == 0 { + prev = &RefFloatHistogramSample{ + Ref: firstRef, + ST: firstST, + } + } else { + prev = &histograms[len(histograms)-1] + } + + ref := int64(prev.Ref) + dec.Varint64() + t := firstT + dec.Varint64() + stMarker := dec.Byte() + var ST int64 + switch stMarker { + case noST: + case sameST: + ST = prev.ST + default: + ST = firstST + dec.Varint64() + } + + rfh := RefFloatHistogramSample{ + Ref: chunks.HeadSeriesRef(ref), + ST: ST, + T: t, + FH: &histogram.FloatHistogram{}, + } + DecodeFloatHistogram(dec, rfh.FH) + + if !histogram.IsKnownSchema(rfh.FH.Schema) { + d.logger.Warn("skipping histogram with unknown schema in WAL record", "schema", rfh.FH.Schema, "timestamp", rfh.T) + continue + } + if rfh.FH.Schema > histogram.ExponentialSchemaMax && rfh.FH.Schema <= histogram.ExponentialSchemaMaxReserved { + // This is a very slow path, but it should only happen if the + // record is from a newer Prometheus version that supports higher + // resolution. + if err := rfh.FH.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { + return nil, fmt.Errorf("error reducing resolution of histogram #%d: %w", len(histograms)+1, err) + } + } + + histograms = append(histograms, rfh) + } + + if dec.Err() != nil { + return nil, fmt.Errorf("decode error after %d histograms: %w", len(histograms), dec.Err()) + } + if len(dec.B) > 0 { + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) + } + return histograms, nil +} + // DecodeFloatHistogram decodes a Histogram from a byte slice. func DecodeFloatHistogram(buf *encoding.Decbuf, fh *histogram.FloatHistogram) { fh.CounterResetHint = histogram.CounterResetHint(buf.Byte()) @@ -914,7 +1076,14 @@ func (*Encoder) MmapMarkers(markers []RefMmapMarker, b []byte) []byte { // HistogramSamples encode exponential histograms while returning all the excluded custom bucket histograms. // Callers can encode the returned custom bucket histograms via CustomBucketsHistogramSamples. -func (*Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { +func (e *Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + if e.EnableSTStorage { + return e.histogramSamplesV2(histograms, b) + } + return e.histogramSamplesV1(histograms, b) +} + +func (*Encoder) histogramSamplesV1(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { buf := encoding.Encbuf{B: b} buf.PutByte(byte(HistogramSamples)) @@ -948,8 +1117,65 @@ func (*Encoder) HistogramSamples(histograms []RefHistogramSample, b []byte) ([]b return buf.Get(), customBucketHistograms } -// CustomBucketsHistogramSamples encodes given histograms as custom bucket histograms. -func (*Encoder) CustomBucketsHistogramSamples(histograms []RefHistogramSample, b []byte) []byte { +// histogramSamplesV2 encodes the given histogram samples, and splits off all +// custom bucket histograms into a separate slice that is returned to be +// processed separately. +func (*Encoder) histogramSamplesV2(histograms []RefHistogramSample, b []byte) ([]byte, []RefHistogramSample) { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(HistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get(), nil + } + + var customBucketHistograms []RefHistogramSample + + var first, prev *RefHistogramSample + for _, h := range histograms { + if h.H.UsesCustomBuckets() { + customBucketHistograms = append(customBucketHistograms, h) + continue + } + if first == nil { + first = &h + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + prev = first + } + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeHistogram(&buf, h.H) + prev = &h + } + + // Reset buffer if only custom bucket histograms existed in list of histogram samples. + if len(histograms) == len(customBucketHistograms) { + buf.Reset() + } + + return buf.Get(), customBucketHistograms +} + +func (e *Encoder) CustomBucketsHistogramSamples(histograms []RefHistogramSample, b []byte) []byte { + if e.EnableSTStorage { + return e.customBucketsHistogramSamplesV2(histograms, b) + } + return e.customBucketsHistogramSamplesV1(histograms, b) +} + +func (*Encoder) customBucketsHistogramSamplesV1(histograms []RefHistogramSample, b []byte) []byte { buf := encoding.Encbuf{B: b} buf.PutByte(byte(CustomBucketsHistogramSamples)) @@ -973,7 +1199,48 @@ func (*Encoder) CustomBucketsHistogramSamples(histograms []RefHistogramSample, b return buf.Get() } -// EncodeHistogram encodes a Histogram into a byte slice. +// customBucketsHistogramSamplesV2 encodes the givem custom bucket histograms. +func (*Encoder) customBucketsHistogramSamplesV2(histograms []RefHistogramSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(CustomBucketsHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get() + } + + var first *RefHistogramSample + for i, h := range histograms { + var prev *RefHistogramSample + if i == 0 { + first = &h + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + prev = first + } else { + prev = &histograms[i-1] + } + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeHistogram(&buf, h.H) + } + + return buf.Get() +} + +// EncodeHistogram encodes a Histogram into a byte slice. Handles both +// regular and custom bucket histograms. func EncodeHistogram(buf *encoding.Encbuf, h *histogram.Histogram) { buf.PutByte(byte(h.CounterResetHint)) @@ -1016,7 +1283,14 @@ func EncodeHistogram(buf *encoding.Encbuf, h *histogram.Histogram) { // FloatHistogramSamples encode exponential float histograms while returning all the excluded custom bucket float histograms. // Callers can encode the returned custom bucket float histograms via CustomBucketsFloatHistogramSamples. -func (*Encoder) FloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) ([]byte, []RefFloatHistogramSample) { +func (e *Encoder) FloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) ([]byte, []RefFloatHistogramSample) { + if e.EnableSTStorage { + return e.floatHistogramSamplesV2(histograms, b) + } + return e.floatHistogramSamplesV1(histograms, b) +} + +func (*Encoder) floatHistogramSamplesV1(histograms []RefFloatHistogramSample, b []byte) ([]byte, []RefFloatHistogramSample) { buf := encoding.Encbuf{B: b} buf.PutByte(byte(FloatHistogramSamples)) @@ -1052,7 +1326,62 @@ func (*Encoder) FloatHistogramSamples(histograms []RefFloatHistogramSample, b [] } // CustomBucketsFloatHistogramSamples encodes given float histograms as custom bucket float histograms. -func (*Encoder) CustomBucketsFloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) []byte { +func (*Encoder) floatHistogramSamplesV2(histograms []RefFloatHistogramSample, b []byte) ([]byte, []RefFloatHistogramSample) { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(FloatHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get(), nil + } + + var customBucketsFloatHistograms []RefFloatHistogramSample + + var first, prev *RefFloatHistogramSample + for _, fh := range histograms { + if fh.FH.UsesCustomBuckets() { + customBucketsFloatHistograms = append(customBucketsFloatHistograms, fh) + continue + } + if first == nil { + first = &fh + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + prev = first + } + + buf.PutVarint64(int64(fh.Ref) - int64(prev.Ref)) + buf.PutVarint64(fh.T - first.T) + + switch fh.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(fh.ST - first.ST) + } + EncodeFloatHistogram(&buf, fh.FH) + prev = &fh + } + + // Reset buffer if only custom bucket histograms existed in list of histogram samples + if len(histograms) == len(customBucketsFloatHistograms) { + buf.Reset() + } + + return buf.Get(), customBucketsFloatHistograms +} + +func (e *Encoder) CustomBucketsFloatHistogramSamples(histograms []RefFloatHistogramSample, b []byte) []byte { + if e.EnableSTStorage { + return e.customBucketsFloatHistogramSamplesV2(histograms, b) + } + return e.customBucketsFloatHistogramSamplesV1(histograms, b) +} + +func (*Encoder) customBucketsFloatHistogramSamplesV1(histograms []RefFloatHistogramSample, b []byte) []byte { buf := encoding.Encbuf{B: b} buf.PutByte(byte(CustomBucketsFloatHistogramSamples)) @@ -1076,6 +1405,45 @@ func (*Encoder) CustomBucketsFloatHistogramSamples(histograms []RefFloatHistogra return buf.Get() } +func (*Encoder) customBucketsFloatHistogramSamplesV2(histograms []RefFloatHistogramSample, b []byte) []byte { + buf := encoding.Encbuf{B: b} + buf.PutByte(byte(CustomBucketsFloatHistogramSamplesV2)) + + if len(histograms) == 0 { + return buf.Get() + } + + var first *RefFloatHistogramSample + for i, h := range histograms { + var prev *RefFloatHistogramSample + if i == 0 { + first = &h + buf.PutVarint64(int64(first.Ref)) + buf.PutVarint64(first.T) + buf.PutVarint64(first.ST) + prev = first + } else { + prev = &histograms[i-1] + } + + buf.PutVarint64(int64(h.Ref) - int64(prev.Ref)) + buf.PutVarint64(h.T - first.T) + + switch h.ST { + case 0: + buf.PutByte(noST) + case prev.ST: + buf.PutByte(sameST) + default: + buf.PutByte(explicitST) + buf.PutVarint64(h.ST - first.ST) + } + EncodeFloatHistogram(&buf, h.FH) + } + + return buf.Get() +} + // EncodeFloatHistogram encodes the Float Histogram into a byte slice. func EncodeFloatHistogram(buf *encoding.Encbuf, h *histogram.FloatHistogram) { buf.PutByte(byte(h.CounterResetHint)) diff --git a/tsdb/record/record_test.go b/tsdb/record/record_test.go index 970930fbe5..2a6bdfcc5d 100644 --- a/tsdb/record/record_test.go +++ b/tsdb/record/record_test.go @@ -245,6 +245,174 @@ func TestRecord_EncodeDecode(t *testing.T) { decFloatHistograms = append(decFloatHistograms, decCustomBucketsFloatHistograms...) require.Equal(t, floatHistograms, decFloatHistograms) + // V2 int-histogram round-trip tests covering all ST scenarios. + enc = Encoder{EnableSTStorage: true} + + t.Run("V2 int-histogram no ST", func(t *testing.T) { + histsV2NoST := []RefHistogramSample{ + {Ref: 56, T: 1234, H: histograms[0].H}, + {Ref: 42, T: 5678, H: histograms[1].H}, + {Ref: 67, T: 5678, H: histograms[2].H}, + } + histSamplesV2, customBucketsV2 := enc.HistogramSamples(histsV2NoST, nil) + customBucketsHistSamplesV2 := enc.CustomBucketsHistogramSamples(customBucketsV2, nil) + decHistsV2, err := dec.HistogramSamples(histSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsV2, err := dec.HistogramSamples(customBucketsHistSamplesV2, nil) + require.NoError(t, err) + decHistsV2 = append(decHistsV2, decCustomBucketsV2...) + require.Equal(t, histsV2NoST, decHistsV2) + }) + + t.Run("V2 int-histogram constant ST", func(t *testing.T) { + histsV2ConstST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1000, H: histograms[1].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, + } + histSamplesV2, customBucketsV2 := enc.HistogramSamples(histsV2ConstST, nil) + customBucketsHistSamplesV2 := enc.CustomBucketsHistogramSamples(customBucketsV2, nil) + decHistsV2, err := dec.HistogramSamples(histSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsV2, err := dec.HistogramSamples(customBucketsHistSamplesV2, nil) + require.NoError(t, err) + decHistsV2 = append(decHistsV2, decCustomBucketsV2...) + require.Equal(t, histsV2ConstST, decHistsV2) + }) + + t.Run("V2 int-histogram varying ST", func(t *testing.T) { + histsV2VarST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1234, H: histograms[1].H}, + {Ref: 67, T: 9012, ST: 5678, H: histograms[2].H}, + } + histSamplesV2, customBucketsV2 := enc.HistogramSamples(histsV2VarST, nil) + customBucketsHistSamplesV2 := enc.CustomBucketsHistogramSamples(customBucketsV2, nil) + decHistsV2, err := dec.HistogramSamples(histSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsV2, err := dec.HistogramSamples(customBucketsHistSamplesV2, nil) + require.NoError(t, err) + decHistsV2 = append(decHistsV2, decCustomBucketsV2...) + require.Equal(t, histsV2VarST, decHistsV2) + }) + + t.Run("V2 int-histogram same ST across samples", func(t *testing.T) { + histsV2SameST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 900, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 900, H: histograms[1].H}, + {Ref: 67, T: 9012, ST: 900, H: histograms[2].H}, + } + histSamplesV2, customBucketsV2 := enc.HistogramSamples(histsV2SameST, nil) + customBucketsHistSamplesV2 := enc.CustomBucketsHistogramSamples(customBucketsV2, nil) + decHistsV2, err := dec.HistogramSamples(histSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsV2, err := dec.HistogramSamples(customBucketsHistSamplesV2, nil) + require.NoError(t, err) + decHistsV2 = append(decHistsV2, decCustomBucketsV2...) + require.Equal(t, histsV2SameST, decHistsV2) + }) + + // V2 float-histogram round-trip tests covering all ST scenarios. + t.Run("V2 float-histogram no ST", func(t *testing.T) { + histsV2NoST := []RefHistogramSample{ + {Ref: 56, T: 1234, H: histograms[0].H}, + {Ref: 42, T: 5678, H: histograms[1].H}, + {Ref: 67, T: 5678, H: histograms[2].H}, + } + floatHistsV2 := make([]RefFloatHistogramSample, len(histsV2NoST)) + for i, h := range histsV2NoST { + floatHistsV2[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } + } + floatHistSamplesV2, customBucketsFloatV2 := enc.FloatHistogramSamples(floatHistsV2, nil) + customBucketsFloatHistSamplesV2 := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatV2, nil) + decFloatHistsV2, err := dec.FloatHistogramSamples(floatHistSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsFloatV2, err := dec.FloatHistogramSamples(customBucketsFloatHistSamplesV2, nil) + require.NoError(t, err) + decFloatHistsV2 = append(decFloatHistsV2, decCustomBucketsFloatV2...) + require.Equal(t, floatHistsV2, decFloatHistsV2) + }) + + t.Run("V2 float-histogram constant ST", func(t *testing.T) { + histsV2ConstST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1000, H: histograms[1].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, + } + floatHistsV2 := make([]RefFloatHistogramSample, len(histsV2ConstST)) + for i, h := range histsV2ConstST { + floatHistsV2[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } + } + floatHistSamplesV2, customBucketsFloatV2 := enc.FloatHistogramSamples(floatHistsV2, nil) + customBucketsFloatHistSamplesV2 := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatV2, nil) + decFloatHistsV2, err := dec.FloatHistogramSamples(floatHistSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsFloatV2, err := dec.FloatHistogramSamples(customBucketsFloatHistSamplesV2, nil) + require.NoError(t, err) + decFloatHistsV2 = append(decFloatHistsV2, decCustomBucketsFloatV2...) + require.Equal(t, floatHistsV2, decFloatHistsV2) + }) + + t.Run("V2 float-histogram varying ST", func(t *testing.T) { + histsV2VarST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1234, H: histograms[1].H}, + {Ref: 67, T: 9012, ST: 5678, H: histograms[2].H}, + } + floatHistsV2 := make([]RefFloatHistogramSample, len(histsV2VarST)) + for i, h := range histsV2VarST { + floatHistsV2[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } + } + floatHistSamplesV2, customBucketsFloatV2 := enc.FloatHistogramSamples(floatHistsV2, nil) + customBucketsFloatHistSamplesV2 := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatV2, nil) + decFloatHistsV2, err := dec.FloatHistogramSamples(floatHistSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsFloatV2, err := dec.FloatHistogramSamples(customBucketsFloatHistSamplesV2, nil) + require.NoError(t, err) + decFloatHistsV2 = append(decFloatHistsV2, decCustomBucketsFloatV2...) + require.Equal(t, floatHistsV2, decFloatHistsV2) + }) + + t.Run("V2 float-histogram same ST across samples", func(t *testing.T) { + histsV2SameST := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 900, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 900, H: histograms[1].H}, + {Ref: 67, T: 9012, ST: 900, H: histograms[2].H}, + } + floatHistsV2 := make([]RefFloatHistogramSample, len(histsV2SameST)) + for i, h := range histsV2SameST { + floatHistsV2[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } + } + floatHistSamplesV2, customBucketsFloatV2 := enc.FloatHistogramSamples(floatHistsV2, nil) + customBucketsFloatHistSamplesV2 := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatV2, nil) + decFloatHistsV2, err := dec.FloatHistogramSamples(floatHistSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsFloatV2, err := dec.FloatHistogramSamples(customBucketsFloatHistSamplesV2, nil) + require.NoError(t, err) + decFloatHistsV2 = append(decFloatHistsV2, decCustomBucketsFloatV2...) + require.Equal(t, floatHistsV2, decFloatHistsV2) + }) + // Gauge integer histograms. for i := range histograms { histograms[i].H.CounterResetHint = histogram.GaugeType @@ -272,6 +440,349 @@ func TestRecord_EncodeDecode(t *testing.T) { require.NoError(t, err) decGaugeFloatHistograms = append(decGaugeFloatHistograms, decCustomBucketsGaugeFloatHistograms...) require.Equal(t, floatHistograms, decGaugeFloatHistograms) + + // V2 gauge int-histogram round-trip. + t.Run("V2 gauge int-histogram", func(t *testing.T) { + enc = Encoder{EnableSTStorage: true} + gaugeHistsV2 := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1000, H: histograms[1].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, + } + histSamplesV2, customBucketsV2 := enc.HistogramSamples(gaugeHistsV2, nil) + customBucketsHistSamplesV2 := enc.CustomBucketsHistogramSamples(customBucketsV2, nil) + decHistsV2, err := dec.HistogramSamples(histSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsV2, err := dec.HistogramSamples(customBucketsHistSamplesV2, nil) + require.NoError(t, err) + decHistsV2 = append(decHistsV2, decCustomBucketsV2...) + require.Equal(t, gaugeHistsV2, decHistsV2) + }) + + // V2 gauge float-histogram round-trip. + t.Run("V2 gauge float-histogram", func(t *testing.T) { + gaugeHistsV2 := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[0].H}, + {Ref: 42, T: 5678, ST: 1000, H: histograms[1].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, + } + gaugeFloatHistsV2 := make([]RefFloatHistogramSample, len(gaugeHistsV2)) + for i, h := range gaugeHistsV2 { + gaugeFloatHistsV2[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + ST: h.ST, + FH: h.H.ToFloat(nil), + } + } + floatHistSamplesV2, customBucketsFloatV2 := enc.FloatHistogramSamples(gaugeFloatHistsV2, nil) + customBucketsFloatHistSamplesV2 := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatV2, nil) + decFloatHistsV2, err := dec.FloatHistogramSamples(floatHistSamplesV2, nil) + require.NoError(t, err) + decCustomBucketsFloatV2, err := dec.FloatHistogramSamples(customBucketsFloatHistSamplesV2, nil) + require.NoError(t, err) + decFloatHistsV2 = append(decFloatHistsV2, decCustomBucketsFloatV2...) + require.Equal(t, gaugeFloatHistsV2, decFloatHistsV2) + }) + + for _, enableSTStorage := range []bool{false, true} { + t.Run(fmt.Sprintf("int-histogram empty slice stStorage=%v", enableSTStorage), func(t *testing.T) { + enc := Encoder{EnableSTStorage: enableSTStorage} + histBuf, customBuckets := enc.HistogramSamples(nil, nil) + require.Nil(t, customBuckets) + + decoded, err := dec.HistogramSamples(histBuf, nil) + require.NoError(t, err) + require.Empty(t, decoded) + }) + + t.Run(fmt.Sprintf("float-histogram empty slice stStorage=%v", enableSTStorage), func(t *testing.T) { + enc := Encoder{EnableSTStorage: enableSTStorage} + floatBuf, customBucketsFloat := enc.FloatHistogramSamples(nil, nil) + require.Nil(t, customBucketsFloat) + + decoded, err := dec.FloatHistogramSamples(floatBuf, nil) + require.NoError(t, err) + require.Empty(t, decoded) + }) + } + + // When all histograms are custom-bucket, HistogramSamples must return an + // empty buffer (buf.Reset path) and pass every sample through as custom. + t.Run("V2 int-histogram all custom bucket", func(t *testing.T) { + allCustom := []RefHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, H: histograms[2].H}, + {Ref: 67, T: 5678, ST: 1000, H: histograms[2].H}, + } + histBuf, customBuckets := enc.HistogramSamples(allCustom, nil) + require.Empty(t, histBuf, "regular histogram buffer must be empty when all samples are custom bucket") + require.Equal(t, allCustom, customBuckets) + + customBuf := enc.CustomBucketsHistogramSamples(customBuckets, nil) + decoded, err := dec.HistogramSamples(customBuf, nil) + require.NoError(t, err) + require.Equal(t, allCustom, decoded) + }) + + t.Run("V2 float-histogram all custom bucket", func(t *testing.T) { + allCustomFloat := []RefFloatHistogramSample{ + {Ref: 56, T: 1234, ST: 1000, FH: histograms[2].H.ToFloat(nil)}, + {Ref: 67, T: 5678, ST: 1000, FH: histograms[2].H.ToFloat(nil)}, + } + floatBuf, customBucketsFloat := enc.FloatHistogramSamples(allCustomFloat, nil) + require.Empty(t, floatBuf, "regular float histogram buffer must be empty when all samples are custom bucket") + require.Equal(t, allCustomFloat, customBucketsFloat) + + customFloatBuf := enc.CustomBucketsFloatHistogramSamples(customBucketsFloat, nil) + decoded, err := dec.FloatHistogramSamples(customFloatBuf, nil) + require.NoError(t, err) + require.Equal(t, allCustomFloat, decoded) + }) + + // When all histograms are custom-bucket, V1 HistogramSamples must return an + // empty buffer (buf.Reset path) and pass every sample through as custom. + t.Run("V1 int-histogram all custom bucket", func(t *testing.T) { + encV1 := Encoder{} + allCustom := []RefHistogramSample{ + {Ref: 56, T: 1234, H: histograms[2].H}, + {Ref: 67, T: 5678, H: histograms[2].H}, + } + histBuf, customBuckets := encV1.HistogramSamples(allCustom, nil) + require.Empty(t, histBuf, "regular histogram buffer must be empty when all samples are custom bucket") + require.Equal(t, allCustom, customBuckets) + + customBuf := encV1.CustomBucketsHistogramSamples(customBuckets, nil) + decoded, err := dec.HistogramSamples(customBuf, nil) + require.NoError(t, err) + require.Equal(t, allCustom, decoded) + }) + + t.Run("V1 float-histogram all custom bucket", func(t *testing.T) { + encV1 := Encoder{} + allCustomFloat := []RefFloatHistogramSample{ + {Ref: 56, T: 1234, FH: histograms[2].H.ToFloat(nil)}, + {Ref: 67, T: 5678, FH: histograms[2].H.ToFloat(nil)}, + } + floatBuf, customBucketsFloat := encV1.FloatHistogramSamples(allCustomFloat, nil) + require.Empty(t, floatBuf, "regular float histogram buffer must be empty when all samples are custom bucket") + require.Equal(t, allCustomFloat, customBucketsFloat) + + customFloatBuf := encV1.CustomBucketsFloatHistogramSamples(customBucketsFloat, nil) + decoded, err := dec.FloatHistogramSamples(customFloatBuf, nil) + require.NoError(t, err) + require.Equal(t, allCustomFloat, decoded) + }) + + // Backward compat: V1-encoded histograms decode with ST=0. + t.Run("V1 backward compat int-histogram ST=0", func(t *testing.T) { + encV1 := Encoder{} + v1HistSamples, v1CustomBucketsHists := encV1.HistogramSamples(histograms, nil) + v1CustomBucketsHistSamples := encV1.CustomBucketsHistogramSamples(v1CustomBucketsHists, nil) + decV1Hists, err := dec.HistogramSamples(v1HistSamples, nil) + require.NoError(t, err) + decV1CustomBuckets, err := dec.HistogramSamples(v1CustomBucketsHistSamples, nil) + require.NoError(t, err) + for _, h := range append(decV1Hists, decV1CustomBuckets...) { + require.Equal(t, int64(0), h.ST, "V1 histogram records must decode with ST=0") + } + }) + + // Backward compat: V1-encoded float histograms decode with ST=0. + t.Run("V1 backward compat float-histogram ST=0", func(t *testing.T) { + encV1 := Encoder{} + v1FloatHistSamples, v1CustomBucketsFloatHists := encV1.FloatHistogramSamples(floatHistograms, nil) + v1CustomBucketsFloatHistSamples := encV1.CustomBucketsFloatHistogramSamples(v1CustomBucketsFloatHists, nil) + decV1FloatHists, err := dec.FloatHistogramSamples(v1FloatHistSamples, nil) + require.NoError(t, err) + decV1CustomBucketsFloatHists, err := dec.FloatHistogramSamples(v1CustomBucketsFloatHistSamples, nil) + require.NoError(t, err) + for _, h := range append(decV1FloatHists, decV1CustomBucketsFloatHists...) { + require.Equal(t, int64(0), h.ST, "V1 float histogram records must decode with ST=0") + } + }) +} + +func TestRecord_MixedRegularAndCustomBucketHistogramPermutations(t *testing.T) { + dec := NewDecoder(labels.NewSymbolTable(), promslog.NewNopLogger()) + + regularA := &histogram.Histogram{ + Count: 5, + ZeroCount: 2, + ZeroThreshold: 0.001, + Sum: 18.4, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + } + regularB := &histogram.Histogram{ + Count: 11, + ZeroCount: 4, + ZeroThreshold: 0.001, + Sum: 35.5, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + NegativeSpans: []histogram.Span{ + {Offset: 0, Length: 1}, + {Offset: 1, Length: 2}, + }, + NegativeBuckets: []int64{1, 2, -1}, + } + custom := &histogram.Histogram{ + Count: 8, + ZeroThreshold: 0.001, + Sum: 42.0, + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 2}, + }, + PositiveBuckets: []int64{2, -1, 2, 0}, + CustomValues: []float64{0, 2, 4, 6, 8}, + } + + type stMode struct { + name string + sts []int64 + } + type tc struct { + name string + kinds []string + stModes []stMode + } + + testCases := []tc{ + { + name: "regular then custom", + kinds: []string{"regularA", "custom"}, + }, + { + name: "custom then regular", + kinds: []string{"custom", "regularA"}, + }, + { + name: "regular custom regular", + kinds: []string{"regularA", "custom", "regularB"}, + }, + { + name: "custom regular custom", + kinds: []string{"custom", "regularA", "custom"}, + }, + } + + stModes := []stMode{ + {name: "no ST", sts: []int64{0, 0, 0}}, + {name: "constant ST", sts: []int64{1000, 1000, 1000}}, + {name: "varying ST", sts: []int64{1000, 1234, 5678}}, + } + + buildIntSamples := func(kinds []string, sts []int64) []RefHistogramSample { + samples := make([]RefHistogramSample, 0, len(kinds)) + for i, kind := range kinds { + var h *histogram.Histogram + switch kind { + case "regularA": + h = regularA + case "regularB": + h = regularB + case "custom": + h = custom + default: + t.Fatalf("unknown histogram kind %q", kind) + } + + samples = append(samples, RefHistogramSample{ + Ref: chunks.HeadSeriesRef(100 + i*11), + T: int64(1000 + i*250), + ST: sts[i], + H: h, + }) + } + return samples + } + + toExpectedIntPartitions := func(samples []RefHistogramSample) ([]RefHistogramSample, []RefHistogramSample) { + var regularSamples []RefHistogramSample + var customSamples []RefHistogramSample + for _, sample := range samples { + if sample.H.UsesCustomBuckets() { + customSamples = append(customSamples, sample) + continue + } + regularSamples = append(regularSamples, sample) + } + return regularSamples, customSamples + } + + toFloatSamples := func(samples []RefHistogramSample) []RefFloatHistogramSample { + floatSamples := make([]RefFloatHistogramSample, 0, len(samples)) + for _, sample := range samples { + floatSamples = append(floatSamples, RefFloatHistogramSample{ + Ref: sample.Ref, + T: sample.T, + ST: sample.ST, + FH: sample.H.ToFloat(nil), + }) + } + return floatSamples + } + + toExpectedFloatPartitions := func(samples []RefFloatHistogramSample) ([]RefFloatHistogramSample, []RefFloatHistogramSample) { + var regularSamples []RefFloatHistogramSample + var customSamples []RefFloatHistogramSample + for _, sample := range samples { + if sample.FH.UsesCustomBuckets() { + customSamples = append(customSamples, sample) + continue + } + regularSamples = append(regularSamples, sample) + } + return regularSamples, customSamples + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, stMode := range stModes { + t.Run(stMode.name, func(t *testing.T) { + enc := Encoder{EnableSTStorage: true} + + intSamples := buildIntSamples(tc.kinds, stMode.sts[:len(tc.kinds)]) + wantRegularInt, wantCustomInt := toExpectedIntPartitions(intSamples) + + histBuf, customInt := enc.HistogramSamples(intSamples, nil) + customBuf := enc.CustomBucketsHistogramSamples(customInt, nil) + + gotRegularInt, err := dec.HistogramSamples(histBuf, nil) + require.NoError(t, err) + gotCustomInt, err := dec.HistogramSamples(customBuf, nil) + require.NoError(t, err) + + require.Equal(t, wantRegularInt, gotRegularInt) + require.Equal(t, wantCustomInt, gotCustomInt) + + floatSamples := toFloatSamples(intSamples) + wantRegularFloat, wantCustomFloat := toExpectedFloatPartitions(floatSamples) + + floatBuf, customFloat := enc.FloatHistogramSamples(floatSamples, nil) + customFloatBuf := enc.CustomBucketsFloatHistogramSamples(customFloat, nil) + + gotRegularFloat, err := dec.FloatHistogramSamples(floatBuf, nil) + require.NoError(t, err) + gotCustomFloat, err := dec.FloatHistogramSamples(customFloatBuf, nil) + require.NoError(t, err) + + require.Equal(t, wantRegularFloat, gotRegularFloat) + require.Equal(t, wantCustomFloat, gotCustomFloat) + }) + } + }) + } } func TestRecord_DecodeInvalidHistogramSchema(t *testing.T) { @@ -597,6 +1108,8 @@ func TestRecord_Type(t *testing.T) { }, }, } + // V1 histogram type recognition (requires EnableSTStorage off). + enc = Encoder{} hists, customBucketsHistograms := enc.HistogramSamples(histograms, nil) recordType = dec.Type(hists) require.Equal(t, HistogramSamples, recordType) @@ -604,6 +1117,31 @@ func TestRecord_Type(t *testing.T) { recordType = dec.Type(customBucketsHists) require.Equal(t, CustomBucketsHistogramSamples, recordType) + // V2 histogram type recognition. + enc = Encoder{EnableSTStorage: true} + hists, customBucketsHistograms = enc.HistogramSamples(histograms, nil) + recordType = dec.Type(hists) + require.Equal(t, HistogramSamplesV2, recordType) + customBucketsHists = enc.CustomBucketsHistogramSamples(customBucketsHistograms, nil) + recordType = dec.Type(customBucketsHists) + require.Equal(t, CustomBucketsHistogramSamplesV2, recordType) + + // V2 float-histogram type recognition. + floatHistograms := make([]RefFloatHistogramSample, len(histograms)) + for i, h := range histograms { + floatHistograms[i] = RefFloatHistogramSample{ + Ref: h.Ref, + T: h.T, + FH: h.H.ToFloat(nil), + } + } + floatHists, customBucketsFloatHistograms := enc.FloatHistogramSamples(floatHistograms, nil) + recordType = dec.Type(floatHists) + require.Equal(t, FloatHistogramSamplesV2, recordType) + customBucketsFloatHists := enc.CustomBucketsFloatHistogramSamples(customBucketsFloatHistograms, nil) + recordType = dec.Type(customBucketsFloatHists) + require.Equal(t, CustomBucketsFloatHistogramSamplesV2, recordType) + recordType = dec.Type(nil) require.Equal(t, Unknown, recordType) diff --git a/util/testrecord/record.go b/util/testrecord/record.go index e5071d42c8..dab721e8b8 100644 --- a/util/testrecord/record.go +++ b/util/testrecord/record.go @@ -17,6 +17,7 @@ import ( "math" "testing" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/record" ) @@ -81,6 +82,127 @@ func GenTestRefSamplesCase(t testing.TB, c RefSamplesCase) []record.RefSample { return ret } +// HistSTCase selects the start-time pattern for histogram test data generators. +type HistSTCase string + +const ( + HistNoST HistSTCase = "no-st" + HistConstST HistSTCase = "const-st" + HistPrevTST HistSTCase = "prevt-st" + HistVariableST HistSTCase = "var-st" +) + +// HistSTCases is the standard set of histogram ST cases for benchmarks. +var HistSTCases = []HistSTCase{HistNoST, HistConstST, HistPrevTST, HistVariableST} + +// applyHistST sets the ST field on histogram samples according to the given case. +func applyHistST(out []record.RefHistogramSample, stCase HistSTCase) { + switch stCase { + case HistNoST: + // Nothing to do. + case HistConstST: + for i := range out { + out[i].ST = 1709000000 + } + case HistPrevTST: + for i := range out { + if i == 0 { + continue + } + out[i].ST = out[i-1].T + } + case HistVariableST: + for i := range out { + out[i].ST = highVarianceInt(i+1) / 1024 + } + } +} + +// GenExpHistograms generates n standard exponential histograms (schema=1) +// with incrementing refs, same timestamp, and realistic bucket distributions. +// The stCase parameter controls how the ST field is populated. +func GenExpHistograms(n int, stCase HistSTCase) []record.RefHistogramSample { + out := make([]record.RefHistogramSample, n) + for i := range out { + out[i] = record.RefHistogramSample{ + Ref: chunks.HeadSeriesRef(i), + T: 1709000000 + int64(i)*15, + H: &histogram.Histogram{ + Count: uint64(10 + i%100), + ZeroCount: uint64(1 + i%5), + ZeroThreshold: 0.001, + Sum: float64(100+i) * 1.5, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 4}, + {Offset: 2, Length: 3}, + }, + PositiveBuckets: []int64{1, 2, -1, 0, 3, -2, 1}, + NegativeSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + NegativeBuckets: []int64{1, 1, -1, 0}, + }, + } + } + applyHistST(out, stCase) + return out +} + +// GenCustomBucketHistograms generates n custom-bucket (NHCB) histograms (schema=-53) +// with incrementing refs. The stCase parameter controls how the ST field is populated. +func GenCustomBucketHistograms(n int, stCase HistSTCase) []record.RefHistogramSample { + out := make([]record.RefHistogramSample, n) + for i := range out { + out[i] = record.RefHistogramSample{ + Ref: chunks.HeadSeriesRef(i), + T: 1709000000 + int64(i)*15, + H: &histogram.Histogram{ + Count: uint64(10 + i%100), + Sum: float64(100+i) * 1.5, + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 8}, + }, + PositiveBuckets: []int64{5, -2, 3, -1, 4, 0, -3, 2}, + CustomValues: []float64{0.001, 0.01, 0.1, 1, 10, 100, 1000}, + }, + } + } + applyHistST(out, stCase) + return out +} + +// GenFloatHistograms converts int histograms to float histograms, preserving ST. +func GenFloatHistograms(src []record.RefHistogramSample) []record.RefFloatHistogramSample { + out := make([]record.RefFloatHistogramSample, len(src)) + for i, h := range src { + out[i] = record.RefFloatHistogramSample{ + Ref: h.Ref, + ST: h.ST, + T: h.T, + FH: h.H.ToFloat(nil), + } + } + return out +} + +// HistDataCase pairs a name with a histogram generator for benchmark tables. +type HistDataCase struct { + Name string + Gen func(n int, stCase HistSTCase) []record.RefHistogramSample +} + +// HistDataCases is the standard set of histogram data cases for benchmarks. +var HistDataCases = []HistDataCase{ + {"exp", GenExpHistograms}, + {"nhcb", GenCustomBucketHistograms}, +} + +// HistCounts is the standard set of histogram counts for benchmarks. +var HistCounts = []int{10, 100, 1000} + func highVarianceInt(i int) int64 { if i%2 == 0 { return math.MinInt32