Implement ZTEST: One-Sample t-Test Statistic with validation tests

- Added Ztest class to compute the one-sample t-statistic using sample standard deviation with Bessel correction.
- Implemented validation tests for Ztest to ensure accuracy against manual calculations and PineScript.
- Updated documentation for Ztest, detailing its mathematical foundation, performance profile, and common pitfalls.
- Adjusted NDepend badges to reflect changes in code metrics after implementation.
- Updated missing indicators report to reflect the completion of statistical indicators, including ZTEST.
This commit is contained in:
Miha Kralj
2026-02-16 16:54:36 -08:00
parent 09ffd31a40
commit b3a64f18fa
73 changed files with 13041 additions and 88 deletions
@@ -0,0 +1,116 @@
using TradingPlatform.BusinessLayer;
using QuanTAlib;
namespace QuanTAlib.Tests;
public sealed class ZscoreIndicatorTests
{
[Fact]
public void ZscoreIndicator_Constructor_SetsDefaults()
{
var indicator = new ZscoreIndicator();
Assert.Equal(14, indicator.Period);
Assert.True(indicator.ShowColdValues);
Assert.Contains("ZSCORE", indicator.Name, StringComparison.Ordinal);
Assert.True(indicator.SeparateWindow);
Assert.True(indicator.OnBackGround);
Assert.Equal(SourceType.Close, indicator.Source);
}
[Fact]
public void ZscoreIndicator_MinHistoryDepths_EqualsZero()
{
var indicator = new ZscoreIndicator { Period = 14 };
Assert.Equal(0, ZscoreIndicator.MinHistoryDepths);
IWatchlistIndicator watchlistIndicator = indicator;
Assert.Equal(0, watchlistIndicator.MinHistoryDepths);
}
[Fact]
public void ZscoreIndicator_Initialize_CreatesInternalZscore()
{
var indicator = new ZscoreIndicator { Period = 10 };
// Initialize should not throw
indicator.Initialize();
// After init, line series should exist
Assert.Single(indicator.LinesSeries);
Assert.Equal("Z-Score", indicator.LinesSeries[0].Name);
}
[Fact]
public void ZscoreIndicator_ProcessUpdate_HistoricalBar_ComputesValue()
{
var indicator = new ZscoreIndicator { Period = 5 };
indicator.Initialize();
var now = DateTime.UtcNow;
for (int i = 0; i < 20; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), 100 + i, 110 + i, 90 + i, 105 + i);
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
}
double zscore = indicator.LinesSeries[0].GetValue(0);
Assert.True(double.IsFinite(zscore));
}
[Fact]
public void ZscoreIndicator_DifferentSourceTypes()
{
var indicator = new ZscoreIndicator { Period = 5, Source = SourceType.Open };
indicator.Initialize();
var now = DateTime.UtcNow;
for (int i = 0; i < 10; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), 100 + i, 110 + i, 90 + i, 105 + i);
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
}
double zscore = indicator.LinesSeries[0].GetValue(0);
Assert.True(double.IsFinite(zscore));
}
[Fact]
public void ZscoreIndicator_ShortName_IncludesPeriod()
{
var indicator = new ZscoreIndicator { Period = 20 };
Assert.Equal("ZSCORE(20)", indicator.ShortName);
}
[Fact]
public void ZscoreIndicator_NewBar_UpdatesValue()
{
var indicator = new ZscoreIndicator { Period = 5 };
indicator.Initialize();
var now = DateTime.UtcNow;
// Add enough bars to warm up
for (int i = 0; i < 10; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), 100 + i, 110 + i, 90 + i, 105 + i);
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
}
_ = indicator.LinesSeries[0].GetValue(0);
// Add a new bar with a very different value
indicator.HistoricalData.AddBar(now.AddMinutes(10), 200, 210, 190, 205);
var newArgs = new UpdateArgs(UpdateReason.NewBar);
indicator.ProcessUpdate(newArgs);
double valueAfter = indicator.LinesSeries[0].GetValue(0);
// Value should change after adding a significantly different bar
Assert.True(double.IsFinite(valueAfter));
}
}
+60
View File
@@ -0,0 +1,60 @@
using System.Drawing;
using System.Runtime.CompilerServices;
using TradingPlatform.BusinessLayer;
namespace QuanTAlib;
[SkipLocalsInit]
public sealed class ZscoreIndicator : Indicator, IWatchlistIndicator
{
[InputParameter("Period", sortIndex: 1, 2, 2000, 1, 0)]
public int Period { get; set; } = 14;
[IndicatorExtensions.DataSourceInput]
public SourceType Source { get; set; } = SourceType.Close;
[InputParameter("Show cold values", sortIndex: 21)]
public bool ShowColdValues { get; set; } = true;
private Zscore _zscore = null!;
private readonly LineSeries _series;
private Func<IHistoryItem, double> _priceSelector = null!;
public static int MinHistoryDepths => 0;
int IWatchlistIndicator.MinHistoryDepths => MinHistoryDepths;
public override string ShortName => $"ZSCORE({Period})";
public override string SourceCodeLink => "https://github.com/mihakralj/QuanTAlib/blob/main/lib/statistics/zscore/Zscore.Quantower.cs";
public ZscoreIndicator()
{
OnBackGround = true;
SeparateWindow = true;
Name = "ZSCORE - Z-Score (Population Standard Score)";
Description = "Measures how many population standard deviations a value is from the mean";
_series = new LineSeries(name: "Z-Score", color: IndicatorExtensions.Statistics, width: 2, style: LineStyle.Solid);
AddLineSeries(_series);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnInit()
{
_zscore = new Zscore(Period);
_priceSelector = Source.GetPriceSelector();
base.OnInit();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnUpdate(UpdateArgs args)
{
var item = this.HistoricalData[this.Count - 1, SeekOriginHistory.Begin];
double value = _priceSelector(item);
var time = this.HistoricalData.Time();
var input = new TValue(time, value);
TValue result = _zscore.Update(input, args.IsNewBar());
_series.SetValue(result.Value, _zscore.IsHot, ShowColdValues);
}
}
+441
View File
@@ -0,0 +1,441 @@
namespace QuanTAlib.Tests;
public class ZscoreTests
{
// A) Constructor validation
[Fact]
public void Constructor_DefaultPeriod_Is14()
{
var z = new Zscore();
Assert.Equal("Zscore(14)", z.Name);
}
[Fact]
public void Constructor_PeriodLessThan2_Throws()
{
var ex = Assert.Throws<ArgumentException>(() => new Zscore(1));
Assert.Equal("period", ex.ParamName);
}
[Fact]
public void Constructor_PeriodEquals2_Works()
{
var z = new Zscore(2);
Assert.Equal("Zscore(2)", z.Name);
}
// B) Basic calculation — constant series => z = 0
[Fact]
public void Update_ConstantSeries_ReturnsZero()
{
var z = new Zscore(5);
for (int i = 0; i < 10; i++)
{
var tv = z.Update(new TValue(DateTime.UtcNow, 100.0));
Assert.Equal(0.0, tv.Value);
}
}
// B) Known values: {1, 2, 3, 4, 5} => z(5) = (5 - 3) / sqrt(2) ≈ 1.4142
[Fact]
public void Update_KnownSequence_CorrectZScore()
{
var z = new Zscore(5);
for (int i = 1; i <= 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, i));
}
// mean = 3, pop variance = ((1-3)²+(2-3)²+(3-3)²+(4-3)²+(5-3)²)/5 = 10/5 = 2
// sigma = sqrt(2) ≈ 1.4142
// z(5) = (5 - 3) / sqrt(2) = 2/sqrt(2) = sqrt(2) ≈ 1.4142
double expected = Math.Sqrt(2.0);
Assert.Equal(expected, z.Last.Value, 1e-9);
}
// B) Check z-score of mean value = 0
[Fact]
public void Update_MeanValue_ReturnsZero()
{
var z = new Zscore(3);
z.Update(new TValue(DateTime.UtcNow, 10.0));
z.Update(new TValue(DateTime.UtcNow, 20.0));
var result = z.Update(new TValue(DateTime.UtcNow, 15.0));
// mean of {10, 20, 15} = 15, so z(15) = 0
Assert.Equal(0.0, result.Value, 1e-9);
}
// B) Negative z-score for below-mean value
[Fact]
public void Update_BelowMean_ReturnsNegative()
{
var z = new Zscore(5);
for (int i = 1; i <= 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, i));
}
// Replace last with value 1 (below mean=3)
var result = z.Update(new TValue(DateTime.UtcNow, 1.0));
Assert.True(result.Value < 0);
}
// C) State + bar correction
[Fact]
public void Update_IsNewTrue_AdvancesState()
{
var z = new Zscore(5);
z.Update(new TValue(DateTime.UtcNow, 10.0));
z.Update(new TValue(DateTime.UtcNow, 20.0));
double v1 = z.Last.Value;
z.Update(new TValue(DateTime.UtcNow, 30.0));
double v2 = z.Last.Value;
Assert.NotEqual(v1, v2);
}
[Fact]
public void Update_IsNewFalse_Rewrites()
{
var z = new Zscore(5);
for (int i = 0; i < 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
}
double before = z.Last.Value;
z.Update(new TValue(DateTime.UtcNow, 999.0), false);
double after = z.Last.Value;
Assert.NotEqual(before, after);
}
[Fact]
public void Update_IterativeCorrections_Restore()
{
var z = new Zscore(5);
for (int i = 0; i < 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
}
double snapshot = z.Last.Value;
// Correct multiple times with isNew=false
z.Update(new TValue(DateTime.UtcNow, 50.0), false);
z.Update(new TValue(DateTime.UtcNow, 100.0), false);
z.Update(new TValue(DateTime.UtcNow, 10.0 + 4), false); // restore original
Assert.Equal(snapshot, z.Last.Value, 1e-9);
}
[Fact]
public void Reset_ClearsState()
{
var z = new Zscore(5);
for (int i = 0; i < 10; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
}
Assert.True(z.IsHot);
z.Reset();
Assert.False(z.IsHot);
Assert.Equal(default, z.Last);
}
// D) Warmup/convergence
[Fact]
public void IsHot_FlipsWhenBufferFull()
{
var z = new Zscore(5);
for (int i = 0; i < 4; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
Assert.False(z.IsHot);
}
z.Update(new TValue(DateTime.UtcNow, 14.0));
Assert.True(z.IsHot);
}
[Fact]
public void WarmupPeriod_EqualsPeriod()
{
var z = new Zscore(10);
Assert.Equal(10, z.WarmupPeriod);
}
// E) Robustness — NaN/Infinity
[Fact]
public void Update_NaN_UsesLastValid()
{
var z = new Zscore(5);
for (int i = 0; i < 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
}
_ = z.Last.Value;
z.Update(new TValue(DateTime.UtcNow, double.NaN));
// NaN substituted with last valid — result may differ but should be finite
Assert.True(double.IsFinite(z.Last.Value));
}
[Fact]
public void Update_Infinity_UsesLastValid()
{
var z = new Zscore(5);
for (int i = 0; i < 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
}
z.Update(new TValue(DateTime.UtcNow, double.PositiveInfinity));
Assert.True(double.IsFinite(z.Last.Value));
}
[Fact]
public void Update_BatchNaN_AllFinite()
{
var z = new Zscore(5);
for (int i = 0; i < 5; i++)
{
z.Update(new TValue(DateTime.UtcNow, 10.0 + i));
}
for (int i = 0; i < 10; i++)
{
z.Update(new TValue(DateTime.UtcNow, double.NaN));
Assert.True(double.IsFinite(z.Last.Value));
}
}
// F) Consistency — batch == streaming == span == eventing
[Fact]
public void Consistency_AllModesMatch()
{
int period = 10;
int count = 50;
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 42);
var source = new TSeries(count);
for (int i = 0; i < count; i++)
{
TBar bar = rng.Next();
source.Add(new TValue(bar.Time, bar.Close), true);
}
// 1. Batch via TSeries
TSeries batchResult = Zscore.Batch(source, period);
// 2. Streaming
var streaming = new Zscore(period);
var streamResult = new List<double>(count);
for (int i = 0; i < source.Count; i++)
{
streaming.Update(source[i]);
streamResult.Add(streaming.Last.Value);
}
// 3. Span
Span<double> spanOutput = new double[count];
Zscore.Batch(source.Values, spanOutput, period);
// 4. Eventing
var publisher = new TSeries(count);
var eventIndicator = new Zscore(publisher, period);
var eventResult = new List<double>(count);
eventIndicator.Pub += (object? _, in TValueEventArgs _) => eventResult.Add(eventIndicator.Last.Value);
for (int i = 0; i < source.Count; i++)
{
publisher.Add(source[i], true);
}
for (int i = 0; i < count; i++)
{
Assert.Equal(batchResult[i].Value, streamResult[i], 1e-9);
Assert.Equal(batchResult[i].Value, spanOutput[i], 1e-8); // FP addition order differs between ring scan paths
Assert.Equal(batchResult[i].Value, eventResult[i], 1e-9);
}
}
// G) Span API tests
[Fact]
public void Batch_Span_EmptySource_Throws()
{
var ex = Assert.Throws<ArgumentException>(() =>
Zscore.Batch(ReadOnlySpan<double>.Empty, Span<double>.Empty, 5));
Assert.Equal("source", ex.ParamName);
}
[Fact]
public void Batch_Span_OutputTooShort_Throws()
{
double[] src = [1, 2, 3];
double[] output = new double[2];
var ex = Assert.Throws<ArgumentException>(() =>
Zscore.Batch(src, output, 2));
Assert.Equal("output", ex.ParamName);
}
[Fact]
public void Batch_Span_PeriodTooSmall_Throws()
{
double[] src = [1, 2, 3];
double[] output = new double[3];
var ex = Assert.Throws<ArgumentException>(() =>
Zscore.Batch(src, output, 1));
Assert.Equal("period", ex.ParamName);
}
[Fact]
public void Batch_Span_MatchesTSeries()
{
int period = 5;
int count = 30;
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 99);
var source = new TSeries(count);
for (int i = 0; i < count; i++)
{
TBar bar = rng.Next();
source.Add(new TValue(bar.Time, bar.Close), true);
}
TSeries batchResult = Zscore.Batch(source, period);
Span<double> spanOutput = new double[count];
Zscore.Batch(source.Values, spanOutput, period);
for (int i = 0; i < count; i++)
{
Assert.Equal(batchResult[i].Value, spanOutput[i], 1e-8); // FP addition order differs between ring scan paths
}
}
[Fact]
public void Batch_Span_HandlesNaN()
{
ReadOnlySpan<double> src = stackalloc double[] { 1, 2, double.NaN, 4, 5 };
Span<double> output = stackalloc double[5];
Zscore.Batch(src, output, 3);
for (int i = 0; i < 5; i++)
{
Assert.True(double.IsFinite(output[i]));
}
}
[Fact]
public void Batch_Span_LargeData_NoStackOverflow()
{
int size = 1000;
double[] src = new double[size];
double[] output = new double[size];
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 77);
for (int i = 0; i < size; i++)
{
src[i] = rng.Next().Close;
}
Zscore.Batch(src, output, 300); // above stackalloc threshold
for (int i = 0; i < size; i++)
{
Assert.True(double.IsFinite(output[i]));
}
}
// H) Chainability
[Fact]
public void Pub_Fires_OnUpdate()
{
var z = new Zscore(5);
int fireCount = 0;
z.Pub += (object? _, in TValueEventArgs _) => fireCount++;
z.Update(new TValue(DateTime.UtcNow, 10.0));
Assert.Equal(1, fireCount);
}
[Fact]
public void EventChaining_Works()
{
var publisher = new TSeries(10);
var z = new Zscore(publisher, 5);
publisher.Add(new TValue(DateTime.UtcNow, 10.0), true);
Assert.True(double.IsFinite(z.Last.Value));
}
// Additional: population stddev vs sample stddev distinction
[Fact]
public void Update_UsesPopulationStdDev()
{
// For data {2, 4, 4, 4, 5, 5, 7, 9}, population σ = 2
// Population mean = 5, pop variance = 4, σ = 2
// z(9) = (9 - 5) / 2 = 2.0
var z = new Zscore(8);
double[] data = [2, 4, 4, 4, 5, 5, 7, 9];
foreach (double d in data)
{
z.Update(new TValue(DateTime.UtcNow, d));
}
Assert.Equal(2.0, z.Last.Value, 1e-9);
}
// Symmetry: z-score of min value should be negative of z-score of max value for symmetric data
[Fact]
public void Update_SymmetricData_SymmetricZScores()
{
// {1, 2, 3, 4, 5} => z(1) = -sqrt(2), z(5) = +sqrt(2)
var z1 = new Zscore(5);
for (int i = 1; i <= 5; i++)
{
z1.Update(new TValue(DateTime.UtcNow, i));
}
double zMax = z1.Last.Value; // z(5)
var z2 = new Zscore(5);
for (int i = 5; i >= 1; i--)
{
z2.Update(new TValue(DateTime.UtcNow, i));
}
double zMin = z2.Last.Value; // z(1) with reversed input
Assert.Equal(zMax, -zMin, 1e-9);
}
// Calculate tuple method
[Fact]
public void Calculate_ReturnsTupleWithResults()
{
int count = 20;
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 55);
var source = new TSeries(count);
for (int i = 0; i < count; i++)
{
source.Add(new TValue(rng.Next().Time, rng.Next().Close), true);
}
var (results, indicator) = Zscore.Calculate(source, 5);
Assert.Equal(source.Count, results.Count);
Assert.True(indicator.IsHot);
}
// Prime method
[Fact]
public void Prime_WarmsUpIndicator()
{
var z = new Zscore(5);
double[] data = [10, 20, 30, 40, 50];
z.Prime(data);
Assert.True(z.IsHot);
}
}
@@ -0,0 +1,122 @@
namespace QuanTAlib.Validation;
/// <summary>
/// Validation tests for ZSCORE indicator.
/// No direct TA-Lib/Tulip/Skender/Ooples equivalent exists for population z-score.
/// Validates against manual computation and mathematical properties.
/// </summary>
public sealed class ZscoreValidationTests
{
[Fact]
public void Zscore_ManualComputation_MatchesPineScript()
{
// PineScript formula: z = (x - mean) / sqrt(popVariance)
// Data: {10, 20, 30, 40, 50}, period=5
// mean = 30, popVar = ((10-30)²+(20-30)²+(30-30)²+(40-30)²+(50-30)²)/5 = 1000/5 = 200
// sigma = sqrt(200) ≈ 14.1421
// z(50) = (50-30)/sqrt(200) = 20/14.1421 ≈ 1.4142
var z = new Zscore(5);
double[] data = [10, 20, 30, 40, 50];
foreach (double d in data)
{
z.Update(new TValue(DateTime.UtcNow, d));
}
double expected = 20.0 / Math.Sqrt(200.0);
Assert.Equal(expected, z.Last.Value, 1e-9);
}
[Fact]
public void Zscore_GBMData_BoundedRange()
{
// For GBM-generated data, z-scores should typically be within [-4, 4]
int period = 20;
var z = new Zscore(period);
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 42);
for (int i = 0; i < 200; i++)
{
TBar bar = rng.Next();
z.Update(new TValue(bar.Time, bar.Close));
if (z.IsHot)
{
Assert.True(z.Last.Value > -10.0 && z.Last.Value < 10.0,
$"Z-score {z.Last.Value} outside expected range at i={i}");
}
}
}
[Fact]
public void Zscore_ScalingInvariance_HoldsForLinearTransform()
{
// z(a*x + b) should equal z(x) for constant a > 0, any b
int period = 10;
var z1 = new Zscore(period);
var z2 = new Zscore(period);
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 88);
for (int i = 0; i < 30; i++)
{
double val = rng.Next().Close;
z1.Update(new TValue(DateTime.UtcNow, val));
z2.Update(new TValue(DateTime.UtcNow, val * 3.0 + 100.0)); // linear transform
if (z1.IsHot && z2.IsHot)
{
Assert.Equal(z1.Last.Value, z2.Last.Value, 1e-8); // FP accumulation drift with scaled values
}
}
}
[Fact]
public void Zscore_MeanIsZero_ForWindowMeanValue()
{
// If the current value equals the window mean, z-score = 0
var z = new Zscore(5);
double[] data = [10, 20, 30, 40, 50];
foreach (double d in data)
{
z.Update(new TValue(DateTime.UtcNow, d));
}
// Now add 30 (== current mean)
_ = z.Update(new TValue(DateTime.UtcNow, 30.0)); // window: {20,30,40,50,30}, mean=34
// Not exactly 0 since window shifts, but demonstrates the property
// Instead test with window where current val == mean
var z2 = new Zscore(3);
z2.Update(new TValue(DateTime.UtcNow, 10.0));
z2.Update(new TValue(DateTime.UtcNow, 20.0));
var r = z2.Update(new TValue(DateTime.UtcNow, 15.0)); // mean = 15, z(15) = 0
Assert.Equal(0.0, r.Value, 1e-9);
}
[Fact]
public void Zscore_MatchesStandardize_WithPopulationCorrection()
{
// ZSCORE uses population stddev, Standardize uses sample stddev
// zscore = value_offset / pop_sigma
// standardize = value_offset / sample_sigma
// sample_sigma = pop_sigma * sqrt(n/(n-1))
// So: zscore = standardize * sqrt(n/(n-1))
int period = 10;
var zs = new Zscore(period);
var st = new Standardize(period);
var rng = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 99);
for (int i = 0; i < 20; i++)
{
double val = rng.Next().Close;
var tv = new TValue(DateTime.UtcNow, val);
zs.Update(tv);
st.Update(tv);
if (zs.IsHot && st.IsHot)
{
// zscore = standardize * sqrt(n / (n-1))
double correction = Math.Sqrt((double)period / (period - 1));
Assert.Equal(st.Last.Value * correction, zs.Last.Value, 1e-6);
}
}
}
}
+295
View File
@@ -0,0 +1,295 @@
// ZSCORE: Z-Score (Population Standard Score)
// Calculates z = (x - μ) / σ using population standard deviation (N denominator)
// Formula: z = (x - mean) / sqrt(Σ(xi - mean)² / N)
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace QuanTAlib;
/// <summary>
/// ZSCORE: Z-Score — measures how many population standard deviations a value
/// lies from the rolling mean over a lookback window.
/// </summary>
/// <remarks>
/// Key properties:
/// - Uses population standard deviation (N denominator, no Bessel correction)
/// - Output is unbounded (typically -3 to +3 for normally distributed data)
/// - When σ = 0 (constant data), returns 0.0
/// - Period must be >= 2
/// </remarks>
/// <seealso href="zscore.pine">Reference Pine Script implementation</seealso>
[SkipLocalsInit]
public sealed class Zscore : AbstractBase
{
private readonly int _period;
private readonly RingBuffer _buffer;
private readonly TValuePublishedHandler _handler;
private double _lastValidValue;
[StructLayout(LayoutKind.Auto)]
private record struct State(double LastValidZScore, double LastValidValue);
private State _s, _ps;
public override bool IsHot => _buffer.Count >= _period;
/// <param name="period">Lookback period (default 14, must be >= 2)</param>
public Zscore(int period = 14)
{
if (period < 2)
{
throw new ArgumentException("Period must be >= 2 for standard deviation calculation.", nameof(period));
}
_period = period;
_buffer = new RingBuffer(period);
Name = $"Zscore({period})";
WarmupPeriod = period;
_s = new State(0.0, 0.0);
_ps = _s;
_handler = Handle;
}
/// <param name="source">Source indicator for event-based chaining</param>
/// <param name="period">Lookback period (default 14)</param>
public Zscore(ITValuePublisher source, int period = 14) : this(period)
{
source.Pub += _handler;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override TValue Update(TValue input, bool isNew = true)
{
if (isNew)
{
_ps = _s;
}
else
{
_s = _ps;
_lastValidValue = _s.LastValidValue;
}
double value = input.Value;
if (!double.IsFinite(value))
{
value = _lastValidValue;
}
else
{
_lastValidValue = value;
}
_buffer.Add(value, isNew);
double result;
ReadOnlySpan<double> data = _buffer.GetSpan();
int n = data.Length;
if (n < 2)
{
result = 0.0;
}
else
{
double sum = 0.0;
double sumSq = 0.0;
for (int i = 0; i < n; i++)
{
double v = data[i];
sum += v;
sumSq += v * v;
}
double mean = sum / n;
// Population variance: E[X²] - (E[X])²
double popVariance = (sumSq / n) - (mean * mean);
if (popVariance < 0.0)
{
popVariance = 0.0;
}
double stdDev = Math.Sqrt(popVariance);
if (stdDev > 1e-10)
{
result = (value - mean) / stdDev;
}
else
{
result = 0.0;
}
}
_s = new State(result, _lastValidValue);
Last = new TValue(input.Time, result);
PubEvent(Last, isNew);
return Last;
}
public override TSeries Update(TSeries source)
{
var result = new TSeries(source.Count);
ReadOnlySpan<double> values = source.Values;
ReadOnlySpan<long> times = source.Times;
for (int i = 0; i < source.Count; i++)
{
var tv = Update(new TValue(new DateTime(times[i], DateTimeKind.Utc), values[i]), true);
result.Add(tv, true);
}
return result;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void Handle(object? sender, in TValueEventArgs args) => Update(args.Value, args.IsNew);
public override void Reset()
{
_buffer.Clear();
_lastValidValue = 0;
_s = new State(0.0, 0.0);
_ps = _s;
Last = default;
}
public override void Prime(ReadOnlySpan<double> source, TimeSpan? step = null)
{
TimeSpan interval = step ?? TimeSpan.FromSeconds(1);
DateTime time = DateTime.UtcNow - (interval * source.Length);
for (int i = 0; i < source.Length; i++)
{
Update(new TValue(time, source[i]), true);
time += interval;
}
}
public static TSeries Batch(TSeries source, int period = 14)
{
var indicator = new Zscore(period);
return indicator.Update(source);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Batch(ReadOnlySpan<double> source, Span<double> output, int period = 14)
{
if (source.Length == 0)
{
throw new ArgumentException("Source span must not be empty.", nameof(source));
}
if (output.Length < source.Length)
{
throw new ArgumentException("Output span must be at least as long as source.", nameof(output));
}
if (period < 2)
{
throw new ArgumentException("Period must be >= 2.", nameof(period));
}
const int StackallocThreshold = 256;
double[]? rented = null;
int ringSize = period;
scoped Span<double> ring;
if (ringSize <= StackallocThreshold)
{
ring = stackalloc double[ringSize];
}
else
{
rented = ArrayPool<double>.Shared.Rent(ringSize);
ring = rented.AsSpan(0, ringSize);
}
try
{
int head = 0;
int count = 0;
double lastValid = 0.0;
for (int i = 0; i < source.Length; i++)
{
double val = source[i];
if (!double.IsFinite(val))
{
val = lastValid;
}
else
{
lastValid = val;
}
if (count < ringSize)
{
ring[count] = val;
count++;
}
else
{
ring[head] = val;
}
head = (head + 1) % ringSize;
if (count < 2)
{
output[i] = 0.0;
continue;
}
double sum = 0.0;
double sumSq = 0.0;
int n = count;
for (int j = 0; j < n; j++)
{
double v = ring[j];
sum += v;
sumSq += v * v;
}
double mean = sum / n;
double popVariance = (sumSq / n) - (mean * mean);
if (popVariance < 0.0)
{
popVariance = 0.0;
}
double stdDev = Math.Sqrt(popVariance);
if (stdDev > 1e-10)
{
output[i] = (val - mean) / stdDev;
}
else
{
output[i] = 0.0;
}
}
}
finally
{
if (rented != null)
{
ArrayPool<double>.Shared.Return(rented);
}
}
}
public static (TSeries Results, Zscore Indicator) Calculate(TSeries source, int period = 14)
{
var indicator = new Zscore(period);
TSeries results = indicator.Update(source);
return (results, indicator);
}
}
+128
View File
@@ -0,0 +1,128 @@
# ZSCORE: Z-Score (Population Standard Score)
> "How far from normal is this?" — Every risk manager, every day.
## Introduction
The Z-Score measures how many population standard deviations a value lies from the rolling mean over a lookback window. Unlike the related Standardize indicator (which uses sample standard deviation with Bessel's correction), ZSCORE uses population standard deviation, matching the PineScript `ta.zscore` convention. Output is unbounded, typically ranging from -3 to +3 for normally distributed data. A z-score of 0 means the value equals the window mean; ±2 flags statistical outliers at the 95% level.
## Historical Context
The z-score originates from Karl Pearson's work in the 1890s on the theory of statistics. It transforms any distribution into units of standard deviation, making cross-series comparison possible. In trading, z-scores power mean-reversion strategies (enter when |z| > 2, exit when |z| < 0.5), pairs trading (z-score of spread), and anomaly detection. The population variant (N denominator) is standard in PineScript and most trading platforms because the rolling window IS the population of interest — not a sample from a larger population.
## Architecture and Physics
### 1. Core Formula
$$z = \frac{x - \mu}{\sigma}$$
where:
- $\mu = \frac{1}{N} \sum_{i=1}^{N} x_i$ (population mean over window)
- $\sigma = \sqrt{\frac{1}{N} \sum_{i=1}^{N} (x_i - \mu)^2}$ (population standard deviation)
### 2. Computational Form
Using the identity $\text{Var}(X) = E[X^2] - (E[X])^2$:
$$\sigma = \sqrt{\frac{\sum x_i^2}{N} - \left(\frac{\sum x_i}{N}\right)^2}$$
This avoids a two-pass algorithm. One pass computes both $\sum x_i$ and $\sum x_i^2$.
### 3. Edge Cases
| Condition | Result |
|-----------|--------|
| $N < 2$ | 0.0 |
| $\sigma < 10^{-10}$ | 0.0 (constant data) |
| Input is NaN/Infinity | Substitute last valid value |
| Negative variance (floating-point) | Clamp to 0.0 |
### 4. Population vs Sample
| Variant | Denominator | Use Case |
|---------|-------------|----------|
| ZSCORE (this) | $N$ | Rolling window IS the population |
| Standardize | $N - 1$ | Window is sample from larger population |
Relationship: $z_{\text{pop}} = z_{\text{sample}} \cdot \sqrt{\frac{N}{N-1}}$
### 5. State Management
Uses `RingBuffer` for the sliding window. State rollback via `record struct State` with `_s`/`_ps` pattern for bar correction support.
## Mathematical Foundation
### Z-Score Derivation
Given a window of $N$ values $\{x_1, x_2, \ldots, x_N\}$:
$$\mu = \frac{1}{N} \sum_{i=1}^{N} x_i$$
$$\sigma^2 = \frac{1}{N} \sum_{i=1}^{N} (x_i - \mu)^2 = \frac{1}{N} \sum_{i=1}^{N} x_i^2 - \mu^2$$
$$z = \frac{x_N - \mu}{\sigma}$$
### Scale Invariance
For any linear transform $y = ax + b$ where $a > 0$:
$$z(y) = \frac{(ax + b) - (a\mu + b)}{a\sigma} = \frac{x - \mu}{\sigma} = z(x)$$
Z-scores are invariant under positive linear transformations. This property makes them ideal for comparing series measured in different units.
## Performance Profile
### Operation Count (per Update)
| Operation | Count |
|-----------|-------|
| Additions | $N$ (sum scan) |
| Multiplications | $N$ (sumSq scan) |
| Division | 3 |
| Square root | 1 |
| Comparison | 2 |
### Complexity
| Method | Time | Space |
|--------|------|-------|
| `Update` | $O(N)$ | $O(1)$ auxiliary |
| `Batch(Span)` | $O(N \cdot P)$ | stackalloc or ArrayPool |
### Quality Metrics
| Metric | Score |
|--------|-------|
| Accuracy | 9/10 |
| Numerical stability | 8/10 |
| Memory efficiency | 9/10 |
| SIMD potential | Limited (sequential dependency on current value) |
## Validation
| Library | Status | Notes |
|---------|--------|-------|
| Manual | Verified | Known-value tests match hand computation |
| Standardize | Cross-validated | $z_{\text{pop}} = z_{\text{sample}} \cdot \sqrt{N/(N-1)}$ holds |
| PineScript | Formula match | Population stddev, same edge-case handling |
## Common Pitfalls
1. **Population vs sample confusion.** ZSCORE uses N denominator. Standardize uses N-1. The difference matters for small windows: at period=5, the ratio is $\sqrt{5/4} = 1.118$, an 11.8% discrepancy.
2. **Assuming normality.** Z-scores measure distance in sigma units but don't guarantee the underlying distribution is normal. Fat-tailed financial returns make |z| > 3 more common than the 0.3% a normal distribution predicts.
3. **Constant data edge case.** When all values in the window are identical, $\sigma = 0$ and division is undefined. Implementation returns 0.0.
4. **Floating-point variance.** The formula $E[X^2] - (E[X])^2$ can produce tiny negative values due to floating-point arithmetic. Clamped to zero before taking square root.
5. **Warmup period.** Requires at least 2 data points for meaningful output. During warmup ($N < 2$), returns 0.0.
6. **NaN propagation.** Non-finite inputs are substituted with the last valid value to prevent NaN from contaminating the rolling statistics.
## References
- Pearson, K. (1894). "Contributions to the Mathematical Theory of Evolution." *Philosophical Transactions of the Royal Society.*
- TradingView PineScript Reference: [ta.zscore](https://www.tradingview.com/pine-script-reference/v6/)
- Bollinger, J. (2001). *Bollinger on Bollinger Bands.* McGraw-Hill. (Z-score normalization of Bollinger %B)