Implement ZTEST: One-Sample t-Test Statistic with validation tests

- Added Ztest class to compute the one-sample t-statistic using sample standard deviation with Bessel correction.
- Implemented validation tests for Ztest to ensure accuracy against manual calculations and PineScript.
- Updated documentation for Ztest, detailing its mathematical foundation, performance profile, and common pitfalls.
- Adjusted NDepend badges to reflect changes in code metrics after implementation.
- Updated missing indicators report to reflect the completion of statistical indicators, including ZTEST.
This commit is contained in:
Miha Kralj
2026-02-16 16:54:36 -08:00
parent 09ffd31a40
commit b3a64f18fa
73 changed files with 13041 additions and 88 deletions
@@ -0,0 +1,136 @@
using TradingPlatform.BusinessLayer;
namespace QuanTAlib.Tests;
public sealed class SpearmanIndicatorTests
{
[Fact]
public void SpearmanIndicator_Constructor_SetsDefaults()
{
var indicator = new SpearmanIndicator();
Assert.Equal(20, indicator.Period);
Assert.Equal(SourceType.Close, indicator.Source);
Assert.Equal(SourceType.Open, indicator.Source2);
Assert.True(indicator.ShowColdValues);
Assert.Equal("SPEARMAN - Spearman Rank Correlation", indicator.Name);
Assert.True(indicator.SeparateWindow);
Assert.True(indicator.OnBackGround);
}
[Fact]
public void SpearmanIndicator_MinHistoryDepths_EqualsTwo()
{
var indicator = new SpearmanIndicator();
Assert.Equal(2, SpearmanIndicator.MinHistoryDepths);
Assert.Equal(2, ((IWatchlistIndicator)indicator).MinHistoryDepths);
}
[Fact]
public void SpearmanIndicator_ShortName_IncludesPeriodAndSources()
{
var indicator = new SpearmanIndicator { Period = 20 };
Assert.Contains("SPEARMAN", indicator.ShortName, StringComparison.Ordinal);
Assert.Contains("20", indicator.ShortName, StringComparison.Ordinal);
}
[Fact]
public void SpearmanIndicator_Initialize_CreatesInternalSpearman()
{
var indicator = new SpearmanIndicator { Period = 10 };
indicator.Initialize();
Assert.Single(indicator.LinesSeries);
}
[Fact]
public void SpearmanIndicator_ProcessUpdate_HistoricalBar_ComputesValue()
{
var indicator = new SpearmanIndicator { Period = 5 };
indicator.Initialize();
var now = DateTime.UtcNow;
indicator.HistoricalData.AddBar(now, 100, 105, 95, 102);
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
Assert.Equal(1, indicator.LinesSeries[0].Count);
}
[Fact]
public void SpearmanIndicator_ProcessUpdate_NewBar_ComputesValue()
{
var indicator = new SpearmanIndicator { Period = 5 };
indicator.Initialize();
var now = DateTime.UtcNow;
indicator.HistoricalData.AddBar(now, 100, 105, 95, 102);
indicator.HistoricalData.AddBar(now.AddMinutes(1), 102, 108, 100, 106);
indicator.ProcessUpdate(new UpdateArgs(UpdateReason.HistoricalBar));
indicator.ProcessUpdate(new UpdateArgs(UpdateReason.NewBar));
Assert.Equal(2, indicator.LinesSeries[0].Count);
}
[Fact]
public void SpearmanIndicator_ProcessUpdate_NewTick_ProcessesWithoutError()
{
var indicator = new SpearmanIndicator { Period = 5 };
indicator.Initialize();
var now = DateTime.UtcNow;
indicator.HistoricalData.AddBar(now, 100, 105, 95, 102);
indicator.ProcessUpdate(new UpdateArgs(UpdateReason.HistoricalBar));
double firstValue = indicator.LinesSeries[0].GetValue(0);
indicator.ProcessUpdate(new UpdateArgs(UpdateReason.NewTick));
double secondValue = indicator.LinesSeries[0].GetValue(0);
Assert.True(double.IsNaN(firstValue) || double.IsFinite(firstValue));
Assert.True(double.IsNaN(secondValue) || double.IsFinite(secondValue));
}
[Fact]
public void SpearmanIndicator_MultipleUpdates_ProducesSequence()
{
var indicator = new SpearmanIndicator { Period = 3 };
indicator.Initialize();
var now = DateTime.UtcNow;
double[] opens = [100, 101, 102, 103, 104, 105];
double[] closes = [100, 101, 102, 103, 104, 105];
for (int i = 0; i < opens.Length; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), opens[i], opens[i] + 5, opens[i] - 5, closes[i]);
indicator.ProcessUpdate(new UpdateArgs(i == 0 ? UpdateReason.HistoricalBar : UpdateReason.NewBar));
}
Assert.Equal(opens.Length, indicator.LinesSeries[0].Count);
}
[Fact]
public void SpearmanIndicator_DifferentSourceTypes_Work()
{
var sources = new[] { SourceType.Open, SourceType.High, SourceType.Low, SourceType.Close, SourceType.HL2, SourceType.HLC3 };
foreach (var source in sources)
{
var indicator = new SpearmanIndicator { Period = 5, Source = source, Source2 = SourceType.Close };
indicator.Initialize();
var now = DateTime.UtcNow;
indicator.HistoricalData.AddBar(now, 100, 105, 95, 102);
// Should not throw and should produce output
indicator.ProcessUpdate(new UpdateArgs(UpdateReason.HistoricalBar));
Assert.Equal(1, indicator.LinesSeries[0].Count);
}
}
}
@@ -0,0 +1,79 @@
using System.Drawing;
using System.Runtime.CompilerServices;
using TradingPlatform.BusinessLayer;
namespace QuanTAlib;
/// <summary>
/// Quantower adapter for Spearman Rank Correlation indicator.
/// Measures monotonic association between two price sources from the same symbol.
/// </summary>
/// <remarks>
/// This adapter compares two different price sources from the same symbol (e.g., Close vs Open,
/// Close vs Volume, High vs Low). For cross-symbol correlation, use the core
/// Spearman class directly.
///
/// Output is Spearman's ρ coefficient, ranging from -1 to +1.
/// Values near +1 indicate strong positive monotonic association, near -1 strong negative.
/// </remarks>
[SkipLocalsInit]
public sealed class SpearmanIndicator : Indicator, IWatchlistIndicator
{
[InputParameter("Period", sortIndex: 0, minimum: 2, maximum: 10000)]
public int Period { get; set; } = 20;
[IndicatorExtensions.DataSourceInput]
public SourceType Source { get; set; } = SourceType.Close;
[InputParameter("Source 2 Type", sortIndex: 2)]
public SourceType Source2 { get; set; } = SourceType.Open;
[InputParameter("Show cold values", sortIndex: 21)]
public bool ShowColdValues { get; set; } = true;
private Spearman _spearman = null!;
private readonly LineSeries _series;
private string _sourceName = null!;
private Func<IHistoryItem, double> _priceSelector = null!;
private Func<IHistoryItem, double> _priceSelector2 = null!;
public static int MinHistoryDepths => 2;
int IWatchlistIndicator.MinHistoryDepths => MinHistoryDepths;
public override string ShortName => $"SPEARMAN({Period}):{_sourceName}/{Source2}";
public SpearmanIndicator()
{
OnBackGround = true;
SeparateWindow = true;
Name = "SPEARMAN - Spearman Rank Correlation";
Description = "Measures monotonic association between two price sources. Range: -1 to +1.";
_series = new LineSeries(name: "Spearman", color: IndicatorExtensions.Statistics, width: 2, style: LineStyle.Solid);
AddLineSeries(_series);
}
protected override void OnInit()
{
_priceSelector = Source.GetPriceSelector();
_priceSelector2 = Source2.GetPriceSelector();
_sourceName = Source.ToString();
_spearman = new Spearman(Period);
base.OnInit();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnUpdate(UpdateArgs args)
{
bool isNew = args.IsNewBar();
var item = HistoricalData[Count - 1, SeekOriginHistory.Begin];
double valueA = _priceSelector(item);
double valueB = _priceSelector2(item);
var tvalA = new TValue(item.TimeLeft.Ticks, valueA);
var tvalB = new TValue(item.TimeLeft.Ticks, valueB);
double value = _spearman.Update(tvalA, tvalB, isNew).Value;
_series.SetValue(value, _spearman.IsHot, ShowColdValues);
}
}
+468
View File
@@ -0,0 +1,468 @@
namespace QuanTAlib.Tests;
public class SpearmanTests
{
[Fact]
public void Constructor_PeriodOne_Throws()
{
Assert.Throws<ArgumentException>(() => new Spearman(1));
}
[Fact]
public void Constructor_PeriodZero_Throws()
{
Assert.Throws<ArgumentException>(() => new Spearman(0));
}
[Fact]
public void Constructor_ValidPeriod_SetsName()
{
var s = new Spearman(10);
Assert.Equal("Spearman(10)", s.Name);
}
[Fact]
public void SingleInput_Update_ThrowsNotSupported()
{
var s = new Spearman(5);
Assert.Throws<NotSupportedException>(() => s.Update(new TValue(DateTime.UtcNow, 1.0)));
}
[Fact]
public void SingleInput_UpdateTSeries_ThrowsNotSupported()
{
var s = new Spearman(5);
var ts = new TSeries();
Assert.Throws<NotSupportedException>(() => s.Update(ts));
}
[Fact]
public void Prime_ThrowsNotSupported()
{
var s = new Spearman(5);
Assert.Throws<NotSupportedException>(() => s.Prime(stackalloc double[] { 1, 2, 3 }));
}
[Fact]
public void PerfectConcordance_ReturnsOne()
{
var s = new Spearman(5);
for (int i = 1; i <= 5; i++)
{
s.Update((double)i, (double)i, isNew: true);
}
Assert.Equal(1.0, s.Last.Value, 1e-10);
}
[Fact]
public void PerfectDiscordance_ReturnsMinusOne()
{
var s = new Spearman(5);
for (int i = 1; i <= 5; i++)
{
s.Update((double)i, 6.0 - i, isNew: true);
}
Assert.Equal(-1.0, s.Last.Value, 1e-10);
}
[Fact]
public void KnownSequence_MatchesExpected()
{
// X = [1,2,3,4,5], Y = [1,3,2,5,4]
// Ranks X = [1,2,3,4,5], Ranks Y = [1,3,2,5,4]
// d = [0,-1,1,-1,1], d² = [0,1,1,1,1], Σd² = 4
// ρ = 1 - 6*4/(5*24) = 1 - 24/120 = 1 - 0.2 = 0.8
var s = new Spearman(5);
double[] x = [1, 2, 3, 4, 5];
double[] y = [1, 3, 2, 5, 4];
for (int i = 0; i < 5; i++)
{
s.Update(x[i], y[i], isNew: true);
}
Assert.Equal(0.8, s.Last.Value, 1e-10);
}
[Fact]
public void Symmetry_RhoXY_EqualsRhoYX()
{
var s1 = new Spearman(5);
var s2 = new Spearman(5);
double[] x = [10, 20, 15, 30, 25];
double[] y = [5, 15, 10, 25, 20];
for (int i = 0; i < 5; i++)
{
s1.Update(x[i], y[i], isNew: true);
s2.Update(y[i], x[i], isNew: true);
}
Assert.Equal(s1.Last.Value, s2.Last.Value, 1e-10);
}
[Fact]
public void Antisymmetry_RhoXNegY_EqualsNegRhoXY()
{
var s1 = new Spearman(5);
var s2 = new Spearman(5);
double[] x = [10, 20, 15, 30, 25];
double[] y = [5, 15, 10, 25, 20];
for (int i = 0; i < 5; i++)
{
s1.Update(x[i], y[i], isNew: true);
s2.Update(x[i], -y[i], isNew: true);
}
Assert.Equal(-s1.Last.Value, s2.Last.Value, 1e-10);
}
[Fact]
public void ConstantSeries_ReturnsZero()
{
var s = new Spearman(5);
for (int i = 0; i < 5; i++)
{
s.Update(42.0, (double)(i + 1), isNew: true);
}
Assert.Equal(0.0, s.Last.Value, 1e-10);
}
[Fact]
public void BothConstant_ReturnsZero()
{
var s = new Spearman(5);
for (int i = 0; i < 5; i++)
{
s.Update(42.0, 42.0, isNew: true);
}
Assert.Equal(0.0, s.Last.Value, 1e-10);
}
[Fact]
public void TiedValues_HandledCorrectly()
{
// X = [1, 2, 2, 4, 5], Y = [5, 4, 3, 2, 1]
// Ranks X = [1, 2.5, 2.5, 4, 5] (ties → average rank)
// Ranks Y = [5, 4, 3, 2, 1]
// Pearson on these ranks → negative correlation
var s = new Spearman(5);
double[] x = [1, 2, 2, 4, 5];
double[] y = [5, 4, 3, 2, 1];
for (int i = 0; i < 5; i++)
{
s.Update(x[i], y[i], isNew: true);
}
// Should be close to -1 (strong negative monotonic relationship)
Assert.True(s.Last.Value < -0.9);
}
[Fact]
public void IsHot_RequiresAtLeastTwo()
{
var s = new Spearman(5);
Assert.False(s.IsHot);
s.Update(1.0, 2.0, isNew: true);
Assert.False(s.IsHot);
s.Update(2.0, 3.0, isNew: true);
Assert.True(s.IsHot);
}
[Fact]
public void SingleValue_ReturnsNaN()
{
var s = new Spearman(5);
s.Update(1.0, 2.0, isNew: true);
Assert.True(double.IsNaN(s.Last.Value));
}
[Fact]
public void IsNewFalse_CorrectsBars()
{
var s = new Spearman(5);
double[] x = [1, 2, 3, 4, 5];
double[] y = [2, 4, 6, 8, 10];
for (int i = 0; i < 5; i++)
{
s.Update(x[i], y[i], isNew: true);
}
double original = s.Last.Value;
// Correct with different value — break rank correlation
s.Update(100.0, 1.0, isNew: false);
double corrected = s.Last.Value;
Assert.NotEqual(original, corrected);
// Correct back to original
s.Update(x[4], y[4], isNew: false);
double restored = s.Last.Value;
Assert.Equal(original, restored, 1e-10);
}
[Fact]
public void NaN_SubstitutesLastValid()
{
var s = new Spearman(5);
for (int i = 1; i <= 4; i++)
{
s.Update((double)i, (double)i, isNew: true);
}
// Feed NaN — should use last valid value
s.Update(double.NaN, double.NaN, isNew: true);
Assert.True(double.IsFinite(s.Last.Value));
}
[Fact]
public void Infinity_SubstitutesLastValid()
{
var s = new Spearman(5);
for (int i = 1; i <= 4; i++)
{
s.Update((double)i, (double)i, isNew: true);
}
s.Update(double.PositiveInfinity, double.NegativeInfinity, isNew: true);
Assert.True(double.IsFinite(s.Last.Value));
}
[Fact]
public void Reset_ClearsState()
{
var s = new Spearman(5);
for (int i = 1; i <= 5; i++)
{
s.Update((double)i, (double)i, isNew: true);
}
Assert.True(s.IsHot);
s.Reset();
Assert.False(s.IsHot);
Assert.Equal(default, s.Last);
}
[Fact]
public void SlidingWindow_DropOldValues()
{
var s = new Spearman(3);
// Fill window: X=[1,2,3], Y=[1,2,3] → ρ = 1.0
s.Update(1.0, 1.0, isNew: true);
s.Update(2.0, 2.0, isNew: true);
s.Update(3.0, 3.0, isNew: true);
Assert.Equal(1.0, s.Last.Value, 1e-10);
// Push to window: X=[2,3,100], Y=[2,3,-100] → mixed correlation
s.Update(100.0, -100.0, isNew: true);
// Window now [2,3,100] vs [2,3,-100]: ranks X=[1,2,3], Y=[2,3,1] → not perfect
Assert.True(s.Last.Value < 1.0);
}
[Fact]
public void BatchTSeries_MatchesStreaming()
{
var gbmX = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 42);
var gbmY = new GBM(startPrice: 100, mu: 0.03, sigma: 0.15, seed: 99);
var seriesX = new TSeries();
var seriesY = new TSeries();
for (int i = 0; i < 50; i++)
{
var barX = gbmX.Next();
var barY = gbmY.Next();
seriesX.Add(new TValue(barX.Time, barX.Close));
seriesY.Add(new TValue(barY.Time, barY.Close));
}
TSeries batch = Spearman.Batch(seriesX, seriesY, 10);
var streaming = new Spearman(10);
for (int i = 0; i < 50; i++)
{
streaming.Update(seriesX[i], seriesY[i], isNew: true);
Assert.Equal(streaming.Last.Value, batch[i].Value, 1e-10);
}
}
[Fact]
public void BatchSpan_MatchesStreaming()
{
var gbmX = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 42);
var gbmY = new GBM(startPrice: 100, mu: 0.03, sigma: 0.15, seed: 99);
double[] xValues = new double[50];
double[] yValues = new double[50];
for (int i = 0; i < 50; i++)
{
xValues[i] = gbmX.Next().Close;
yValues[i] = gbmY.Next().Close;
}
double[] output = new double[50];
Spearman.Batch(xValues.AsSpan(), yValues.AsSpan(), output.AsSpan(), 10);
var streaming = new Spearman(10);
for (int i = 0; i < 50; i++)
{
streaming.Update(xValues[i], yValues[i], isNew: true);
Assert.Equal(streaming.Last.Value, output[i], 1e-10);
}
}
[Fact]
public void BatchSpan_MismatchedLengths_Throws()
{
double[] x = new double[10];
double[] y = new double[5];
double[] output = new double[10];
Assert.Throws<ArgumentException>(() =>
Spearman.Batch(x.AsSpan(), y.AsSpan(), output.AsSpan(), 3));
}
[Fact]
public void BatchSpan_MismatchedOutput_Throws()
{
double[] x = new double[10];
double[] y = new double[10];
double[] output = new double[5];
Assert.Throws<ArgumentException>(() =>
Spearman.Batch(x.AsSpan(), y.AsSpan(), output.AsSpan(), 3));
}
[Fact]
public void BatchSpan_InvalidPeriod_Throws()
{
double[] x = new double[10];
double[] y = new double[10];
double[] output = new double[10];
Assert.Throws<ArgumentException>(() =>
Spearman.Batch(x.AsSpan(), y.AsSpan(), output.AsSpan(), 1));
}
[Fact]
public void BatchTSeries_MismatchedLengths_Throws()
{
var sx = new TSeries();
var sy = new TSeries();
sx.Add(new TValue(DateTime.UtcNow, 1.0));
Assert.Throws<ArgumentException>(() => Spearman.Batch(sx, sy, 3));
}
[Fact]
public void Calculate_ReturnsTupleWithResults()
{
var gbmX = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 42);
var gbmY = new GBM(startPrice: 100, mu: 0.03, sigma: 0.15, seed: 99);
var seriesX = new TSeries();
var seriesY = new TSeries();
for (int i = 0; i < 30; i++)
{
var barX = gbmX.Next();
var barY = gbmY.Next();
seriesX.Add(new TValue(barX.Time, barX.Close));
seriesY.Add(new TValue(barY.Time, barY.Close));
}
var (results, indicator) = Spearman.Calculate(seriesX, seriesY, 10);
Assert.Equal(30, results.Count);
Assert.NotNull(indicator);
}
[Fact]
public void OutputBounded_BetweenMinusOneAndPlusOne()
{
var gbmX = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 42);
var gbmY = new GBM(startPrice: 100, mu: 0.03, sigma: 0.15, seed: 99);
var s = new Spearman(10);
for (int i = 0; i < 100; i++)
{
var barX = gbmX.Next();
var barY = gbmY.Next();
s.Update(barX.Close, barY.Close, isNew: true);
if (double.IsFinite(s.Last.Value))
{
Assert.InRange(s.Last.Value, -1.0, 1.0);
}
}
}
[Fact]
public void MonotonicTransform_PreservesCorrelation()
{
// Spearman measures monotonic association — applying a strictly increasing
// transform to either series should not change ρ
var s1 = new Spearman(5);
var s2 = new Spearman(5);
double[] x = [1, 2, 3, 4, 5];
double[] y = [10, 20, 15, 30, 25];
for (int i = 0; i < 5; i++)
{
s1.Update(x[i], y[i], isNew: true);
// Apply f(x) = x³ (strictly increasing)
s2.Update(x[i] * x[i] * x[i], y[i], isNew: true);
}
Assert.Equal(s1.Last.Value, s2.Last.Value, 1e-10);
}
[Fact]
public void EventChaining_Fires()
{
var s = new Spearman(3);
int eventCount = 0;
s.Pub += (object? _, in TValueEventArgs _) => eventCount++;
for (int i = 1; i <= 5; i++)
{
s.Update((double)i, (double)i, isNew: true);
}
Assert.Equal(5, eventCount);
}
[Fact]
public void BatchSpan_NaN_HandledSafely()
{
double[] x = [1, 2, double.NaN, 4, 5];
double[] y = [5, 4, 3, 2, 1];
double[] output = new double[5];
Spearman.Batch(x.AsSpan(), y.AsSpan(), output.AsSpan(), 3);
for (int i = 0; i < 5; i++)
{
Assert.True(double.IsFinite(output[i]) || double.IsNaN(output[i]));
}
}
[Fact]
public void LargePeriod_NoStackOverflow()
{
// Test with period > StackallocThreshold (256)
var s = new Spearman(300);
for (int i = 1; i <= 300; i++)
{
s.Update((double)i, (double)i, isNew: true);
}
Assert.Equal(1.0, s.Last.Value, 1e-10);
}
}
@@ -0,0 +1,107 @@
namespace QuanTAlib.Validation;
public sealed class SpearmanValidationTests
{
[Fact]
public void PerfectLinear_RhoEqualsOne()
{
// Perfect linear relationship: Y = 2X + 5
// Ranks of X and Y are identical → ρ = 1.0
var s = new Spearman(10);
for (int i = 1; i <= 10; i++)
{
s.Update((double)i, 2.0 * i + 5.0, isNew: true);
}
Assert.Equal(1.0, s.Last.Value, 1e-10);
}
[Fact]
public void PerfectNonlinearMonotonic_RhoEqualsOne()
{
// Perfect monotonic but nonlinear: Y = X³
// Ranks are identical → ρ = 1.0 (Spearman captures monotonic, not just linear)
var s = new Spearman(10);
for (int i = 1; i <= 10; i++)
{
double x = i;
s.Update(x, x * x * x, isNew: true);
}
Assert.Equal(1.0, s.Last.Value, 1e-10);
}
[Fact]
public void BatchAndStreaming_ProduceSameResults()
{
var gbmX = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 777);
var gbmY = new GBM(startPrice: 100, mu: 0.03, sigma: 0.15, seed: 888);
var seriesX = new TSeries();
var seriesY = new TSeries();
for (int i = 0; i < 50; i++)
{
var barX = gbmX.Next();
var barY = gbmY.Next();
seriesX.Add(new TValue(barX.Time, barX.Close));
seriesY.Add(new TValue(barY.Time, barY.Close));
}
TSeries batch = Spearman.Batch(seriesX, seriesY, 10);
var streaming = new Spearman(10);
for (int i = 0; i < 50; i++)
{
streaming.Update(seriesX[i], seriesY[i], isNew: true);
Assert.Equal(streaming.Last.Value, batch[i].Value, 1e-10);
}
}
[Fact]
public void KnownRanks_NoTies_MatchesSimplifiedFormula()
{
// Without ties: ρ = 1 - 6·Σd²/(n(n²-1))
// X = [10,20,30,40,50], Y = [50,30,10,40,20]
// Ranks X = [1,2,3,4,5], Ranks Y = [5,3,1,4,2]
// d = [-4,-1,2,0,3], d² = [16,1,4,0,9], Σd² = 30
// ρ = 1 - 6*30 / (5*24) = 1 - 180/120 = 1 - 1.5 = -0.5
var s = new Spearman(5);
double[] x = [10, 20, 30, 40, 50];
double[] y = [50, 30, 10, 40, 20];
for (int i = 0; i < 5; i++)
{
s.Update(x[i], y[i], isNew: true);
}
Assert.Equal(-0.5, s.Last.Value, 1e-10);
}
[Fact]
public void SpearmanVsKendall_BothDetectMonotonic()
{
// Both Spearman and Kendall should be +1 for perfectly concordant data
var spearman = new Spearman(5);
var kendall = new Kendall(5);
for (int i = 1; i <= 5; i++)
{
spearman.Update((double)i, (double)i, isNew: true);
kendall.Update(new TValue(DateTime.UtcNow, i), new TValue(DateTime.UtcNow, i), isNew: true);
}
Assert.Equal(1.0, spearman.Last.Value, 1e-10);
Assert.Equal(1.0, kendall.Last.Value, 1e-10);
}
[Fact]
public void BoundaryValues_AllTied()
{
// All X values identical → zero variance in ranks → ρ = 0
var s = new Spearman(5);
for (int i = 0; i < 5; i++)
{
s.Update(42.0, (double)(i + 1), isNew: true);
}
Assert.Equal(0.0, s.Last.Value, 1e-10);
}
}
+346
View File
@@ -0,0 +1,346 @@
using System.Buffers;
using System.Runtime.CompilerServices;
namespace QuanTAlib;
/// <summary>
/// Computes the Spearman Rank Correlation Coefficient (Spearman's ρ), which measures
/// the monotonic relationship between two series by applying Pearson correlation to
/// their ranks.
/// </summary>
/// <remarks>
/// Spearman's Rho Algorithm:
/// <c>ρ = Pearson(rank(X), rank(Y))</c>
///
/// Ranks are 1-based with average-rank tie-breaking: if k values share the same value,
/// each receives the mean of the positions they would occupy.
///
/// When no ties exist, the simplified formula applies:
/// <c>ρ = 1 - 6·Σd² / (n·(n²-1))</c>, where d_i = rank(x_i) - rank(y_i).
///
/// This implementation uses the general Pearson-on-ranks method because ties can occur
/// in financial data (identical closes, rounded prices). Ranking is O(n²) per series.
///
/// Non-finite inputs (NaN/±Inf) are sanitized by substituting the last finite value observed.
///
/// For the authoritative algorithm reference, full rationale, and behavioral contracts, see the
/// companion files in the same directory.
/// </remarks>
/// <seealso href="Spearman.md">Detailed documentation</seealso>
/// <seealso href="spearman.pine">Reference Pine Script implementation</seealso>
[SkipLocalsInit]
public sealed class Spearman : AbstractBase
{
private readonly RingBuffer _bufferX;
private readonly RingBuffer _bufferY;
private double _lastValidX, _lastValidY;
private const double Epsilon = 1e-10;
private const int StackallocThreshold = 256;
public override bool IsHot => _bufferX.Count >= 2;
/// <summary>
/// Creates a new Spearman Rank Correlation indicator.
/// </summary>
/// <param name="period">Lookback period for calculation (must be &gt; 1)</param>
public Spearman(int period = 20)
{
if (period <= 1)
{
throw new ArgumentException("Period must be greater than 1", nameof(period));
}
_bufferX = new RingBuffer(period);
_bufferY = new RingBuffer(period);
Name = $"Spearman({period})";
WarmupPeriod = period;
}
/// <summary>
/// Updates the Spearman indicator with new values from both series.
/// </summary>
/// <param name="seriesX">First series value</param>
/// <param name="seriesY">Second series value</param>
/// <param name="isNew">Whether this is a new bar</param>
/// <returns>Spearman's ρ coefficient (-1 to +1)</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TValue Update(TValue seriesX, TValue seriesY, bool isNew = true)
{
double x = SanitizeX(seriesX.Value);
double y = SanitizeY(seriesY.Value);
if (isNew || _bufferX.Count == 0)
{
_bufferX.Add(x);
_bufferY.Add(y);
}
else
{
_bufferX.UpdateNewest(x);
_bufferY.UpdateNewest(y);
}
double rho = CalculateRho();
Last = new TValue(seriesX.Time, rho);
PubEvent(Last);
return Last;
}
/// <summary>
/// Updates with raw double values.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TValue Update(double seriesX, double seriesY, bool isNew = true)
{
return Update(new TValue(DateTime.UtcNow, seriesX), new TValue(DateTime.UtcNow, seriesY), isNew);
}
/// <inheritdoc/>
/// <remarks>Not supported for dual-input indicator. Use Update(seriesX, seriesY) instead.</remarks>
public override TValue Update(TValue input, bool isNew = true)
{
throw new NotSupportedException("Spearman requires two inputs (seriesX and seriesY). Use Update(seriesX, seriesY).");
}
/// <inheritdoc/>
/// <remarks>Not supported for dual-input indicator. Use Batch(seriesX, seriesY, period) instead.</remarks>
public override TSeries Update(TSeries source)
{
throw new NotSupportedException("Spearman requires two inputs. Use Batch(seriesX, seriesY, period).");
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private double SanitizeX(double value)
{
if (double.IsFinite(value))
{
_lastValidX = value;
return value;
}
return double.IsFinite(_lastValidX) ? _lastValidX : 0.0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private double SanitizeY(double value)
{
if (double.IsFinite(value))
{
_lastValidY = value;
return value;
}
return double.IsFinite(_lastValidY) ? _lastValidY : 0.0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private double CalculateRho()
{
int n = _bufferX.Count;
if (n < 2)
{
return double.NaN;
}
// Allocate rank arrays — stackalloc for small, ArrayPool for large
double[]? rentedRx = null;
double[]? rentedRy = null;
scoped Span<double> rankX;
scoped Span<double> rankY;
if (n <= StackallocThreshold)
{
rankX = stackalloc double[n];
rankY = stackalloc double[n];
}
else
{
rentedRx = ArrayPool<double>.Shared.Rent(n);
rentedRy = ArrayPool<double>.Shared.Rent(n);
rankX = rentedRx.AsSpan(0, n);
rankY = rentedRy.AsSpan(0, n);
}
try
{
// Compute ranks for X and Y (average-rank tie-breaking)
ComputeRanks(_bufferX, n, rankX);
ComputeRanks(_bufferY, n, rankY);
// Pearson correlation on ranks
// For ranks 1..n without ties, mean = (n+1)/2
// With ties, mean still = (n+1)/2 because average-rank preserves sum
double meanRank = (n + 1) * 0.5;
double sumXY = 0;
double sumXX = 0;
double sumYY = 0;
for (int i = 0; i < n; i++)
{
double dx = rankX[i] - meanRank;
double dy = rankY[i] - meanRank;
sumXY += dx * dy;
sumXX += dx * dx;
sumYY += dy * dy;
}
if (sumXX < Epsilon || sumYY < Epsilon)
{
return 0.0; // Constant series → zero correlation
}
return sumXY / Math.Sqrt(sumXX * sumYY);
}
finally
{
if (rentedRx is not null)
{
ArrayPool<double>.Shared.Return(rentedRx);
}
if (rentedRy is not null)
{
ArrayPool<double>.Shared.Return(rentedRy);
}
}
}
/// <summary>
/// Computes 1-based average ranks for buffer values. O(n²).
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ComputeRanks(RingBuffer buffer, int n, Span<double> ranks)
{
for (int i = 0; i < n; i++)
{
double vi = buffer[i];
int countSmaller = 0;
int countEqual = 0;
for (int j = 0; j < n; j++)
{
double vj = buffer[j];
if (vj < vi)
{
countSmaller++;
}
if (vj == vi)
{
countEqual++; // includes self
}
}
// Average rank: 1-based position = countSmaller + (countEqual - 1) / 2.0 + 1
ranks[i] = countSmaller + (countEqual - 1) * 0.5 + 1.0;
}
}
/// <inheritdoc/>
public override void Prime(ReadOnlySpan<double> source, TimeSpan? step = null)
{
throw new NotSupportedException("Spearman requires two inputs.");
}
public override void Reset()
{
_bufferX.Clear();
_bufferY.Clear();
_lastValidX = 0;
_lastValidY = 0;
Last = default;
}
/// <summary>
/// Calculates Spearman's ρ for two time series.
/// </summary>
public static TSeries Batch(TSeries seriesX, TSeries seriesY, int period = 20, Spearman? indicator = null)
{
if (seriesX.Count != seriesY.Count)
{
throw new ArgumentException("Series must have the same length", nameof(seriesY));
}
indicator ??= new Spearman(period);
var result = new TSeries(seriesX.Count);
var timesX = seriesX.Times;
var valuesX = seriesX.Values;
var valuesY = seriesY.Values;
for (int i = 0; i < seriesX.Count; i++)
{
var tvalX = new TValue(timesX[i], valuesX[i]);
var tvalY = new TValue(timesX[i], valuesY[i]);
result.Add(indicator.Update(tvalX, tvalY, isNew: true));
}
return result;
}
/// <summary>
/// Static batch calculation for span-based processing with NaN sanitization.
/// </summary>
public static void Batch(
ReadOnlySpan<double> seriesX,
ReadOnlySpan<double> seriesY,
Span<double> output,
int period = 20)
{
if (seriesX.Length != seriesY.Length)
{
throw new ArgumentException("Series must have the same length", nameof(seriesY));
}
if (seriesX.Length != output.Length)
{
throw new ArgumentException("Output must have the same length as input", nameof(output));
}
if (period <= 1)
{
throw new ArgumentException("Period must be greater than 1", nameof(period));
}
var indicator = new Spearman(period);
double lastValidX = 0;
double lastValidY = 0;
for (int i = 0; i < seriesX.Length; i++)
{
double x = seriesX[i];
double y = seriesY[i];
if (double.IsFinite(x))
{
lastValidX = x;
}
else
{
x = lastValidX;
}
if (double.IsFinite(y))
{
lastValidY = y;
}
else
{
y = lastValidY;
}
var result = indicator.Update(x, y, isNew: true);
output[i] = result.Value;
}
}
public static (TSeries Results, Spearman Indicator) Calculate(TSeries seriesX, TSeries seriesY, int period = 20)
{
var indicator = new Spearman(period);
TSeries results = Batch(seriesX, seriesY, period, indicator);
return (results, indicator);
}
}
+138
View File
@@ -0,0 +1,138 @@
# SPEARMAN: Spearman Rank Correlation Coefficient
> "The person who asks whether rank correlation exists is not asking a wholly foolish question." — Maurice Kendall (1970)
Spearman's ρ (rho) measures the strength and direction of monotonic association between two variables. Unlike Pearson's correlation, which measures linear relationship, Spearman captures any monotonic relationship. A portfolio of stocks whose returns move monotonically together has different risk than one whose components merely share a linear trend. Spearman detects both.
## Historical Context
Charles Spearman introduced his rank correlation coefficient in 1904 while studying intelligence factor models. He needed a measure of association that did not require the assumption of normal distributions — a common problem with psychological test scores that tend toward ceiling and floor effects.
The insight was elegant: rank the data, then apply Pearson correlation to the ranks. This converts any monotonic relationship into a linear one, making Pearson applicable regardless of the original distribution shape. The resulting coefficient ρ inherits Pearson's bounded [-1, +1] range and its interpretation as a correlation measure, but measures monotonic rather than linear dependence.
In finance, Spearman sees use in pairs trading (detecting monotonic co-movement even when the functional form is unknown), risk modeling (copula estimation), and factor analysis (ranking stocks by multiple criteria). Its robustness to outliers and distribution shape makes it preferable to Pearson when price distributions exhibit fat tails.
## Architecture and Physics
### 1. Dual-Input Indicator
Spearman extends `AbstractBase` following the dual-input pattern established by `Correlation` and `Kendall`. Two `RingBuffer` instances track the rolling windows for X and Y series. Single-input `Update(TValue)` and `Update(TSeries)` throw `NotSupportedException` because the indicator requires paired observations.
### 2. Ranking Algorithm
For each update, the algorithm assigns ranks to both buffered series using average-rank tie-breaking:
$$\text{rank}(x_i) = |\{j : x_j < x_i\}| + \frac{|\{j : x_j = x_i\}| - 1}{2} + 1$$
This assigns each tied value the mean of the positions those tied values would occupy if they were distinct. The ranking step is O(n²) per series — each element is compared against all others.
### 3. Pearson on Ranks
After ranking, Spearman's ρ equals the Pearson correlation of the rank arrays:
$$\rho = \frac{\sum_{i=1}^{n}(R_{x_i} - \bar{R})(R_{y_i} - \bar{R})}{\sqrt{\sum_{i=1}^{n}(R_{x_i} - \bar{R})^2 \cdot \sum_{i=1}^{n}(R_{y_i} - \bar{R})^2}}$$
For ranks with average-rank tie-breaking, the mean rank is always $(n+1)/2$, regardless of ties. This is because the sum of ranks is preserved: tied elements receive the average of positions they span, which sums to the same total as distinct ranks.
### 4. Simplified Formula (No Ties)
When no ties exist, an algebraically equivalent shortcut applies:
$$\rho = 1 - \frac{6 \sum d_i^2}{n(n^2 - 1)}$$
where $d_i = R_{x_i} - R_{y_i}$. This implementation uses the general Pearson-on-ranks method because tied values occur in financial data (identical closes, rounded prices, trading halts).
## Mathematical Foundation
### Rank Assignment
Given values $\{v_1, v_2, \ldots, v_n\}$, the rank of $v_i$ is:
$$R_i = 1 + |\{j : v_j < v_i\}| + \frac{|\{j : v_j = v_i\}| - 1}{2}$$
### Pearson Correlation of Ranks
With $\bar{R} = (n+1)/2$:
$$\rho = \frac{\sum(R_{x_i} - \bar{R})(R_{y_i} - \bar{R})}{\sqrt{\sum(R_{x_i} - \bar{R})^2 \cdot \sum(R_{y_i} - \bar{R})^2}}$$
### Special Cases
| Condition | Result |
|-----------|--------|
| Perfect concordance (all ranks agree) | ρ = +1 |
| Perfect discordance (ranks reversed) | ρ = -1 |
| One or both series constant | ρ = 0 (zero-variance guard) |
| Fewer than 2 observations | ρ = NaN |
| All values tied | ρ = 0 |
### Relationship to Kendall's Tau
Both Spearman and Kendall measure monotonic association. For bivariate normal data:
$$\rho \approx \frac{3}{2}\tau$$
Spearman is more sensitive to large rank differences; Kendall weights all discordant pairs equally. In practice, both detect the same direction of association but differ in magnitude.
## Performance Profile
| Operation | Complexity | Notes |
|-----------|------------|-------|
| Ranking (per series) | O(n²) | Pairwise comparison for each element |
| Pearson on ranks | O(n) | Single pass over rank arrays |
| Total per update | O(n²) | Dominated by ranking |
| Memory | O(n) | Two RingBuffers + stackalloc ranks |
### SIMD Potential
Limited. The ranking step involves data-dependent branching (comparison counting) that resists vectorization. The Pearson correlation on ranks could theoretically use SIMD, but the O(n) savings are dwarfed by the O(n²) ranking cost. Not worth the complexity.
### Quality Metrics
| Metric | Score (1-10) |
|--------|-------------|
| Lag | 10 (no lag — contemporaneous measurement) |
| Smoothness | 3 (jumps when window slides) |
| Responsiveness | 7 (reacts to rank changes) |
| Robustness | 9 (outlier-resistant via ranking) |
| Interpretability | 9 ([-1, +1] bounded, intuitive) |
## Validation
No external TA library implements Spearman rank correlation. Validation relies on mathematical properties:
| Test | Method | Status |
|------|--------|--------|
| Perfect concordance | X = Y → ρ = 1 | ✔️ |
| Perfect discordance | X = -Y → ρ = -1 | ✔️ |
| Known sequence | Manual calculation verified | ✔️ |
| Symmetry | ρ(X,Y) = ρ(Y,X) | ✔️ |
| Antisymmetry | ρ(X,-Y) = -ρ(X,Y) | ✔️ |
| Monotonic invariance | f(X) monotone → ρ(f(X),Y) = ρ(X,Y) | ✔️ |
| Constant series | zero variance → ρ = 0 | ✔️ |
| Ties handled | average-rank tie-breaking verified | ✔️ |
| Batch/streaming consistency | identical outputs verified | ✔️ |
| Spearman vs Kendall | both = 1 for concordant data | ✔️ |
## Common Pitfalls
1. **Confusing Spearman with Pearson.** Pearson measures linear association; Spearman measures monotonic. A perfect exponential relationship gives ρ = 1 but r < 1. Choose based on the relationship type you expect.
2. **Period too large.** O(n²) ranking makes period > 60 expensive for real-time use. Period 20: ~800 comparisons per update; period 60: ~7200. Keep periods practical.
3. **Ignoring ties.** The simplified formula ρ = 1 - 6Σd²/(n(n²-1)) is incorrect when ties exist. Financial data with rounded prices creates ties. This implementation uses the general method.
4. **Single-series usage.** Spearman requires two series. Calling `Update(TValue)` throws `NotSupportedException`. Use `Update(seriesX, seriesY)`.
5. **Interpreting as causation.** High Spearman correlation indicates monotonic co-movement, not causation. Two stocks may correlate because of shared sector exposure, not because one drives the other.
6. **Small windows.** With n = 2, the only possible ρ values are -1 and +1 (or NaN if tied). Use period ≥ 5 for meaningful results.
7. **Comparing magnitude with Kendall.** For the same data, |ρ| ≥ |τ| generally holds. Do not compare raw values across methods without accounting for this scaling difference.
## References
- Spearman, C. (1904). "The Proof and Measurement of Association between Two Things." *American Journal of Psychology*, 15(1), 72-101.
- Kendall, M. G., & Gibbons, J. D. (1990). *Rank Correlation Methods*. 5th ed. Oxford University Press.
- Croux, C., & Dehon, C. (2010). "Influence Functions of the Spearman and Kendall Correlation Measures." *Statistical Methods & Applications*, 19(4), 497-515.
- Embrechts, P., McNeil, A., & Straumann, D. (2002). "Correlation and Dependence in Risk Management: Properties and Pitfalls." In *Risk Management: Value at Risk and Beyond*, Cambridge University Press.