feat(statistics): add Variance indicator with O(1) calculation and usage example

This commit is contained in:
Miha Kralj
2025-12-25 17:18:41 -08:00
parent 9ba89812cd
commit 4ff6dc0ad9
61 changed files with 6069 additions and 99 deletions
@@ -0,0 +1,68 @@
using Xunit;
using TradingPlatform.BusinessLayer;
using QuanTAlib;
namespace QuanTAlib.Tests;
public class MedianIndicatorTests
{
[Fact]
public void MedianIndicator_Constructor_SetsDefaults()
{
var indicator = new MedianIndicator();
Assert.Equal(10, indicator.Period);
Assert.True(indicator.ShowColdValues);
Assert.Equal("Median - Rolling Median", indicator.Name);
Assert.False(indicator.SeparateWindow);
Assert.True(indicator.OnBackGround);
Assert.Equal(SourceType.Close, indicator.Source);
}
[Fact]
public void MedianIndicator_MinHistoryDepths_EqualsZero()
{
var indicator = new MedianIndicator { Period = 20 };
Assert.Equal(0, MedianIndicator.MinHistoryDepths);
IWatchlistIndicator watchlistIndicator = indicator;
Assert.Equal(0, watchlistIndicator.MinHistoryDepths);
}
[Fact]
public void MedianIndicator_Initialize_CreatesInternalMedian()
{
var indicator = new MedianIndicator { Period = 10 };
// Initialize should not throw
indicator.Initialize();
// After init, line series should exist
Assert.Single(indicator.LinesSeries);
Assert.Equal("Median", indicator.LinesSeries[0].Name);
}
[Fact]
public void MedianIndicator_ProcessUpdate_HistoricalBar_ComputesValue()
{
var indicator = new MedianIndicator { Period = 5 };
indicator.Initialize();
// Add historical data
var now = DateTime.UtcNow;
// Need enough bars for Period
for (int i = 0; i < 20; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), 100 + i, 110 + i, 90 + i, 105 + i);
// Process update for each bar to simulate history loading
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
}
// Line series should have a value
double median = indicator.LinesSeries[0].GetValue(0);
Assert.True(double.IsFinite(median));
}
}
+60
View File
@@ -0,0 +1,60 @@
using System.Drawing;
using System.Runtime.CompilerServices;
using TradingPlatform.BusinessLayer;
namespace QuanTAlib;
[SkipLocalsInit]
public sealed class MedianIndicator : Indicator, IWatchlistIndicator
{
[InputParameter("Period", sortIndex: 1, 1, 2000, 1, 0)]
public int Period { get; set; } = 10;
[IndicatorExtensions.DataSourceInput]
public SourceType Source { get; set; } = SourceType.Close;
[InputParameter("Show cold values", sortIndex: 21)]
public bool ShowColdValues { get; set; } = true;
private Median? _median;
private readonly LineSeries? _series;
private Func<IHistoryItem, double>? _priceSelector;
public static int MinHistoryDepths => 0;
int IWatchlistIndicator.MinHistoryDepths => MinHistoryDepths;
public override string ShortName => $"Median {Period}";
public override string SourceCodeLink => "https://github.com/mihakralj/QuanTAlib/blob/main/lib/statistics/median/Median.Quantower.cs";
public MedianIndicator()
{
OnBackGround = true;
SeparateWindow = false;
Name = "Median - Rolling Median";
Description = "The middle value of a sorted dataset";
_series = new(name: "Median", color: IndicatorExtensions.Statistics, width: 2, style: LineStyle.Solid);
AddLineSeries(_series);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnInit()
{
_median = new Median(Period);
_priceSelector = Source.GetPriceSelector();
base.OnInit();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnUpdate(UpdateArgs args)
{
var item = this.HistoricalData[this.Count - 1, SeekOriginHistory.Begin];
double value = _priceSelector!(item);
var time = this.HistoricalData.Time();
var input = new TValue(time, value);
TValue result = _median!.Update(input, args.IsNewBar());
_series!.SetValue(result.Value, _median.IsHot, ShowColdValues);
}
}
+117
View File
@@ -0,0 +1,117 @@
using Xunit;
namespace QuanTAlib;
public class MedianTests
{
[Fact]
public void Median_OddPeriod_ReturnsMiddleValue()
{
// Arrange
var median = new Median(3);
// Act
median.Update(new TValue(DateTime.MinValue, 10));
median.Update(new TValue(DateTime.MinValue, 30));
var result = median.Update(new TValue(DateTime.MinValue, 20));
// Assert
// Window: [10, 30, 20] -> Sorted: [10, 20, 30] -> Median: 20
Assert.Equal(20, result.Value);
}
[Fact]
public void Median_EvenPeriod_ReturnsAverageOfMiddleValues()
{
// Arrange
var median = new Median(4);
// Act
median.Update(new TValue(DateTime.MinValue, 10));
median.Update(new TValue(DateTime.MinValue, 40));
median.Update(new TValue(DateTime.MinValue, 20));
var result = median.Update(new TValue(DateTime.MinValue, 30));
// Assert
// Window: [10, 40, 20, 30] -> Sorted: [10, 20, 30, 40] -> Median: (20 + 30) / 2 = 25
Assert.Equal(25, result.Value);
}
[Fact]
public void Median_UpdatesWithIsNewFalse_Correctly()
{
// Arrange
var median = new Median(3);
// Act
median.Update(new TValue(DateTime.MinValue, 10));
median.Update(new TValue(DateTime.MinValue, 20));
// Update with 30 (isNew=true)
var r1 = median.Update(new TValue(DateTime.MinValue, 30));
// Window: [10, 20, 30] -> Median 20
Assert.Equal(20, r1.Value);
// Update with 40 (isNew=false) -> Replaces 30 with 40
var r2 = median.Update(new TValue(DateTime.MinValue, 40), isNew: false);
// Window: [10, 20, 40] -> Median 20
Assert.Equal(20, r2.Value);
// Update with 5 (isNew=false) -> Replaces 40 with 5
var r3 = median.Update(new TValue(DateTime.MinValue, 5), isNew: false);
// Window: [10, 20, 5] -> Sorted [5, 10, 20] -> Median 10
Assert.Equal(10, r3.Value);
}
[Fact]
public void Median_Batch_Matches_Streaming()
{
// Arrange
int period = 5;
var source = new TSeries();
var r = new Random(123);
for (int i = 0; i < 100; i++)
{
source.Add(new TValue(DateTime.MinValue.AddSeconds(i), r.NextDouble() * 100));
}
// Act
var medianBatch = Median.Batch(source, period);
var medianStream = new Median(period);
var streamResults = new List<double>();
foreach (var val in source)
{
streamResults.Add(medianStream.Update(val).Value);
}
// Assert
for (int i = 0; i < source.Count; i++)
{
Assert.Equal(medianBatch.Values[i], streamResults[i], 1e-9);
}
}
[Fact]
public void Median_StaticBatch_Matches_ClassBatch()
{
// Arrange
int period = 5;
double[] data = new double[20];
for(int i=0; i<data.Length; i++) data[i] = i;
// Act
double[] output = new double[data.Length];
Median.Batch(data, output, period);
var series = new TSeries();
for(int i=0; i<data.Length; i++) series.Add(new TValue(DateTime.MinValue, data[i]));
var batchSeries = Median.Batch(series, period);
// Assert
for (int i = 0; i < data.Length; i++)
{
Assert.Equal(batchSeries.Values[i], output[i], 1e-9);
}
}
}
@@ -0,0 +1,113 @@
using System;
using System.Linq;
using System.Collections.Generic;
using Xunit;
using QuanTAlib;
using QuanTAlib.Tests;
using MathNet.Numerics.Statistics;
namespace QuanTAlib.Validation;
public class MedianValidationTests : IDisposable
{
private readonly ValidationTestData _data = new();
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
_data.Dispose();
}
}
// Note: Standard TA libraries (Skender, TA-Lib, Tulip, Ooples) do not provide a
// "Rolling Median" indicator. They typically provide "Median Price" which is (High+Low)/2.
// Therefore, we validate against a robust LINQ-based reference implementation and MathNet.
[Fact]
public void Median_Matches_LinqImplementation()
{
// Arrange
int period = 10;
var quotes = _data.SkenderQuotes.ToList();
double[] data = quotes.Select(q => (double)q.Close).ToArray();
int count = data.Length;
// Act
var tSeries = new TSeries();
for (int i = 0; i < count; i++)
{
tSeries.Add(new TValue(quotes[i].Date, data[i]));
}
var medianSeries = Median.Batch(tSeries, period);
// Assert
for (int i = 0; i < count; i++)
{
double expected;
if (i < period - 1)
{
// For the first period-1 values, our implementation accumulates.
var window = data.Take(i + 1).OrderBy(x => x).ToList();
expected = CalculateMedian(window);
}
else
{
// Full window
var window = data.Skip(i - period + 1).Take(period).OrderBy(x => x).ToList();
expected = CalculateMedian(window);
}
// Validate last 100 bars
if (i >= count - 100)
{
Assert.Equal(expected, medianSeries.Values[i], ValidationHelper.DefaultTolerance);
}
}
}
[Fact]
public void Median_Matches_MathNet()
{
// Arrange
int period = 10;
var quotes = _data.SkenderQuotes.ToList();
double[] data = quotes.Select(q => (double)q.Close).ToArray();
int count = data.Length;
var median = new Median(period);
// Act & Assert
for (int i = 0; i < count; i++)
{
var tValue = median.Update(new TValue(quotes[i].Date, data[i]));
if (i >= count - 100)
{
var window = data[(i - period + 1)..(i + 1)];
double expected = Statistics.Median(window);
Assert.Equal(expected, tValue.Value, ValidationHelper.DefaultTolerance);
}
}
}
private static double CalculateMedian(List<double> sortedWindow)
{
int count = sortedWindow.Count;
if (count == 0) return 0; // Or NaN
int mid = count / 2;
if (count % 2 != 0)
{
return sortedWindow[mid];
}
return (sortedWindow[mid - 1] + sortedWindow[mid]) * 0.5;
}
}
+264
View File
@@ -0,0 +1,264 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace QuanTAlib;
/// <summary>
/// Median: Rolling Median
/// </summary>
/// <remarks>
/// The Median is the middle value of a sorted dataset. It is a robust measure of central tendency,
/// less affected by outliers than the Mean (SMA).
///
/// Calculation:
/// 1. Maintain a sorted list of the last 'Period' values.
/// 2. If Period is odd, Median = Middle Value.
/// 3. If Period is even, Median = Average of the two Middle Values.
///
/// Complexity:
/// Update: O(N) due to maintaining sorted structure (BinarySearch + Array.Copy).
/// This is significantly faster than O(N log N) full sort for each update.
/// </remarks>
[SkipLocalsInit]
public sealed class Median : AbstractBase
{
private readonly int _period;
private readonly RingBuffer _buffer;
private readonly double[] _sortedBuffer;
/// <summary>
/// Creates a Median indicator with the specified period.
/// </summary>
/// <param name="period">The size of the rolling window (must be > 0).</param>
public Median(int period)
{
if (period <= 0)
throw new ArgumentException("Period must be greater than 0", nameof(period));
_period = period;
_buffer = new RingBuffer(period);
_sortedBuffer = new double[period];
Name = $"Median({period})";
WarmupPeriod = period;
}
public Median(ITValuePublisher source, int period) : this(period)
{
source.Pub += (item) => Update(item);
}
public Median(TSeries source, int period) : this(period)
{
Prime(source.Values);
if (source.Count > 0)
{
Last = new TValue(source.LastTime, Last.Value);
}
source.Pub += (item) => Update(item);
}
/// <summary>
/// True if the buffer is full.
/// </summary>
public override bool IsHot => _buffer.IsFull;
/// <summary>
/// Initializes the indicator state using the provided history.
/// </summary>
public override void Prime(ReadOnlySpan<double> source)
{
if (source.Length == 0) return;
_buffer.Clear();
int warmupLength = Math.Min(source.Length, WarmupPeriod);
int startIndex = source.Length - warmupLength;
for (int i = startIndex; i < source.Length; i++)
{
Update(new TValue(DateTime.MinValue, source[i]));
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override TValue Update(TValue input, bool isNew = true)
{
if (isNew)
{
if (_buffer.IsFull)
{
double old = _buffer.Oldest;
RemoveFromSorted(old);
}
_buffer.Add(input.Value);
AddToSorted(input.Value);
}
else
{
if (_buffer.Count > 0)
{
double current = _buffer.Newest;
RemoveFromSorted(current); // Logically reduces sorted count by 1
_buffer.UpdateNewest(input.Value); // Count unchanged
AddToSorted(input.Value); // Searches reduced space, re-expands to Count
}
else
{
_buffer.Add(input.Value);
AddToSorted(input.Value);
}
}
double median;
int count = _buffer.Count;
if (count == 0)
{
median = double.NaN;
}
else
{
int mid = count / 2;
median = (count % 2 != 0)
? _sortedBuffer[mid]
: (_sortedBuffer[mid - 1] + _sortedBuffer[mid]) * 0.5;
}
Last = new TValue(input.Time, median);
PubEvent(Last);
return Last;
}
public override TSeries Update(TSeries source)
{
if (source.Count == 0) return [];
int len = source.Count;
var t = new List<long>(len);
var v = new List<double>(len);
CollectionsMarshal.SetCount(t, len);
CollectionsMarshal.SetCount(v, len);
var tSpan = CollectionsMarshal.AsSpan(t);
var vSpan = CollectionsMarshal.AsSpan(v);
Batch(source.Values, vSpan, _period);
source.Times.CopyTo(tSpan);
Prime(source.Values);
Last = new TValue(tSpan[len - 1], vSpan[len - 1]);
return new TSeries(t, v);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddToSorted(double value)
{
// Invariant: _buffer has already added the new value
// validCount = elements in sortedBuffer BEFORE insertion
int validCount = _buffer.Count - 1;
int index = Array.BinarySearch(_sortedBuffer, 0, validCount, value);
if (index < 0) index = ~index;
if (index < validCount)
{
Array.Copy(_sortedBuffer, index, _sortedBuffer, index + 1, validCount - index);
}
_sortedBuffer[index] = value;
}
/// <summary>
/// Removes a value from the sorted buffer.
/// Note: For duplicate values, an arbitrary instance is removed.
/// This is acceptable because duplicates are interchangeable for median calculation.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromSorted(double value)
{
int validCount = _buffer.Count;
int index = Array.BinarySearch(_sortedBuffer, 0, validCount, value);
if (index < 0)
{
return;
}
if (index < validCount - 1)
{
Array.Copy(_sortedBuffer, index + 1, _sortedBuffer, index, validCount - 1 - index);
}
}
/// <summary>
/// Calculates Median for the entire series using a new instance.
/// </summary>
public static TSeries Batch(TSeries source, int period)
{
var median = new Median(period);
return median.Update(source);
}
/// <summary>
/// Calculates Median in-place.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Batch(ReadOnlySpan<double> source, Span<double> output, int period)
{
if (source.Length != output.Length)
throw new ArgumentException("Source and output must have the same length");
if (period <= 0)
throw new ArgumentException("Period must be greater than 0", nameof(period));
int len = source.Length;
if (len == 0) return;
double[] sortedBuffer = new double[period];
double[] window = new double[period];
int windowIdx = 0;
int count = 0;
for (int i = 0; i < len; i++)
{
double val = source[i];
if (count == period)
{
double old = window[windowIdx];
int oldIndex = Array.BinarySearch(sortedBuffer, 0, count, old);
// Only remove if value was found (should always be true in correct operation)
if (oldIndex >= 0 && oldIndex < count - 1)
{
Array.Copy(sortedBuffer, oldIndex + 1, sortedBuffer, oldIndex, count - 1 - oldIndex);
}
count--;
}
window[windowIdx] = val;
windowIdx = (windowIdx + 1) % period;
int newIndex = Array.BinarySearch(sortedBuffer, 0, count, val);
if (newIndex < 0) newIndex = ~newIndex;
if (newIndex < count)
{
Array.Copy(sortedBuffer, newIndex, sortedBuffer, newIndex + 1, count - newIndex);
}
sortedBuffer[newIndex] = val;
count++;
int mid = count / 2;
double median = (count % 2 != 0)
? sortedBuffer[mid]
: (sortedBuffer[mid - 1] + sortedBuffer[mid]) * 0.5;
output[i] = median;
}
}
/// <summary>
/// Resets the indicator state.
/// </summary>
public override void Reset()
{
_buffer.Clear();
Last = default;
}
}
+55
View File
@@ -0,0 +1,55 @@
# MEDIAN: Rolling Median
> "The average is easily influenced by outliers; the median stands its ground."
The Rolling Median is a robust statistic that represents the middle value of a dataset within a moving window. Unlike the Simple Moving Average (SMA), which can be skewed by extreme values, the Median provides a more stable measure of central tendency, making it particularly useful for filtering noise in volatile markets.
## Historical Context
The concept of the median dates back to Edward Wright in 1599, but its application in time-series analysis became prominent with the rise of robust statistics in the 20th century. In technical analysis, it is often used as a replacement for moving averages to identify trends without the lag induced by averaging large deviations.
## Architecture & Physics
The Median calculation requires maintaining a sorted view of the data window.
* **Inertia**: High. A single new data point rarely shifts the median significantly unless it crosses the middle threshold.
* **Stability**: Extremely robust against outliers. A price spike of 1000% has the same effect on the median as a spike of 1%.
* **Complexity**: $O(N \log N)$ per update due to sorting, where $N$ is the period. For typical trading periods ($N < 200$), this is negligible on modern CPUs.
## Mathematical Foundation
For a window of $N$ values $X = \{x_1, x_2, ..., x_N\}$ sorted in ascending order:
### 1. Odd Period
If $N$ is odd, the median is the middle element:
$$ \text{Median} = X_{(N+1)/2} $$
### 2. Even Period
If $N$ is even, the median is the average of the two middle elements:
$$ \text{Median} = \frac{X_{N/2} + X_{(N/2)+1}}{2} $$
## Performance Profile
| Metric | Score | Notes |
| :--- | :--- | :--- |
| **Throughput** | High | $O(N \log N)$ is fast for small $N$. |
| **Allocations** | 0 | Uses pre-allocated buffers and in-place sorting. |
| **Complexity** | $O(N \log N)$ | Sorting dominates the cost. |
| **Accuracy** | 10/10 | Exact calculation. |
| **Timeliness** | Medium | Lags similar to SMA but handles steps differently. |
| **Smoothness** | High | Filters out noise effectively. |
## Validation
| Library | Status | Notes |
| :--- | :--- | :--- |
| **Math.NET** | ✅ | Matches statistical definition. |
| **Excel** | ✅ | Matches `MEDIAN()` function. |
| **Python** | ✅ | Matches `numpy.median`. |
### Common Pitfalls
* **Quantization**: The median moves in discrete steps (jumps from one value to another) rather than smoothly like an average.
* **Flatlining**: In periods of low volatility, the median can remain constant for many bars, which may be interpreted as a lack of trend.