feat(statistics): add Variance indicator with O(1) calculation and usage example

This commit is contained in:
Miha Kralj
2025-12-25 17:18:41 -08:00
parent 9ba89812cd
commit 4ff6dc0ad9
61 changed files with 6069 additions and 99 deletions
@@ -0,0 +1,69 @@
using Xunit;
using TradingPlatform.BusinessLayer;
using QuanTAlib;
namespace QuanTAlib.Tests;
public class VarianceIndicatorTests
{
[Fact]
public void VarianceIndicator_Constructor_SetsDefaults()
{
var indicator = new VarianceIndicator();
Assert.Equal(20, indicator.Period);
Assert.False(indicator.IsPopulation);
Assert.True(indicator.ShowColdValues);
Assert.Equal("Variance - Rolling Variance", indicator.Name);
Assert.True(indicator.SeparateWindow);
Assert.True(indicator.OnBackGround);
Assert.Equal(SourceType.Close, indicator.Source);
}
[Fact]
public void VarianceIndicator_MinHistoryDepths_EqualsZero()
{
var indicator = new VarianceIndicator { Period = 20 };
Assert.Equal(0, VarianceIndicator.MinHistoryDepths);
IWatchlistIndicator watchlistIndicator = indicator;
Assert.Equal(0, watchlistIndicator.MinHistoryDepths);
}
[Fact]
public void VarianceIndicator_Initialize_CreatesInternalVariance()
{
var indicator = new VarianceIndicator { Period = 10 };
// Initialize should not throw
indicator.Initialize();
// After init, line series should exist
Assert.Single(indicator.LinesSeries);
Assert.Equal("Variance", indicator.LinesSeries[0].Name);
}
[Fact]
public void VarianceIndicator_ProcessUpdate_HistoricalBar_ComputesValue()
{
var indicator = new VarianceIndicator { Period = 5 };
indicator.Initialize();
// Add historical data
var now = DateTime.UtcNow;
// Need enough bars for Period
for (int i = 0; i < 20; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), 100 + i, 110 + i, 90 + i, 105 + i);
// Process update for each bar to simulate history loading
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
}
// Line series should have a value
double variance = indicator.LinesSeries[0].GetValue(0);
Assert.True(double.IsFinite(variance));
}
}
@@ -0,0 +1,63 @@
using System.Drawing;
using System.Runtime.CompilerServices;
using TradingPlatform.BusinessLayer;
namespace QuanTAlib;
[SkipLocalsInit]
public sealed class VarianceIndicator : Indicator, IWatchlistIndicator
{
[InputParameter("Period", sortIndex: 1, 1, 2000, 1, 0)]
public int Period { get; set; } = 20;
[InputParameter("Population Variance", sortIndex: 2)]
public bool IsPopulation { get; set; } = false;
[IndicatorExtensions.DataSourceInput]
public SourceType Source { get; set; } = SourceType.Close;
[InputParameter("Show cold values", sortIndex: 21)]
public bool ShowColdValues { get; set; } = true;
private Variance? _variance;
private readonly LineSeries? _series;
private Func<IHistoryItem, double>? _priceSelector;
public static int MinHistoryDepths => 0;
int IWatchlistIndicator.MinHistoryDepths => MinHistoryDepths;
public override string ShortName => $"Variance {Period}";
public override string SourceCodeLink => "https://github.com/mihakralj/QuanTAlib/blob/main/lib/statistics/variance/Variance.Quantower.cs";
public VarianceIndicator()
{
OnBackGround = true;
SeparateWindow = true;
Name = "Variance - Rolling Variance";
Description = "Measures the dispersion of a set of data points around their mean";
_series = new(name: "Variance", color: IndicatorExtensions.Statistics, width: 2, style: LineStyle.Solid);
AddLineSeries(_series);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnInit()
{
_variance = new Variance(Period, IsPopulation);
_priceSelector = Source.GetPriceSelector();
base.OnInit();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnUpdate(UpdateArgs args)
{
var item = this.HistoricalData[this.Count - 1, SeekOriginHistory.Begin];
double value = _priceSelector!(item);
var time = this.HistoricalData.Time();
var input = new TValue(time, value);
TValue result = _variance!.Update(input, args.IsNewBar());
_series!.SetValue(result.Value, _variance.IsHot, ShowColdValues);
}
}
+124
View File
@@ -0,0 +1,124 @@
using System;
using Xunit;
namespace QuanTAlib.Tests;
public class VarianceTests
{
[Fact]
public void Constructor_ValidatesPeriod()
{
Assert.Throws<ArgumentOutOfRangeException>(() => new Variance(1));
}
[Fact]
public void Calculation_KnownValues()
{
// Data: 2, 4, 4, 4, 5, 5, 7, 9
// Mean: 5
// Deviations: -3, -1, -1, -1, 0, 0, 2, 4
// Sq Devs: 9, 1, 1, 1, 0, 0, 4, 16
// Sum Sq Devs: 32
// Population Variance (N=8): 32 / 8 = 4
// Sample Variance (N-1=7): 32 / 7 = 4.571428...
var data = new double[] { 2, 4, 4, 4, 5, 5, 7, 9 };
// Test Population Variance
var popVar = new Variance(8, isPopulation: true);
foreach (var val in data)
{
popVar.Update(new TValue(DateTime.UtcNow, val));
}
Assert.Equal(4.0, popVar.Last.Value, precision: 6);
// Test Sample Variance
var sampVar = new Variance(8, isPopulation: false);
foreach (var val in data)
{
sampVar.Update(new TValue(DateTime.UtcNow, val));
}
Assert.Equal(32.0 / 7.0, sampVar.Last.Value, precision: 6);
}
[Fact]
public void IsHot_BecomesTrueAfterPeriod()
{
int period = 5;
var variance = new Variance(period);
for (int i = 0; i < period; i++)
{
Assert.False(variance.IsHot);
variance.Update(new TValue(DateTime.UtcNow, i));
}
Assert.True(variance.IsHot);
}
[Fact]
public void Reset_ClearsState()
{
var variance = new Variance(5);
for (int i = 0; i < 10; i++)
{
variance.Update(new TValue(DateTime.UtcNow, i));
}
Assert.True(variance.IsHot);
variance.Reset();
Assert.False(variance.IsHot);
Assert.Equal(0, variance.Last.Value);
}
[Fact]
public void Update_IsNewFalse_UpdatesCorrectly()
{
// Test differential update
var variance = new Variance(3, isPopulation: true);
// Add 1, 2, 3. Mean=2. Var = ((1-2)^2 + (2-2)^2 + (3-2)^2)/3 = (1+0+1)/3 = 2/3 = 0.666...
variance.Update(new TValue(DateTime.UtcNow, 1));
variance.Update(new TValue(DateTime.UtcNow, 2));
variance.Update(new TValue(DateTime.UtcNow, 3));
Assert.Equal(2.0/3.0, variance.Last.Value, precision: 6);
// Update last value from 3 to 6.
// Data: 1, 2, 6. Mean=3. Var = ((1-3)^2 + (2-3)^2 + (6-3)^2)/3 = (4+1+9)/3 = 14/3 = 4.666...
variance.Update(new TValue(DateTime.UtcNow, 6), isNew: false);
Assert.Equal(14.0/3.0, variance.Last.Value, precision: 6);
}
[Fact]
public void Batch_Matches_Iterative()
{
int period = 10;
int count = 1000;
var data = new double[count];
var random = new Random(123);
for (int i = 0; i < count; i++)
{
data[i] = random.NextDouble() * 100;
}
// Iterative
var variance = new Variance(period);
var iterativeResults = new double[count];
for (int i = 0; i < count; i++)
{
variance.Update(new TValue(DateTime.UtcNow, data[i]));
iterativeResults[i] = variance.Last.Value;
}
// Batch
var batchResults = new double[count];
Variance.Batch(data, batchResults, period);
// Compare
for (int i = 0; i < count; i++)
{
Assert.Equal(iterativeResults[i], batchResults[i], precision: 7);
}
}
}
@@ -0,0 +1,135 @@
using System;
using System.Linq;
using Xunit;
using QuanTAlib;
using QuanTAlib.Tests;
using Skender.Stock.Indicators;
using TALib;
using Tulip;
using OoplesFinance.StockIndicators;
using OoplesFinance.StockIndicators.Models;
using OoplesFinance.StockIndicators.Enums;
using MathNet.Numerics.Statistics;
namespace QuanTAlib.Validation;
public class VarianceValidationTests
{
private readonly ValidationTestData _data = new();
[Fact]
public void Variance_Matches_Skender_StdDev_Squared()
{
// Skender StdDev uses Population Standard Deviation (N) for calculation,
// despite documentation often implying Sample (N-1).
// Variance(isPopulation: true) should match StdDev^2.
int period = 20;
var variance = new Variance(period, isPopulation: true);
var skenderStdDev = _data.SkenderQuotes.GetStdDev(period);
var skenderList = skenderStdDev.ToList();
var quotes = _data.SkenderQuotes.ToList();
for (int i = 0; i < quotes.Count; i++)
{
var tValue = variance.Update(new TValue(quotes[i].Date, (double)quotes[i].Close));
var skenderVal = skenderList[i].StdDev;
if (i >= period && skenderVal.HasValue)
{
double expectedVariance = skenderVal.Value * skenderVal.Value;
Assert.Equal(expectedVariance, tValue.Value, ValidationHelper.DefaultTolerance);
}
}
}
[Fact]
public void Variance_Matches_Talib_Var()
{
// TA-Lib VAR uses Population Variance (N)
int period = 20;
var variance = new Variance(period, isPopulation: true);
var quotes = _data.SkenderQuotes.ToList();
double[] input = quotes.Select(q => (double)q.Close).ToArray();
double[] output = new double[input.Length];
// TA-Lib calculation
// VAR(real, timeperiod=5, nbdev=1)
var retCode = TALib.Functions.Var(input, 0..^0, output, out var outRange, period);
Assert.Equal(TALib.Core.RetCode.Success, retCode);
for (int i = 0; i < quotes.Count; i++)
{
var tValue = variance.Update(new TValue(quotes[i].Date, (double)quotes[i].Close));
if (i >= outRange.Start.Value)
{
double talibVal = output[i - outRange.Start.Value];
Assert.Equal(talibVal, tValue.Value, ValidationHelper.DefaultTolerance);
}
}
}
[Fact]
public void Variance_Matches_Tulip_Var()
{
// Tulip VAR uses Population Variance (N)
int period = 20;
var variance = new Variance(period, isPopulation: true);
var quotes = _data.SkenderQuotes.ToList();
double[] input = quotes.Select(q => (double)q.Close).ToArray();
// Tulip calculation
var varInd = Tulip.Indicators.var;
double[][] inputs = { input };
double[] options = { period };
double[][] outputs = { new double[input.Length - varInd.Start(options)] };
varInd.Run(inputs, options, outputs);
double[] output = outputs[0];
int lookback = varInd.Start(options);
for (int i = 0; i < quotes.Count; i++)
{
var tValue = variance.Update(new TValue(quotes[i].Date, (double)quotes[i].Close));
if (i >= lookback)
{
double tulipVal = output[i - lookback];
Assert.Equal(tulipVal, tValue.Value, ValidationHelper.DefaultTolerance);
}
}
}
[Fact]
public void Variance_Matches_MathNet()
{
int period = 20;
var variance = new Variance(period, isPopulation: false);
var popVariance = new Variance(period, isPopulation: true);
var quotes = _data.SkenderQuotes.ToList();
double[] input = quotes.Select(q => (double)q.Close).ToArray();
for (int i = 0; i < input.Length; i++)
{
var val = variance.Update(new TValue(DateTime.UtcNow, input[i]));
var popVal = popVariance.Update(new TValue(DateTime.UtcNow, input[i]));
if (i >= input.Length - 100)
{
var window = input[(i - period + 1)..(i + 1)];
double expected = Statistics.Variance(window);
double expectedPop = Statistics.PopulationVariance(window);
Assert.Equal(expected, val.Value, ValidationHelper.DefaultTolerance);
Assert.Equal(expectedPop, popVal.Value, ValidationHelper.DefaultTolerance);
}
}
}
}
+641
View File
@@ -0,0 +1,641 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
namespace QuanTAlib;
/// <summary>
/// Variance: Measures the dispersion of a set of data points around their mean.
/// </summary>
/// <remarks>
/// Variance is calculated as the average of the squared differences from the Mean.
///
/// Formula:
/// Population Variance = Sum((x - Mean)^2) / N
/// Sample Variance = Sum((x - Mean)^2) / (N - 1)
///
/// This implementation uses the O(1) running sum of squares formula:
/// Variance = (SumSq - (Sum * Sum) / N) / (N - 1) (for Sample)
/// </remarks>
[SkipLocalsInit]
public sealed class Variance : AbstractBase
{
private readonly int _period;
private readonly RingBuffer _buffer;
private readonly bool _isPopulation;
private double _sumSq;
private int _updateCount;
private const int ResyncInterval = 1000;
public override bool IsHot => _buffer.IsFull;
/// <summary>
/// Creates a new Variance indicator.
/// </summary>
/// <param name="period">The lookback period.</param>
/// <param name="isPopulation">If true, calculates Population Variance (div by N). If false, Sample Variance (div by N-1). Default is false (Sample).</param>
public Variance(int period, bool isPopulation = false)
{
if (period < 2)
{
throw new ArgumentOutOfRangeException(nameof(period), "Period must be greater than or equal to 2.");
}
_period = period;
_isPopulation = isPopulation;
_buffer = new RingBuffer(period);
Name = $"Variance({period})";
WarmupPeriod = period;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override TValue Update(TValue input, bool isNew = true)
{
if (isNew)
{
if (_buffer.IsFull)
{
double oldVal = _buffer.Oldest;
_sumSq = Math.FusedMultiplyAdd(-oldVal, oldVal, _sumSq);
}
_buffer.Add(input.Value);
_sumSq = Math.FusedMultiplyAdd(input.Value, input.Value, _sumSq);
_updateCount++;
if (_updateCount % ResyncInterval == 0)
{
Resync();
}
}
else
{
// Differential update
double oldNewest = _buffer.Newest;
_buffer.UpdateNewest(input.Value);
// Reconstruct SumSq from previous state is safer/cleaner than differential on current
// But we updated buffer already.
// _sumSq currently includes oldNewest^2.
// We want to remove oldNewest^2 and add input^2.
_sumSq = Math.FusedMultiplyAdd(-oldNewest, oldNewest, _sumSq);
_sumSq = Math.FusedMultiplyAdd(input.Value, input.Value, _sumSq);
}
double variance = 0;
if (_buffer.Count > 1)
{
double n = _buffer.Count;
// Var = (SumSq - 2*Mean*Sum + N*Mean^2) / (N or N-1)
// Var = (SumSq - 2*Mean*(N*Mean) + N*Mean^2) / ...
// Var = (SumSq - 2*N*Mean^2 + N*Mean^2) / ...
// Var = (SumSq - N*Mean^2) / ...
// Using Sum:
// Var = (SumSq - (Sum*Sum)/N) / ...
double numerator = _sumSq - (_buffer.Sum * _buffer.Sum) / n;
// Handle floating point noise
if (numerator < 0) numerator = 0;
double denominator = _isPopulation ? n : (n - 1);
variance = numerator / denominator;
}
Last = new TValue(input.Time, variance);
PubEvent(Last);
return Last;
}
public override TSeries Update(TSeries source)
{
if (source.Count == 0) return [];
int len = source.Count;
var t = new List<long>(len);
var v = new List<double>(len);
CollectionsMarshal.SetCount(t, len);
CollectionsMarshal.SetCount(v, len);
var tSpan = CollectionsMarshal.AsSpan(t);
var vSpan = CollectionsMarshal.AsSpan(v);
Batch(source.Values, vSpan, _period, _isPopulation);
source.Times.CopyTo(tSpan);
// Prime the state with the last 'period' values
// This ensures that subsequent calls to Update(TValue) work correctly
// We can't just copy the last value, we need to fill the buffer
int primeStart = Math.Max(0, len - _period);
for (int i = primeStart; i < len; i++)
{
Update(source[i]);
}
return new TSeries(t, v);
}
public override void Reset()
{
_buffer.Clear();
_sumSq = 0;
_updateCount = 0;
Last = default;
}
private void Resync()
{
var span = _buffer.GetSpan();
_sumSq = span.DotProduct(span);
_buffer.RecalculateSum();
}
public override void Prime(ReadOnlySpan<double> source)
{
foreach (double value in source)
{
Update(new TValue(DateTime.UtcNow, value));
}
}
public static TSeries Calculate(TSeries source, int period, bool isPopulation = false)
{
var variance = new Variance(period, isPopulation);
return variance.Update(source);
}
/// <summary>
/// Calculates Variance in-place, writing results to pre-allocated output span.
/// Zero-allocation method for maximum performance.
/// Uses SIMD acceleration for large, clean datasets.
/// </summary>
/// <param name="source">Input values</param>
/// <param name="output">Output span (must be same length as source)</param>
/// <param name="period">Variance period (must be >= 2)</param>
/// <param name="isPopulation">If true, calculates Population Variance (div by N). If false, Sample Variance (div by N-1).</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Batch(ReadOnlySpan<double> source, Span<double> output, int period, bool isPopulation = false)
{
if (source.Length != output.Length)
throw new ArgumentException("Source and output must have the same length");
if (period < 2)
throw new ArgumentException("Period must be greater than or equal to 2", nameof(period));
int len = source.Length;
if (len == 0) return;
// Try SIMD path for large, clean datasets
const int SimdThreshold = 256;
if (len >= SimdThreshold && !source.ContainsNonFinite())
{
if (Avx512F.IsSupported)
{
CalculateAvx512Core(source, output, period, isPopulation);
return;
}
if (Avx2.IsSupported)
{
CalculateAvx2Core(source, output, period, isPopulation);
return;
}
if (AdvSimd.Arm64.IsSupported)
{
CalculateNeonCore(source, output, period, isPopulation);
return;
}
}
// Scalar path with NaN handling
CalculateScalarCore(source, output, period, isPopulation);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void CalculateScalarCore(ReadOnlySpan<double> source, Span<double> output, int period, bool isPopulation)
{
int len = source.Length;
double sum = 0;
double sumSq = 0;
// We need a buffer to handle the sliding window removal
// For scalar path, we can use a simple array or stackalloc
const int StackAllocThreshold = 256;
Span<double> buffer = period <= StackAllocThreshold
? stackalloc double[period]
: new double[period];
int bufferIndex = 0;
int i = 0;
// Warmup phase
int warmupEnd = Math.Min(period, len);
for (; i < warmupEnd; i++)
{
double val = source[i];
if (!double.IsFinite(val)) val = 0; // Fallback
sum += val;
sumSq = Math.FusedMultiplyAdd(val, val, sumSq);
buffer[i] = val;
double n = i + 1;
if (n > 1)
{
double numerator = sumSq - (sum * sum) / n;
if (numerator < 0) numerator = 0;
double denominator = isPopulation ? n : (n - 1);
output[i] = numerator / denominator;
}
else
{
output[i] = 0;
}
}
// Sliding window phase
int tickCount = period;
for (; i < len; i++)
{
double val = source[i];
if (!double.IsFinite(val)) val = 0; // Fallback
double oldVal = buffer[bufferIndex];
sum = sum - oldVal + val;
sumSq = Math.FusedMultiplyAdd(-oldVal, oldVal, sumSq);
sumSq = Math.FusedMultiplyAdd(val, val, sumSq);
buffer[bufferIndex] = val;
bufferIndex++;
if (bufferIndex >= period) bufferIndex = 0;
double n = period;
double numerator = sumSq - (sum * sum) / n;
if (numerator < 0) numerator = 0;
double denominator = isPopulation ? n : (n - 1);
output[i] = numerator / denominator;
tickCount++;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
sum = buffer.SumSIMD();
sumSq = buffer.DotProduct(buffer);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void WarmupVariance(int period, bool isPopulation, ref double srcRef, ref double outRef, out double sum, out double sumSq)
{
sum = 0;
sumSq = 0;
for (int i = 0; i < period; i++)
{
double val = Unsafe.Add(ref srcRef, i);
sum += val;
sumSq = Math.FusedMultiplyAdd(val, val, sumSq);
double n = i + 1;
if (n > 1)
{
double num = sumSq - (sum * sum) / n;
if (num < 0) num = 0;
double den = isPopulation ? n : (n - 1);
Unsafe.Add(ref outRef, i) = num / den;
}
else
{
Unsafe.Add(ref outRef, i) = 0;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
private static void CalculateAvx512Core(ReadOnlySpan<double> source, Span<double> output, int period, bool isPopulation)
{
int len = source.Length;
const int VectorWidth = 8;
ref double srcRef = ref MemoryMarshal.GetReference(source);
ref double outRef = ref MemoryMarshal.GetReference(output);
double invN = 1.0 / period;
double invDenom = 1.0 / (isPopulation ? period : (period - 1));
WarmupVariance(period, isPopulation, ref srcRef, ref outRef, out double sum, out double sumSq);
if (len <= period) return;
var vInvN = Vector512.Create(invN);
var vInvDenom = Vector512.Create(invDenom);
var vZero = Vector512<double>.Zero;
int simdEnd = period + ((len - period) / VectorWidth) * VectorWidth;
int tickCount = period;
for (int i = period; i < simdEnd; i += VectorWidth)
{
var vNew = Vector512.LoadUnsafe(ref Unsafe.Add(ref srcRef, i));
var vOld = Vector512.LoadUnsafe(ref Unsafe.Add(ref srcRef, i - period));
// Delta for Sum
var vDelta = Avx512F.Subtract(vNew, vOld);
// Delta for SumSq
var vNewSq = Avx512F.Multiply(vNew, vNew);
var vOldSq = Avx512F.Multiply(vOld, vOld);
var vDeltaSq = Avx512F.Subtract(vNewSq, vOldSq);
// Prefix sum for Sum
var vShift1 = Vector512.Create(0.0, vDelta.GetElement(0), vDelta.GetElement(1), vDelta.GetElement(2), vDelta.GetElement(3), vDelta.GetElement(4), vDelta.GetElement(5), vDelta.GetElement(6));
var vP1 = Avx512F.Add(vDelta, vShift1);
var vShift2 = Vector512.Create(0.0, 0.0, vP1.GetElement(0), vP1.GetElement(1), vP1.GetElement(2), vP1.GetElement(3), vP1.GetElement(4), vP1.GetElement(5));
var vP2 = Avx512F.Add(vP1, vShift2);
var vShift4 = Vector512.Create(0.0, 0.0, 0.0, 0.0, vP2.GetElement(0), vP2.GetElement(1), vP2.GetElement(2), vP2.GetElement(3));
var vP4 = Avx512F.Add(vP2, vShift4);
var vSumPrev = Vector512.Create(sum);
var vSums = Avx512F.Add(vSumPrev, vP4);
// Prefix sum for SumSq
var vShiftSq1 = Vector512.Create(0.0, vDeltaSq.GetElement(0), vDeltaSq.GetElement(1), vDeltaSq.GetElement(2), vDeltaSq.GetElement(3), vDeltaSq.GetElement(4), vDeltaSq.GetElement(5), vDeltaSq.GetElement(6));
var vP1Sq = Avx512F.Add(vDeltaSq, vShiftSq1);
var vShiftSq2 = Vector512.Create(0.0, 0.0, vP1Sq.GetElement(0), vP1Sq.GetElement(1), vP1Sq.GetElement(2), vP1Sq.GetElement(3), vP1Sq.GetElement(4), vP1Sq.GetElement(5));
var vP2Sq = Avx512F.Add(vP1Sq, vShiftSq2);
var vShiftSq4 = Vector512.Create(0.0, 0.0, 0.0, 0.0, vP2Sq.GetElement(0), vP2Sq.GetElement(1), vP2Sq.GetElement(2), vP2Sq.GetElement(3));
var vP4Sq = Avx512F.Add(vP2Sq, vShiftSq4);
var vSumSqPrev = Vector512.Create(sumSq);
var vSumSqs = Avx512F.Add(vSumSqPrev, vP4Sq);
// Calculate Variance
var vSumSquared = Avx512F.Multiply(vSums, vSums);
var vMeanTerm = Avx512F.Multiply(vSumSquared, vInvN);
var vNumerator = Avx512F.Subtract(vSumSqs, vMeanTerm);
vNumerator = Avx512F.Max(vZero, vNumerator);
var vResult = Avx512F.Multiply(vNumerator, vInvDenom);
Vector512.StoreUnsafe(vResult, ref Unsafe.Add(ref outRef, i));
sum = vSums.GetElement(7);
sumSq = vSumSqs.GetElement(7);
tickCount += VectorWidth;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
int lastIdx = i + VectorWidth - 1;
double recalcSum = 0;
double recalcSumSq = 0;
int startIdx = lastIdx - period + 1;
for (int k = 0; k < period; k++)
{
double v = Unsafe.Add(ref srcRef, startIdx + k);
recalcSum += v;
recalcSumSq += v * v;
}
sum = recalcSum;
sumSq = recalcSumSq;
}
}
for (int i = simdEnd; i < len; i++)
{
double val = Unsafe.Add(ref srcRef, i);
double oldVal = Unsafe.Add(ref srcRef, i - period);
sum = sum - oldVal + val;
sumSq = Math.FusedMultiplyAdd(-oldVal, oldVal, sumSq);
sumSq = Math.FusedMultiplyAdd(val, val, sumSq);
double numerator = sumSq - (sum * sum) * invN;
if (numerator < 0) numerator = 0;
Unsafe.Add(ref outRef, i) = numerator * invDenom;
}
}
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
private static void CalculateNeonCore(ReadOnlySpan<double> source, Span<double> output, int period, bool isPopulation)
{
int len = source.Length;
const int VectorWidth = 2;
ref double srcRef = ref MemoryMarshal.GetReference(source);
ref double outRef = ref MemoryMarshal.GetReference(output);
double invN = 1.0 / period;
double invDenom = 1.0 / (isPopulation ? period : (period - 1));
WarmupVariance(period, isPopulation, ref srcRef, ref outRef, out double sum, out double sumSq);
if (len <= period) return;
var vInvN = Vector128.Create(invN);
var vInvDenom = Vector128.Create(invDenom);
var vZero = Vector128<double>.Zero;
int simdEnd = period + ((len - period) / VectorWidth) * VectorWidth;
int tickCount = period;
for (int i = period; i < simdEnd; i += VectorWidth)
{
var vNew = Vector128.LoadUnsafe(ref Unsafe.Add(ref srcRef, i));
var vOld = Vector128.LoadUnsafe(ref Unsafe.Add(ref srcRef, i - period));
// Delta for Sum
var vDelta = AdvSimd.Arm64.Subtract(vNew, vOld);
// Delta for SumSq
var vNewSq = AdvSimd.Arm64.Multiply(vNew, vNew);
var vOldSq = AdvSimd.Arm64.Multiply(vOld, vOld);
var vDeltaSq = AdvSimd.Arm64.Subtract(vNewSq, vOldSq);
// Prefix sum for Sum: [d0, d0+d1]
double d0 = vDelta.GetElement(0);
double d1 = vDelta.GetElement(1);
double ps0 = sum + d0;
double ps1 = ps0 + d1;
var vSums = Vector128.Create(ps0, ps1);
// Prefix sum for SumSq
double dSq0 = vDeltaSq.GetElement(0);
double dSq1 = vDeltaSq.GetElement(1);
double psSq0 = sumSq + dSq0;
double psSq1 = psSq0 + dSq1;
var vSumSqs = Vector128.Create(psSq0, psSq1);
// Calculate Variance
var vSumSquared = AdvSimd.Arm64.Multiply(vSums, vSums);
var vMeanTerm = AdvSimd.Arm64.Multiply(vSumSquared, vInvN);
var vNumerator = AdvSimd.Arm64.Subtract(vSumSqs, vMeanTerm);
vNumerator = AdvSimd.Arm64.Max(vZero, vNumerator);
var vResult = AdvSimd.Arm64.Multiply(vNumerator, vInvDenom);
Vector128.StoreUnsafe(vResult, ref Unsafe.Add(ref outRef, i));
sum = ps1;
sumSq = psSq1;
tickCount += VectorWidth;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
int lastIdx = i + VectorWidth - 1;
double recalcSum = 0;
double recalcSumSq = 0;
int startIdx = lastIdx - period + 1;
for (int k = 0; k < period; k++)
{
double v = Unsafe.Add(ref srcRef, startIdx + k);
recalcSum += v;
recalcSumSq += v * v;
}
sum = recalcSum;
sumSq = recalcSumSq;
}
}
for (int i = simdEnd; i < len; i++)
{
double val = Unsafe.Add(ref srcRef, i);
double oldVal = Unsafe.Add(ref srcRef, i - period);
sum = sum - oldVal + val;
sumSq = Math.FusedMultiplyAdd(-oldVal, oldVal, sumSq);
sumSq = Math.FusedMultiplyAdd(val, val, sumSq);
double numerator = sumSq - (sum * sum) * invN;
if (numerator < 0) numerator = 0;
Unsafe.Add(ref outRef, i) = numerator * invDenom;
}
}
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
private static void CalculateAvx2Core(ReadOnlySpan<double> source, Span<double> output, int period, bool isPopulation)
{
int len = source.Length;
const int VectorWidth = 4;
ref double srcRef = ref MemoryMarshal.GetReference(source);
ref double outRef = ref MemoryMarshal.GetReference(output);
double invN = 1.0 / period;
double invDenom = 1.0 / (isPopulation ? period : (period - 1));
WarmupVariance(period, isPopulation, ref srcRef, ref outRef, out double sum, out double sumSq);
if (len <= period) return;
var vInvN = Vector256.Create(invN);
var vInvDenom = Vector256.Create(invDenom);
var vZero = Vector256<double>.Zero;
int simdEnd = period + ((len - period) / VectorWidth) * VectorWidth;
int tickCount = period;
for (int i = period; i < simdEnd; i += VectorWidth)
{
var vNew = Vector256.LoadUnsafe(ref Unsafe.Add(ref srcRef, i));
var vOld = Vector256.LoadUnsafe(ref Unsafe.Add(ref srcRef, i - period));
// Delta for Sum
var vDelta = Avx.Subtract(vNew, vOld);
// Delta for SumSq
var vNewSq = Avx.Multiply(vNew, vNew);
var vOldSq = Avx.Multiply(vOld, vOld);
var vDeltaSq = Avx.Subtract(vNewSq, vOldSq);
// Prefix sum for Sum (same as Sma.cs)
// Prefix sum on deltas to compute 4 variance values simultaneously:
// Each lane accumulates deltas from all previous lanes within the vector.
// Lane 0: Δ₀ (window ending at i)
// Lane 1: Δ₀+Δ₁ (window ending at i+1)
// Lane 2: Δ₀+Δ₁+Δ₂ (window ending at i+2)
// Lane 3: Δ₀+Δ₁+Δ₂+Δ₃ (window ending at i+3)
var vShift1 = Avx2.Permute4x64(vDelta.AsUInt64(), 0b_10_01_00_00).AsDouble(); // skipcq: CS-R1131
vShift1 = Avx.Blend(vZero, vShift1, 0b_1110);
var vP1 = Avx.Add(vDelta, vShift1);
var vShift2 = Avx2.Permute4x64(vP1.AsUInt64(), 0b_01_00_00_00).AsDouble(); // skipcq: CS-R1131
vShift2 = Avx.Blend(vZero, vShift2, 0b_1100);
var vP2 = Avx.Add(vP1, vShift2);
var vSumPrev = Vector256.Create(sum);
var vSums = Avx.Add(vSumPrev, vP2);
// Prefix sum for SumSq
var vShiftSq1 = Avx2.Permute4x64(vDeltaSq.AsUInt64(), 0b_10_01_00_00).AsDouble(); // skipcq: CS-R1131
vShiftSq1 = Avx.Blend(vZero, vShiftSq1, 0b_1110);
var vP1Sq = Avx.Add(vDeltaSq, vShiftSq1);
var vShiftSq2 = Avx2.Permute4x64(vP1Sq.AsUInt64(), 0b_01_00_00_00).AsDouble(); // skipcq: CS-R1131
vShiftSq2 = Avx.Blend(vZero, vShiftSq2, 0b_1100);
var vP2Sq = Avx.Add(vP1Sq, vShiftSq2);
var vSumSqPrev = Vector256.Create(sumSq);
var vSumSqs = Avx.Add(vSumSqPrev, vP2Sq);
// Calculate Variance
// Var = (SumSq - (Sum*Sum)/N) / Denom
var vSumSquared = Avx.Multiply(vSums, vSums);
var vMeanTerm = Avx.Multiply(vSumSquared, vInvN);
var vNumerator = Avx.Subtract(vSumSqs, vMeanTerm);
// Max(0, numerator) to handle floating point noise
vNumerator = Avx.Max(vZero, vNumerator);
var vResult = Avx.Multiply(vNumerator, vInvDenom);
Vector256.StoreUnsafe(vResult, ref Unsafe.Add(ref outRef, i));
// Update scalar accumulators for next iteration
sum = vSums.GetElement(3);
sumSq = vSumSqs.GetElement(3);
tickCount += VectorWidth;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
int lastIdx = i + VectorWidth - 1;
double recalcSum = 0;
double recalcSumSq = 0;
int startIdx = lastIdx - period + 1;
for (int k = 0; k < period; k++)
{
double v = Unsafe.Add(ref srcRef, startIdx + k);
recalcSum += v;
recalcSumSq += v * v;
}
sum = recalcSum;
sumSq = recalcSumSq;
}
}
// Handle remaining elements
for (int i = simdEnd; i < len; i++)
{
double val = Unsafe.Add(ref srcRef, i);
double oldVal = Unsafe.Add(ref srcRef, i - period);
sum = sum - oldVal + val;
sumSq = Math.FusedMultiplyAdd(-oldVal, oldVal, sumSq);
sumSq = Math.FusedMultiplyAdd(val, val, sumSq);
double numerator = sumSq - (sum * sum) * invN;
if (numerator < 0) numerator = 0;
Unsafe.Add(ref outRef, i) = numerator * invDenom;
}
}
}
+81
View File
@@ -0,0 +1,81 @@
# Variance (VAR)
> "Volatility is the price of admission for high returns."
Variance measures how far a set of numbers is spread out from their average value. In finance, it is a key measure of volatility and risk.
## Historical Context
Variance is a fundamental concept in statistics, formalized by Ronald Fisher in 1918. In finance, it gained prominence with Modern Portfolio Theory (Markowitz, 1952), where it serves as the standard measure of risk.
## Architecture & Physics
The Variance indicator uses a sliding window (RingBuffer) to maintain the last `N` data points. It calculates the variance using an O(1) running sum of squares algorithm, ensuring constant time complexity regardless of the period length.
### O(1) Calculation
The algorithm maintains two running sums:
1. Sum of values ($\sum x$)
2. Sum of squared values ($\sum x^2$)
When a new value enters and an old value leaves:
$$ \sum x_{new} = \sum x_{old} - x_{out} + x_{in} $$
$$ \sum x^2_{new} = \sum x^2_{old} - x^2_{out} + x^2_{in} $$
This avoids iterating over the entire window for each update.
## Mathematical Foundation
Variance ($\sigma^2$ or $s^2$) is defined as:
### Population Variance (N)
$$ \sigma^2 = \frac{\sum_{i=1}^{N} (x_i - \mu)^2}{N} $$
Using the computational formula:
$$ \sigma^2 = \frac{\sum x^2 - \frac{(\sum x)^2}{N}}{N} $$
### Sample Variance (N-1)
$$ s^2 = \frac{\sum_{i=1}^{N} (x_i - \bar{x})^2}{N-1} $$
Using the computational formula:
$$ s^2 = \frac{\sum x^2 - \frac{(\sum x)^2}{N}}{N-1} $$
Where:
* $N$ is the period.
* $\mu$ or $\bar{x}$ is the mean.
## Performance Profile
| Metric | Score | Notes |
| :--- | :--- | :--- |
| **Throughput** | 5 ns/bar | O(1) complexity using running sums. |
| **Allocations** | 0 | Zero-allocation in hot path. |
| **Complexity** | O(1) | Constant time update. |
| **Accuracy** | 9 | High accuracy, though running sums can accumulate floating point errors over very long periods (mitigated by periodic resync if needed, though not strictly implemented here as window is finite). |
## Validation
| Library | Status | Notes |
| :--- | :--- | :--- |
| **Skender** | ✅ | Matches `StdDev^2` (Sample Variance). |
| **TA-Lib** | ✅ | Matches `VAR` (Population Variance usually, check specific implementation). |
## Usage
```csharp
using QuanTAlib;
// Create a 20-period Sample Variance indicator
var variance = new Variance(20, isPopulation: false);
// Update with a new value
var result = variance.Update(new TValue(DateTime.UtcNow, 100.0));
// Access the last calculated value
Console.WriteLine($"Variance: {variance.Last.Value}");