feat(statistics): add Variance indicator with O(1) calculation and usage example

This commit is contained in:
Miha Kralj
2025-12-25 17:18:41 -08:00
parent 9ba89812cd
commit 4ff6dc0ad9
61 changed files with 6069 additions and 99 deletions
@@ -0,0 +1,70 @@
using Xunit;
using TradingPlatform.BusinessLayer;
using QuanTAlib;
namespace QuanTAlib.Tests;
public class CovarianceIndicatorTests
{
[Fact]
public void CovarianceIndicator_Constructor_SetsDefaults()
{
var indicator = new CovarianceIndicator();
Assert.Equal(20, indicator.Period);
Assert.False(indicator.IsPopulation);
Assert.Equal(SourceType.Close, indicator.Source1);
Assert.Equal(SourceType.Open, indicator.Source2);
Assert.True(indicator.ShowColdValues);
Assert.Equal("Covariance", indicator.Name);
Assert.True(indicator.SeparateWindow);
Assert.True(indicator.OnBackGround);
}
[Fact]
public void CovarianceIndicator_MinHistoryDepths_EqualsTwo()
{
var indicator = new CovarianceIndicator { Period = 20 };
Assert.Equal(2, CovarianceIndicator.MinHistoryDepths);
IWatchlistIndicator watchlistIndicator = indicator;
Assert.Equal(2, watchlistIndicator.MinHistoryDepths);
}
[Fact]
public void CovarianceIndicator_Initialize_CreatesInternalCovariance()
{
var indicator = new CovarianceIndicator { Period = 10 };
// Initialize should not throw
indicator.Initialize();
// After init, line series should exist
Assert.Single(indicator.LinesSeries);
Assert.Equal("Covariance", indicator.LinesSeries[0].Name);
}
[Fact]
public void CovarianceIndicator_ProcessUpdate_HistoricalBar_ComputesValue()
{
var indicator = new CovarianceIndicator { Period = 5 };
indicator.Initialize();
// Add historical data
var now = DateTime.UtcNow;
// Need enough bars for Period
for (int i = 0; i < 20; i++)
{
indicator.HistoricalData.AddBar(now.AddMinutes(i), 100 + i, 110 + i, 90 + i, 105 + i);
// Process update for each bar to simulate history loading
var args = new UpdateArgs(UpdateReason.HistoricalBar);
indicator.ProcessUpdate(args);
}
// Line series should have a value
double cov = indicator.LinesSeries[0].GetValue(0);
Assert.True(double.IsFinite(cov));
}
}
@@ -0,0 +1,71 @@
using System.Drawing;
using System.Runtime.CompilerServices;
using TradingPlatform.BusinessLayer;
namespace QuanTAlib;
[SkipLocalsInit]
public sealed class CovarianceIndicator : Indicator, IWatchlistIndicator
{
[InputParameter("Period", sortIndex: 1, 1, 2000, 1, 0)]
public int Period { get; set; } = 20;
[InputParameter("Population", sortIndex: 2)]
public bool IsPopulation { get; set; } = false;
[InputParameter("Source 1", sortIndex: 3)]
public SourceType Source1 { get; set; } = SourceType.Close;
[InputParameter("Source 2", sortIndex: 4)]
public SourceType Source2 { get; set; } = SourceType.Open;
[InputParameter("Show cold values", sortIndex: 21)]
public bool ShowColdValues { get; set; } = true;
private Covariance? _cov;
private readonly LineSeries? _series;
private Func<IHistoryItem, double>? _priceSelector1;
private Func<IHistoryItem, double>? _priceSelector2;
public static int MinHistoryDepths => 2;
int IWatchlistIndicator.MinHistoryDepths => MinHistoryDepths;
public override string ShortName => $"Cov({Period})";
public override string SourceCodeLink => "https://github.com/mihakralj/QuanTAlib/blob/main/lib/statistics/covariance/Covariance.Quantower.cs";
public CovarianceIndicator()
{
OnBackGround = true;
SeparateWindow = true;
Name = "Covariance";
Description = "Measures the joint variability of two random variables.";
_series = new(name: "Covariance", color: IndicatorExtensions.Statistics, width: 2, style: LineStyle.Solid);
AddLineSeries(_series);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnInit()
{
_cov = new Covariance(Period, IsPopulation);
_priceSelector1 = Source1.GetPriceSelector();
_priceSelector2 = Source2.GetPriceSelector();
base.OnInit();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void OnUpdate(UpdateArgs args)
{
var item = this.HistoricalData[this.Count - 1, SeekOriginHistory.Begin];
double val1 = _priceSelector1!(item);
double val2 = _priceSelector2!(item);
var time = this.HistoricalData.Time();
var input1 = new TValue(time, val1);
var input2 = new TValue(time, val2);
TValue result = _cov!.Update(input1, input2, args.IsNewBar());
_series!.SetValue(result.Value, _cov.IsHot, ShowColdValues);
}
}
@@ -0,0 +1,133 @@
using System;
using System.Linq;
using Xunit;
namespace QuanTAlib.Tests;
public class CovarianceSimdTests
{
[Fact]
public void Covariance_Simd_Matches_Scalar_LargeDataset()
{
// Arrange
int count = 1000; // > 256 to trigger SIMD
int period = 20;
var r = new Random(42);
var dataX = new double[count];
var dataY = new double[count];
for (int i = 0; i < count; i++)
{
dataX[i] = r.NextDouble() * 100;
dataY[i] = r.NextDouble() * 100;
}
var sourceX = new TSeries();
sourceX.Add(dataX);
var sourceY = new TSeries();
sourceY.Add(dataY);
// Act
// This will use SIMD if available and length >= 256
var simdResult = Covariance.Calculate(sourceX, sourceY, period);
// Calculate expected using scalar loop (simulating by using small chunks or manual calc,
// but easier to just use the streaming update which is scalar)
var scalarCov = new Covariance(period);
var expectedValues = new double[count];
for (int i = 0; i < count; i++)
{
var res = scalarCov.Update(dataX[i], dataY[i]);
expectedValues[i] = res.Value;
}
// Assert
for (int i = 0; i < count; i++)
{
Assert.Equal(expectedValues[i], simdResult.Values[i], precision: 7);
}
}
[Fact]
public void Covariance_Simd_Handles_NaN_Correctly()
{
// Arrange
int count = 500;
int period = 50;
var dataX = Enumerable.Range(0, count).Select(x => (double)x).ToArray();
var dataY = Enumerable.Range(0, count).Select(x => (double)x * 2).ToArray();
// Inject NaN
dataX[300] = double.NaN;
dataY[350] = double.NaN;
var sourceX = new TSeries();
sourceX.Add(dataX);
var sourceY = new TSeries();
sourceY.Add(dataY);
// Act
// The implementation checks for ContainsNonFinite() before using SIMD.
// If NaN is present, it should fall back to Scalar.
// We want to verify that the result is correct regardless of the path taken.
var result = Covariance.Calculate(sourceX, sourceY, period);
// Assert
// Verify around the NaN values
// Index 300 has NaN in X. Covariance should handle it (likely treat as 0 or propagate last valid if logic dictates,
// but current implementation replaces non-finite with 0 in scalar core).
// Let's verify against streaming which we know uses scalar logic
// BUT: Batch implementation replaces NaN with 0, while Streaming propagates NaN.
// To compare, we must feed 0 instead of NaN to streaming.
var scalarCov = new Covariance(period);
for (int i = 0; i < count; i++)
{
double x = dataX[i];
double y = dataY[i];
if (!double.IsFinite(x)) x = 0;
if (!double.IsFinite(y)) y = 0;
var res = scalarCov.Update(x, y);
Assert.Equal(res.Value, result.Values[i], precision: 9);
}
}
[Fact]
public void Covariance_Simd_Resync_Check()
{
// Arrange
// Create a dataset large enough to trigger resync in SIMD loop (ResyncInterval = 1000)
// We need > 1000 elements processed in the SIMD loop.
// The SIMD loop starts at 'period' and goes up to 'simdEnd'.
// So we need length > period + 1000.
int period = 10;
int count = 2000;
// Use simple linear data to make verification easy
// y = 2x
var dataX = Enumerable.Range(0, count).Select(x => (double)x).ToArray();
var dataY = Enumerable.Range(0, count).Select(x => (double)x * 2).ToArray();
var sourceX = new TSeries();
sourceX.Add(dataX);
var sourceY = new TSeries();
sourceY.Add(dataY);
// Act
var result = Covariance.Calculate(sourceX, sourceY, period);
// Assert
// For y=2x, Cov(X,Y) = 2*Var(X)
// Var(X) of sequence 0,1,2... is constant for fixed period?
// For period 10: 0..9. Variance is constant.
// Var(0..9) = 9.16666... (Population) or 10.185... (Sample)?
// Let's just compare with scalar truth.
var scalarCov = new Covariance(period);
for (int i = 0; i < count; i++)
{
var res = scalarCov.Update(dataX[i], dataY[i]);
Assert.Equal(res.Value, result.Values[i], precision: 9);
}
}
}
@@ -0,0 +1,167 @@
using System;
using Xunit;
namespace QuanTAlib.Tests;
public class CovarianceTests
{
[Fact]
public void Covariance_CalculatesCorrectly()
{
// Arrange
var cov = new Covariance(3, isPopulation: false);
// Act & Assert
// 1. Add (1, 2)
// MeanX = 1, MeanY = 2
// Cov = 0 (n=1)
var res1 = cov.Update(1, 2);
Assert.Equal(0, res1.Value);
// 2. Add (2, 4)
// X: {1, 2}, Y: {2, 4}
// MeanX = 1.5, MeanY = 3
// Cov = ((1-1.5)(2-3) + (2-1.5)(4-3)) / 1
// = ((-0.5)(-1) + (0.5)(1)) / 1
// = (0.5 + 0.5) / 1 = 1
var res2 = cov.Update(2, 4);
Assert.Equal(1, res2.Value);
// 3. Add (3, 6)
// X: {1, 2, 3}, Y: {2, 4, 6}
// MeanX = 2, MeanY = 4
// Cov = ((1-2)(2-4) + (2-2)(4-4) + (3-2)(6-4)) / 2
// = ((-1)(-2) + 0 + (1)(2)) / 2
// = (2 + 2) / 2 = 2
var res3 = cov.Update(3, 6);
Assert.Equal(2, res3.Value);
// 4. Add (4, 8) -> Window slides: {2, 3, 4}, {4, 6, 8}
// MeanX = 3, MeanY = 6
// Cov = ((2-3)(4-6) + (3-3)(6-6) + (4-3)(8-6)) / 2
// = ((-1)(-2) + 0 + (1)(2)) / 2
// = (2 + 2) / 2 = 2
var res4 = cov.Update(4, 8);
Assert.Equal(2, res4.Value);
}
[Fact]
public void Covariance_Population_CalculatesCorrectly()
{
// Arrange
var cov = new Covariance(3, isPopulation: true);
// Act & Assert
cov.Update(1, 2);
cov.Update(2, 4);
// 3. Add (3, 6)
// X: {1, 2, 3}, Y: {2, 4, 6}
// MeanX = 2, MeanY = 4
// Cov = ((1-2)(2-4) + (2-2)(4-4) + (3-2)(6-4)) / 3
// = (2 + 2) / 3 = 4/3
var res3 = cov.Update(3, 6);
Assert.Equal(4.0/3.0, res3.Value, precision: 10);
}
[Fact]
public void Covariance_HandlesZeroCovariance()
{
// Arrange
var cov = new Covariance(3);
// Act
cov.Update(1, 1);
cov.Update(2, 1);
var res = cov.Update(3, 1); // Y is constant, variance Y is 0, covariance is 0
// Assert
Assert.Equal(0, res.Value);
}
[Fact]
public void Covariance_HandlesNegativeCovariance()
{
// Arrange
var cov = new Covariance(3);
// Act
cov.Update(1, 3);
cov.Update(2, 2);
var res = cov.Update(3, 1);
// X: {1, 2, 3}, MeanX = 2
// Y: {3, 2, 1}, MeanY = 2
// Cov = ((1-2)(3-2) + (2-2)(2-2) + (3-2)(1-2)) / 2
// = ((-1)(1) + 0 + (1)(-1)) / 2
// = (-1 - 1) / 2 = -1
// Assert
Assert.Equal(-1, res.Value);
}
[Fact]
public void Covariance_Resync_Works()
{
// Arrange
var cov = new Covariance(3);
// Act
// Force many updates to trigger resync (ResyncInterval = 1000)
// We can't easily force 1000 updates in a simple test without loop,
// but we can verify the logic holds for a sequence.
for (int i = 0; i < 1100; i++)
{
cov.Update(i, i * 2);
}
// Last 3: {1097, 1098, 1099}, {2194, 2196, 2198}
// This is a perfect linear relationship y = 2x
// Cov(X, 2X) = 2 * Var(X)
// Var(X) for {x-1, x, x+1} is:
// Mean = x
// SumSqDiff = (-1)^2 + 0 + 1^2 = 2
// Var = 2 / 2 = 1
// Cov = 2 * 1 = 2
// Assert
Assert.Equal(2, cov.Last.Value, precision: 10);
}
[Fact]
public void Covariance_Update_IsNew_False_Works()
{
// Arrange
var cov = new Covariance(3);
// Act
cov.Update(1, 2);
cov.Update(2, 4);
cov.Update(3, 6); // Cov = 2
// Update last bar with new values
// Change (3, 6) to (4, 8)
// X: {1, 2, 4}, MeanX = 7/3 = 2.333...
// Y: {2, 4, 8}, MeanY = 14/3 = 4.666...
// This is harder to calc manually, let's use the property that it should match adding (4, 8) directly
var res = cov.Update(4, 8, isNew: false);
var cov2 = new Covariance(3);
cov2.Update(1, 2);
cov2.Update(2, 4);
var expected = cov2.Update(4, 8);
// Assert
Assert.Equal(expected.Value, res.Value, precision: 10);
}
[Fact]
public void Covariance_Throws_On_Single_Input()
{
var cov = new Covariance(10);
Assert.Throws<NotSupportedException>(() => cov.Update(new TValue(DateTime.UtcNow, 1)));
Assert.Throws<NotSupportedException>(() => cov.Update(new TSeries()));
Assert.Throws<NotSupportedException>(() => cov.Prime(new double[] { 1, 2, 3 }));
}
}
@@ -0,0 +1,89 @@
using System;
using Xunit;
namespace QuanTAlib.Tests;
public class CovarianceValidationTests
{
[Fact]
public void Covariance_Matches_ManualCalculation()
{
// Arrange
int period = 10;
var cov = new Covariance(period, isPopulation: false);
var r = new Random(123);
double[] x = new double[100];
double[] y = new double[100];
for (int i = 0; i < 100; i++)
{
x[i] = r.NextDouble() * 100;
y[i] = r.NextDouble() * 100;
cov.Update(x[i], y[i]);
if (i >= period - 1)
{
// Manual calculation for last 'period' items
double sumX = 0;
double sumY = 0;
for (int j = 0; j < period; j++)
{
sumX += x[i - j];
sumY += y[i - j];
}
double meanX = sumX / period;
double meanY = sumY / period;
double sumProd = 0;
for (int j = 0; j < period; j++)
{
sumProd += (x[i - j] - meanX) * (y[i - j] - meanY);
}
double expected = sumProd / (period - 1);
Assert.Equal(expected, cov.Last.Value, precision: 8);
}
}
}
[Fact]
public void Covariance_Population_Matches_ManualCalculation()
{
// Arrange
int period = 10;
var cov = new Covariance(period, isPopulation: true);
var r = new Random(456);
double[] x = new double[100];
double[] y = new double[100];
for (int i = 0; i < 100; i++)
{
x[i] = r.NextDouble() * 100;
y[i] = r.NextDouble() * 100;
cov.Update(x[i], y[i]);
if (i >= period - 1)
{
// Manual calculation for last 'period' items
double sumX = 0;
double sumY = 0;
for (int j = 0; j < period; j++)
{
sumX += x[i - j];
sumY += y[i - j];
}
double meanX = sumX / period;
double meanY = sumY / period;
double sumProd = 0;
for (int j = 0; j < period; j++)
{
sumProd += (x[i - j] - meanX) * (y[i - j] - meanY);
}
double expected = sumProd / period;
Assert.Equal(expected, cov.Last.Value, precision: 8);
}
}
}
}
+468
View File
@@ -0,0 +1,468 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
namespace QuanTAlib;
/// <summary>
/// Covariance: Measures the joint variability of two random variables.
/// </summary>
/// <remarks>
/// Covariance indicates the direction of the linear relationship between variables.
/// - Positive covariance: Variables tend to move in the same direction.
/// - Negative covariance: Variables tend to move in opposite directions.
/// - Zero covariance: Variables are uncorrelated.
///
/// Formula:
/// Cov(X, Y) = Sum((x - mean(x)) * (y - mean(y))) / n (Population)
/// Cov(X, Y) = Sum((x - mean(x)) * (y - mean(y))) / (n - 1) (Sample)
///
/// This implementation uses the O(1) running sum formula:
/// Cov(X, Y) = (Sum(xy) - Sum(x)*Sum(y)/n) / n (or n-1)
/// </remarks>
[SkipLocalsInit]
public sealed class Covariance : AbstractBase
{
private readonly bool _isPopulation;
private readonly RingBuffer _bufferX;
private readonly RingBuffer _bufferY;
private double _sumX;
private double _sumY;
private double _sumXY;
private int _updateCount;
private const int ResyncInterval = 1000;
public override bool IsHot => _bufferX.IsFull;
/// <summary>
/// Creates a new Covariance indicator.
/// </summary>
/// <param name="period">The lookback period (must be >= 2).</param>
/// <param name="isPopulation">If true, calculates Population Covariance. If false, Sample Covariance (default).</param>
public Covariance(int period, bool isPopulation = false)
{
if (period < 2)
{
throw new ArgumentOutOfRangeException(nameof(period), "Period must be greater than or equal to 2.");
}
_isPopulation = isPopulation;
_bufferX = new RingBuffer(period);
_bufferY = new RingBuffer(period);
Name = $"Cov({period})";
WarmupPeriod = period;
}
/// <summary>
/// Updates the Covariance indicator with new values.
/// </summary>
/// <param name="x">The first value (TValue).</param>
/// <param name="y">The second value (TValue).</param>
/// <param name="isNew">Whether this is a new bar.</param>
/// <returns>The calculated Covariance value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TValue Update(TValue x, TValue y, bool isNew = true)
{
if (isNew)
{
if (_bufferX.IsFull)
{
double oldX = _bufferX.Oldest;
double oldY = _bufferY.Oldest;
_sumX -= oldX;
_sumY -= oldY;
_sumXY -= oldX * oldY;
}
_bufferX.Add(x.Value);
_bufferY.Add(y.Value);
double valX = x.Value;
double valY = y.Value;
_sumX += valX;
_sumY += valY;
_sumXY += valX * valY;
_updateCount++;
if (_updateCount % ResyncInterval == 0)
{
Resync();
}
}
else
{
double oldX = _bufferX.Newest;
double oldY = _bufferY.Newest;
_bufferX.UpdateNewest(x.Value);
_bufferY.UpdateNewest(y.Value);
double valX = x.Value;
double valY = y.Value;
_sumX = _sumX - oldX + valX;
_sumY = _sumY - oldY + valY;
_sumXY = _sumXY - (oldX * oldY) + (valX * valY);
}
double cov = 0;
int n = _bufferX.Count;
if (n >= 2)
{
double numerator = _sumXY - (_sumX * _sumY) / n;
double denominator = _isPopulation ? n : (n - 1);
cov = numerator / denominator;
}
Last = new TValue(x.Time, cov);
PubEvent(Last);
return Last;
}
public TValue Update(double x, double y, bool isNew = true)
{
return Update(new TValue(DateTime.UtcNow, x), new TValue(DateTime.UtcNow, y), isNew);
}
public override TValue Update(TValue input, bool isNew = true)
{
throw new NotSupportedException("Covariance requires two inputs. Use Update(x, y).");
}
public override TSeries Update(TSeries source)
{
throw new NotSupportedException("Covariance requires two inputs. Use Update(x, y).");
}
public override void Prime(ReadOnlySpan<double> source)
{
throw new NotSupportedException("Covariance requires two inputs. Use Update(x, y).");
}
public override void Reset()
{
_bufferX.Clear();
_bufferY.Clear();
_sumX = 0;
_sumY = 0;
_sumXY = 0;
_updateCount = 0;
Last = default;
}
private void Resync()
{
double sumX = 0;
double sumY = 0;
double sumXY = 0;
for (int i = 0; i < _bufferX.Count; i++)
{
double x = _bufferX[i];
double y = _bufferY[i];
sumX += x;
sumY += y;
sumXY += x * y;
}
_sumX = sumX;
_sumY = sumY;
_sumXY = sumXY;
}
public static TSeries Calculate(TSeries sourceX, TSeries sourceY, int period, bool isPopulation = false)
{
if (sourceX.Count != sourceY.Count)
throw new ArgumentException("Source series must have the same length");
int len = sourceX.Count;
var t = new List<long>(len);
var v = new List<double>(len);
CollectionsMarshal.SetCount(t, len);
CollectionsMarshal.SetCount(v, len);
var tSpan = CollectionsMarshal.AsSpan(t);
var vSpan = CollectionsMarshal.AsSpan(v);
Batch(sourceX.Values, sourceY.Values, vSpan, period, isPopulation);
sourceX.Times.CopyTo(tSpan);
return new TSeries(t, v);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Batch(ReadOnlySpan<double> sourceX, ReadOnlySpan<double> sourceY, Span<double> output, int period, bool isPopulation = false)
{
if (sourceX.Length != sourceY.Length || sourceX.Length != output.Length)
throw new ArgumentException("All spans must have the same length");
if (period < 2)
throw new ArgumentException("Period must be greater than or equal to 2", nameof(period));
int len = sourceX.Length;
if (len == 0) return;
// SIMD overhead amortizes well for datasets >= 256 elements
const int SimdThreshold = 256;
if (len >= SimdThreshold && !sourceX.ContainsNonFinite() && !sourceY.ContainsNonFinite() && Avx2.IsSupported)
{
CalculateAvx2Core(sourceX, sourceY, output, period, isPopulation);
return;
}
CalculateScalarCore(sourceX, sourceY, output, period, isPopulation);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void CalculateScalarCore(ReadOnlySpan<double> sourceX, ReadOnlySpan<double> sourceY, Span<double> output, int period, bool isPopulation)
{
int len = sourceX.Length;
double sumX = 0;
double sumY = 0;
double sumXY = 0;
const int StackAllocThreshold = 256;
Span<double> bufferX = period <= StackAllocThreshold ? stackalloc double[period] : new double[period];
Span<double> bufferY = period <= StackAllocThreshold ? stackalloc double[period] : new double[period];
int bufferIndex = 0;
int i = 0;
// Warmup
int warmupEnd = Math.Min(period, len);
for (; i < warmupEnd; i++)
{
double x = sourceX[i];
double y = sourceY[i];
if (!double.IsFinite(x)) x = 0;
if (!double.IsFinite(y)) y = 0;
sumX += x;
sumY += y;
sumXY += x * y;
bufferX[i] = x;
bufferY[i] = y;
double n = i + 1;
if (n >= 2)
{
double numerator = sumXY - (sumX * sumY) / n;
double denominator = isPopulation ? n : (n - 1);
output[i] = numerator / denominator;
}
else
{
output[i] = 0;
}
}
// Sliding window
int tickCount = period;
for (; i < len; i++)
{
double x = sourceX[i];
double y = sourceY[i];
if (!double.IsFinite(x)) x = 0;
if (!double.IsFinite(y)) y = 0;
double oldX = bufferX[bufferIndex];
double oldY = bufferY[bufferIndex];
sumX = sumX - oldX + x;
sumY = sumY - oldY + y;
sumXY = sumXY - (oldX * oldY) + (x * y);
bufferX[bufferIndex] = x;
bufferY[bufferIndex] = y;
bufferIndex++;
if (bufferIndex >= period) bufferIndex = 0;
double n = period;
double numerator = sumXY - (sumX * sumY) / n;
double denominator = isPopulation ? n : (n - 1);
output[i] = numerator / denominator;
tickCount++;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
double recalcSumX = 0;
double recalcSumY = 0;
double recalcSumXY = 0;
for (int k = 0; k < period; k++)
{
double bx = bufferX[k];
double by = bufferY[k];
recalcSumX += bx;
recalcSumY += by;
recalcSumXY += bx * by;
}
sumX = recalcSumX;
sumY = recalcSumY;
sumXY = recalcSumXY;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static (double sumX, double sumY, double sumXY) WarmupCovariance(int period, bool isPopulation, ref double srcXRef, ref double srcYRef, ref double outRef)
{
double sumX = 0;
double sumY = 0;
double sumXY = 0;
for (int i = 0; i < period; i++)
{
double x = Unsafe.Add(ref srcXRef, i);
double y = Unsafe.Add(ref srcYRef, i);
sumX += x;
sumY += y;
sumXY += x * y;
double n = i + 1;
if (n >= 2)
{
double num = sumXY - (sumX * sumY) / n;
double den = isPopulation ? n : (n - 1);
Unsafe.Add(ref outRef, i) = num / den;
}
else
{
Unsafe.Add(ref outRef, i) = 0;
}
}
return (sumX, sumY, sumXY);
}
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
private static void CalculateAvx2Core(ReadOnlySpan<double> sourceX, ReadOnlySpan<double> sourceY, Span<double> output, int period, bool isPopulation)
{
int len = sourceX.Length;
const int VectorWidth = 4;
ref double srcXRef = ref MemoryMarshal.GetReference(sourceX);
ref double srcYRef = ref MemoryMarshal.GetReference(sourceY);
ref double outRef = ref MemoryMarshal.GetReference(output);
double invN = 1.0 / period;
double invDenom = 1.0 / (isPopulation ? period : (period - 1));
(double sumX, double sumY, double sumXY) = WarmupCovariance(period, isPopulation, ref srcXRef, ref srcYRef, ref outRef);
if (len <= period) return;
var vInvN = Vector256.Create(invN);
var vInvDenom = Vector256.Create(invDenom);
var vZero = Vector256<double>.Zero;
int simdEnd = period + ((len - period) / VectorWidth) * VectorWidth;
int tickCount = period;
for (int i = period; i < simdEnd; i += VectorWidth)
{
var vNewX = Vector256.LoadUnsafe(ref Unsafe.Add(ref srcXRef, i));
var vOldX = Vector256.LoadUnsafe(ref Unsafe.Add(ref srcXRef, i - period));
var vNewY = Vector256.LoadUnsafe(ref Unsafe.Add(ref srcYRef, i));
var vOldY = Vector256.LoadUnsafe(ref Unsafe.Add(ref srcYRef, i - period));
// Delta for SumX
var vDeltaX = Avx.Subtract(vNewX, vOldX);
// Delta for SumY
var vDeltaY = Avx.Subtract(vNewY, vOldY);
// Delta for SumXY
var vNewXY = Avx.Multiply(vNewX, vNewY);
var vOldXY = Avx.Multiply(vOldX, vOldY);
var vDeltaXY = Avx.Subtract(vNewXY, vOldXY);
// Prefix sum for SumX
var vShiftX1 = Avx2.Permute4x64(vDeltaX.AsUInt64(), 0b_10_01_00_00).AsDouble(); // skipcq: CS-R1131
vShiftX1 = Avx.Blend(vZero, vShiftX1, 0b_1110);
var vP1X = Avx.Add(vDeltaX, vShiftX1);
var vShiftX2 = Avx2.Permute4x64(vP1X.AsUInt64(), 0b_01_00_00_00).AsDouble(); // skipcq: CS-R1131
vShiftX2 = Avx.Blend(vZero, vShiftX2, 0b_1100);
var vP2X = Avx.Add(vP1X, vShiftX2);
var vSumXPrev = Vector256.Create(sumX);
var vSumsX = Avx.Add(vSumXPrev, vP2X);
// Prefix sum for SumY
var vShiftY1 = Avx2.Permute4x64(vDeltaY.AsUInt64(), 0b_10_01_00_00).AsDouble(); // skipcq: CS-R1131
vShiftY1 = Avx.Blend(vZero, vShiftY1, 0b_1110);
var vP1Y = Avx.Add(vDeltaY, vShiftY1);
var vShiftY2 = Avx2.Permute4x64(vP1Y.AsUInt64(), 0b_01_00_00_00).AsDouble(); // skipcq: CS-R1131
vShiftY2 = Avx.Blend(vZero, vShiftY2, 0b_1100);
var vP2Y = Avx.Add(vP1Y, vShiftY2);
var vSumYPrev = Vector256.Create(sumY);
var vSumsY = Avx.Add(vSumYPrev, vP2Y);
// Prefix sum for SumXY
var vShiftXY1 = Avx2.Permute4x64(vDeltaXY.AsUInt64(), 0b_10_01_00_00).AsDouble(); // skipcq: CS-R1131
vShiftXY1 = Avx.Blend(vZero, vShiftXY1, 0b_1110);
var vP1XY = Avx.Add(vDeltaXY, vShiftXY1);
var vShiftXY2 = Avx2.Permute4x64(vP1XY.AsUInt64(), 0b_01_00_00_00).AsDouble(); // skipcq: CS-R1131
vShiftXY2 = Avx.Blend(vZero, vShiftXY2, 0b_1100);
var vP2XY = Avx.Add(vP1XY, vShiftXY2);
var vSumXYPrev = Vector256.Create(sumXY);
var vSumsXY = Avx.Add(vSumXYPrev, vP2XY);
// Calculate Covariance with FMA
// Cov = (SumXY - (SumX*SumY)/N) / Denom
var vSumXSumY = Avx.Multiply(vSumsX, vSumsY);
var vNumerator = Fma.IsSupported
? Fma.MultiplyAddNegated(vSumXSumY, vInvN, vSumsXY)
: Avx.Subtract(vSumsXY, Avx.Multiply(vSumXSumY, vInvN));
var vResult = Avx.Multiply(vNumerator, vInvDenom);
Vector256.StoreUnsafe(vResult, ref Unsafe.Add(ref outRef, i));
sumX = vSumsX.GetElement(3);
sumY = vSumsY.GetElement(3);
sumXY = vSumsXY.GetElement(3);
tickCount += VectorWidth;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
double recalcSumX = 0;
double recalcSumY = 0;
double recalcSumXY = 0;
int startIdx = i + VectorWidth - period;
for (int k = 0; k < period; k++)
{
double x = Unsafe.Add(ref srcXRef, startIdx + k);
double y = Unsafe.Add(ref srcYRef, startIdx + k);
recalcSumX += x;
recalcSumY += y;
recalcSumXY += x * y;
}
sumX = recalcSumX;
sumY = recalcSumY;
sumXY = recalcSumXY;
}
}
for (int i = simdEnd; i < len; i++)
{
double x = Unsafe.Add(ref srcXRef, i);
double y = Unsafe.Add(ref srcYRef, i);
if (!double.IsFinite(x)) x = 0;
if (!double.IsFinite(y)) y = 0;
double oldX = Unsafe.Add(ref srcXRef, i - period);
double oldY = Unsafe.Add(ref srcYRef, i - period);
if (!double.IsFinite(oldX)) oldX = 0;
if (!double.IsFinite(oldY)) oldY = 0;
sumX = sumX - oldX + x;
sumY = sumY - oldY + y;
sumXY = sumXY - (oldX * oldY) + (x * y);
double numerator = sumXY - (sumX * sumY) * invN;
Unsafe.Add(ref outRef, i) = numerator * invDenom;
}
}
}
+57
View File
@@ -0,0 +1,57 @@
# Covariance: Covariance
> "Correlation is just covariance normalized by standard deviation. But sometimes you want the raw, unadulterated relationship."
Covariance measures the joint variability of two random variables. It indicates the direction of the linear relationship between variables.
## Architecture & Physics
Covariance is calculated using a sliding window approach. It maintains running sums of $x$, $y$, and $xy$ to allow for $O(1)$ updates.
- **Positive Covariance**: Indicates that the two variables tend to move in the same direction.
- **Negative Covariance**: Indicates that the two variables tend to move in opposite directions.
- **Zero Covariance**: Indicates that the two variables are uncorrelated.
## Mathematical Foundation
### 1. Population Covariance
$$ Cov(X, Y) = \frac{\sum_{i=1}^{n} (x_i - \bar{x})(y_i - \bar{y})}{n} $$
### 2. Sample Covariance
$$ Cov(X, Y) = \frac{\sum_{i=1}^{n} (x_i - \bar{x})(y_i - \bar{y})}{n - 1} $$
### 3. Computational Formula (Running Sums)
$$ Cov(X, Y) = \frac{\sum xy - \frac{(\sum x)(\sum y)}{n}}{n} \quad \text{(or } n-1 \text{)} $$
## Performance Profile
| Metric | Score | Notes |
| :--- | :--- | :--- |
| **Throughput** | High | $O(1)$ updates using running sums. |
| **Allocations** | 0 | No heap allocations in hot path. |
| **Complexity** | $O(1)$ | Constant time update regardless of period. |
| **Accuracy** | High | Uses `double` precision; periodic resync prevents drift. |
## Validation
| Library | Status | Notes |
| :--- | :--- | :--- |
| **Manual** | ✅ | Verified against manual calculation. |
| **Excel** | ✅ | Matches `COVARIANCE.P` and `COVARIANCE.S`. |
## Usage
```csharp
using QuanTAlib;
// Create a Covariance indicator with period 20 (Sample Covariance by default)
var cov = new Covariance(20);
// Update with new values
cov.Update(price1, price2);
// Access the result
double result = cov.Last.Value;