Add R² and SMAPE error metrics with comprehensive tests and documentation

- Introduced R² (Coefficient of Determination) metric with detailed mathematical foundation, performance profile, and usage examples.
- Implemented SMAPE (Symmetric Mean Absolute Percentage Error) metric, addressing asymmetry in MAPE with symmetric error calculations.
- Added unit tests for SMAPE covering various scenarios including edge cases and input validation.
- Enhanced Dema class to correctly handle event publishing with isNew parameter.
- Updated Quantower test project to include coverage configuration for better test reporting.
This commit is contained in:
Miha Kralj
2025-12-29 20:58:21 -08:00
parent 4dbb093892
commit bf611d319f
50 changed files with 11327 additions and 21 deletions
+389
View File
@@ -0,0 +1,389 @@
namespace QuanTAlib.Tests;
public class MapeTests
{
[Fact]
public void Constructor_ValidatesInput()
{
Assert.Throws<ArgumentException>(() => new Mape(0));
Assert.Throws<ArgumentException>(() => new Mape(-1));
var mape = new Mape(10);
Assert.NotNull(mape);
}
[Fact]
public void Properties_Accessible()
{
var mape = new Mape(10);
Assert.Equal(0, mape.Last.Value);
Assert.False(mape.IsHot);
Assert.Contains("Mape", mape.Name, StringComparison.Ordinal);
mape.Update(100, 105);
Assert.NotEqual(0, mape.Last.Time);
}
[Fact]
public void IsHot_BecomesTrueWhenBufferFull()
{
int period = 5;
var mape = new Mape(period);
for (int i = 0; i < period - 1; i++)
{
Assert.False(mape.IsHot, $"IsHot should be false at index {i}");
mape.Update(100 + i, 105 + i);
}
mape.Update(104, 109);
Assert.True(mape.IsHot, "IsHot should be true after period updates");
}
[Fact]
public void Mape_CalculatesCorrectly()
{
var mape = new Mape(3);
// |100 - 110| / 100 * 100 = 10%
var res1 = mape.Update(100, 110);
Assert.Equal(10.0, res1.Value, 10);
// |200 - 220| / 200 * 100 = 10%, Mean = (10 + 10) / 2 = 10%
var res2 = mape.Update(200, 220);
Assert.Equal(10.0, res2.Value, 10);
// |50 - 60| / 50 * 100 = 20%, Mean = (10 + 10 + 20) / 3 = 13.333%
var res3 = mape.Update(50, 60);
Assert.Equal(40.0 / 3.0, res3.Value, 10);
}
[Fact]
public void Mape_PerfectPrediction_ReturnsZero()
{
var mape = new Mape(5);
for (int i = 1; i <= 10; i++)
{
mape.Update(i * 10, i * 10); // Perfect prediction
}
Assert.Equal(0.0, mape.Last.Value, 10);
}
[Fact]
public void Mape_ConstantPercentageError()
{
var mape = new Mape(5);
// 10% error consistently
for (int i = 1; i <= 10; i++)
{
mape.Update(100, 110); // |100-110|/100 * 100 = 10%
}
Assert.Equal(10.0, mape.Last.Value, 10);
}
[Fact]
public void Mape_ScaleIndependent()
{
var mape1 = new Mape(3);
var mape2 = new Mape(3);
// Small scale: 10% error
mape1.Update(10, 11);
mape1.Update(10, 11);
mape1.Update(10, 11);
// Large scale: 10% error
mape2.Update(1000, 1100);
mape2.Update(1000, 1100);
mape2.Update(1000, 1100);
Assert.Equal(mape1.Last.Value, mape2.Last.Value, 10);
}
[Fact]
public void Calc_IsNew_AcceptsParameter()
{
var mape = new Mape(10);
mape.Update(100, 110, isNew: true);
double value1 = mape.Last.Value;
mape.Update(100, 120, isNew: true);
double value2 = mape.Last.Value;
Assert.NotEqual(value1, value2);
}
[Fact]
public void Calc_IsNew_False_UpdatesValue()
{
var mape = new Mape(10);
mape.Update(100, 110);
mape.Update(100, 120, isNew: true);
double beforeUpdate = mape.Last.Value;
mape.Update(100, 130, isNew: false);
double afterUpdate = mape.Last.Value;
Assert.NotEqual(beforeUpdate, afterUpdate);
}
[Fact]
public void IterativeCorrections_RestoreToOriginalState()
{
var mape = new Mape(5);
double tenthActual = 0;
double tenthPredicted = 0;
// Feed 10 updates
for (int i = 1; i <= 10; i++)
{
tenthActual = i * 10;
tenthPredicted = i * 10 + 5;
mape.Update(tenthActual, tenthPredicted);
}
double stateAfterTen = mape.Last.Value;
// Apply 5 corrections with isNew=false
for (int i = 0; i < 5; i++)
{
mape.Update(100 + i, 200 + i, isNew: false);
}
// Restore to original values
mape.Update(tenthActual, tenthPredicted, isNew: false);
Assert.Equal(stateAfterTen, mape.Last.Value, 10);
}
[Fact]
public void Reset_ClearsState()
{
var mape = new Mape(5);
for (int i = 1; i <= 10; i++)
{
mape.Update(i * 10, i * 10 + 5);
}
Assert.True(mape.IsHot);
mape.Reset();
Assert.False(mape.IsHot);
Assert.Equal(0, mape.Last.Value);
}
[Fact]
public void NaN_Input_UsesLastValidValue()
{
var mape = new Mape(5);
mape.Update(100, 110);
mape.Update(110, 120);
mape.Update(120, 130);
var result = mape.Update(double.NaN, double.NaN);
Assert.True(double.IsFinite(result.Value));
}
[Fact]
public void Infinity_Input_UsesLastValidValue()
{
var mape = new Mape(5);
mape.Update(100, 110);
mape.Update(110, 120);
var result = mape.Update(double.PositiveInfinity, double.NegativeInfinity);
Assert.True(double.IsFinite(result.Value));
}
[Fact]
public void MultipleNaN_ContinuesWithLastValid()
{
var mape = new Mape(5);
mape.Update(100, 110);
mape.Update(110, 120);
mape.Update(120, 130);
var r1 = mape.Update(double.NaN, double.NaN);
var r2 = mape.Update(double.NaN, double.NaN);
var r3 = mape.Update(double.NaN, double.NaN);
Assert.True(double.IsFinite(r1.Value));
Assert.True(double.IsFinite(r2.Value));
Assert.True(double.IsFinite(r3.Value));
}
[Fact]
public void Mape_Throws_On_Single_Input()
{
var mape = new Mape(10);
Assert.Throws<NotSupportedException>(() => mape.Update(new TValue(DateTime.UtcNow, 1)));
Assert.Throws<NotSupportedException>(() => mape.Update(new TSeries()));
Assert.Throws<NotSupportedException>(() => mape.Prime(new double[] { 1, 2, 3 }));
}
[Fact]
public void BatchSpan_MatchesStreaming()
{
int period = 5;
int count = 100;
var gbm = new GBM(startPrice: 100, mu: 0.05, sigma: 0.2, seed: 123);
double[] actual = new double[count];
double[] predicted = new double[count];
for (int i = 0; i < count; i++)
{
var bar = gbm.Next();
actual[i] = bar.Close;
predicted[i] = bar.Close * 1.05 + 2; // Offset prediction
}
// Streaming
var mape = new Mape(period);
var streamingResults = new double[count];
for (int i = 0; i < count; i++)
{
streamingResults[i] = mape.Update(actual[i], predicted[i]).Value;
}
// Batch
double[] batchResults = new double[count];
Mape.Batch(actual, predicted, batchResults, period);
// Compare
for (int i = 0; i < count; i++)
{
Assert.Equal(streamingResults[i], batchResults[i], 9);
}
}
[Fact]
public void BatchSpan_ValidatesInput()
{
double[] actual = [1, 2, 3, 4, 5];
double[] predicted = [1, 2, 3, 4, 5];
double[] output = new double[5];
double[] wrongSizeOutput = new double[3];
double[] wrongSizePredicted = new double[3];
// Period must be > 0
Assert.Throws<ArgumentException>(() =>
Mape.Batch(actual.AsSpan(), predicted.AsSpan(), output.AsSpan(), 0));
Assert.Throws<ArgumentException>(() =>
Mape.Batch(actual.AsSpan(), predicted.AsSpan(), output.AsSpan(), -1));
// Output must be same length as source
Assert.Throws<ArgumentException>(() =>
Mape.Batch(actual.AsSpan(), predicted.AsSpan(), wrongSizeOutput.AsSpan(), 3));
// Predicted must be same length as actual
Assert.Throws<ArgumentException>(() =>
Mape.Batch(actual.AsSpan(), wrongSizePredicted.AsSpan(), output.AsSpan(), 3));
}
[Fact]
public void Calculate_Works()
{
var actual = new TSeries();
var predicted = new TSeries();
var now = DateTime.UtcNow;
for (int i = 1; i <= 10; i++)
{
actual.Add(now.AddMinutes(i), 100);
predicted.Add(now.AddMinutes(i), 110); // 10% error
}
var results = Mape.Calculate(actual, predicted, 3);
Assert.Equal(10, results.Count);
Assert.Equal(10.0, results.Last.Value, 10);
}
[Fact]
public void Calculate_ValidatesMismatchedLengths()
{
var actual = new TSeries();
var predicted = new TSeries();
for (int i = 0; i < 10; i++) actual.Add(DateTime.UtcNow, i + 1);
for (int i = 0; i < 5; i++) predicted.Add(DateTime.UtcNow, i + 1);
Assert.Throws<ArgumentException>(() => Mape.Calculate(actual, predicted, 3));
}
[Fact]
public void BatchSpan_HandlesNaN()
{
double[] actual = [100, 110, double.NaN, 130, 140];
double[] predicted = [105, 115, 125, double.NaN, 145];
double[] output = new double[5];
Mape.Batch(actual, predicted, output, 3);
foreach (var val in output)
{
Assert.True(double.IsFinite(val), $"Expected finite value but got {val}");
}
}
[Fact]
public void Mape_Resync_Works()
{
var mape = new Mape(5);
// Force many updates to trigger resync (ResyncInterval = 1000)
for (int i = 0; i < 1100; i++)
{
mape.Update(100, 110); // 10% error
}
// After resync, result should still be correct
Assert.Equal(10.0, mape.Last.Value, 10);
}
[Fact]
public void Mape_ZeroActual_HandlesGracefully()
{
var mape = new Mape(3);
mape.Update(100, 110);
mape.Update(100, 110);
// Zero actual should not cause division by zero
var result = mape.Update(0, 10);
Assert.True(double.IsFinite(result.Value));
}
[Fact]
public void Mape_Asymmetric_PenalizesUnderPredictionMore()
{
var mape1 = new Mape(1);
var mape2 = new Mape(1);
// Under-prediction: actual=100, predicted=50
// |100-50|/100 * 100 = 50%
var underPrediction = mape1.Update(100, 50);
// Over-prediction: actual=50, predicted=100
// |50-100|/50 * 100 = 100%
var overPrediction = mape2.Update(50, 100);
// Over-prediction should have higher MAPE due to smaller denominator
Assert.True(overPrediction.Value > underPrediction.Value);
}
}
+225
View File
@@ -0,0 +1,225 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace QuanTAlib;
/// <summary>
/// MAPE: Mean Absolute Percentage Error
/// </summary>
/// <remarks>
/// MAPE measures the average absolute percentage error between actual and predicted values.
/// It expresses accuracy as a percentage, making it scale-independent.
///
/// Formula:
/// MAPE = (100/n) * Σ|((actual - predicted) / actual)|
///
/// Key properties:
/// - Scale-independent (expressed as percentage)
/// - Cannot be calculated when actual = 0
/// - Asymmetric: penalizes under-predictions more than over-predictions
/// - Undefined for zero actual values
/// </remarks>
[SkipLocalsInit]
public sealed class Mape : AbstractBase
{
private readonly RingBuffer _buffer;
[StructLayout(LayoutKind.Auto)]
private record struct State(double Sum, double LastValidActual, double LastValidPredicted, int TickCount);
private State _state;
private State _p_state;
private const int ResyncInterval = 1000;
public Mape(int period)
{
if (period <= 0)
throw new ArgumentException("Period must be greater than 0", nameof(period));
_buffer = new RingBuffer(period);
Name = $"Mape({period})";
WarmupPeriod = period;
}
public override bool IsHot => _buffer.IsFull;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TValue Update(TValue actual, TValue predicted, bool isNew = true)
{
double actualVal = actual.Value;
double predictedVal = predicted.Value;
if (!double.IsFinite(actualVal))
actualVal = double.IsFinite(_state.LastValidActual) ? _state.LastValidActual : 1.0; // Avoid division by zero
else
_state.LastValidActual = actualVal;
if (!double.IsFinite(predictedVal))
predictedVal = double.IsFinite(_state.LastValidPredicted) ? _state.LastValidPredicted : 0.0;
else
_state.LastValidPredicted = predictedVal;
// Avoid division by zero - use small epsilon if actual is zero
double divisor = Math.Abs(actualVal) < 1e-10 ? 1e-10 : actualVal;
double percentageError = 100.0 * Math.Abs((actualVal - predictedVal) / divisor);
if (isNew)
{
_p_state = _state;
double removedValue = _buffer.Count == _buffer.Capacity ? _buffer.Oldest : 0.0;
_state.Sum = _state.Sum - removedValue + percentageError;
_buffer.Add(percentageError);
_state.TickCount++;
if (_buffer.IsFull && _state.TickCount >= ResyncInterval)
{
_state.TickCount = 0;
_state.Sum = _buffer.RecalculateSum();
}
}
else
{
_state = _p_state;
double removedValue = _buffer.Count == _buffer.Capacity ? _buffer.Oldest : 0.0;
_state.Sum = _state.Sum - removedValue + percentageError;
_buffer.UpdateNewest(percentageError);
_state.Sum = _buffer.RecalculateSum();
}
double result = _buffer.Count > 0 ? _state.Sum / _buffer.Count : percentageError;
Last = new TValue(actual.Time, result);
PubEvent(Last, isNew);
return Last;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public TValue Update(double actual, double predicted, bool isNew = true)
{
return Update(new TValue(DateTime.UtcNow, actual), new TValue(DateTime.UtcNow, predicted), isNew);
}
public override TValue Update(TValue input, bool isNew = true)
{
throw new NotSupportedException("MAPE requires two inputs. Use Update(actual, predicted).");
}
public override TSeries Update(TSeries source)
{
throw new NotSupportedException("MAPE requires two inputs. Use Calculate(actualSeries, predictedSeries, period).");
}
public override void Prime(ReadOnlySpan<double> source, TimeSpan? step = null)
{
throw new NotSupportedException("MAPE requires two inputs.");
}
public override void Reset()
{
_buffer.Clear();
_state = default;
_p_state = default;
Last = default;
}
public static TSeries Calculate(TSeries actual, TSeries predicted, int period)
{
if (actual.Count != predicted.Count)
throw new ArgumentException("Actual and predicted series must have the same length", nameof(predicted));
int len = actual.Count;
var t = new List<long>(len);
var v = new List<double>(len);
CollectionsMarshal.SetCount(t, len);
CollectionsMarshal.SetCount(v, len);
var tSpan = CollectionsMarshal.AsSpan(t);
var vSpan = CollectionsMarshal.AsSpan(v);
Batch(actual.Values, predicted.Values, vSpan, period);
actual.Times.CopyTo(tSpan);
return new TSeries(t, v);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Batch(ReadOnlySpan<double> actual, ReadOnlySpan<double> predicted, Span<double> output, int period)
{
if (actual.Length != predicted.Length || actual.Length != output.Length)
throw new ArgumentException("All spans must have the same length", nameof(output));
if (period <= 0)
throw new ArgumentException("Period must be greater than 0", nameof(period));
int len = actual.Length;
if (len == 0) return;
const int StackAllocThreshold = 256;
Span<double> buffer = period <= StackAllocThreshold
? stackalloc double[period]
: new double[period];
double sum = 0;
double lastValidActual = 1.0; // Default to 1 to avoid division by zero
double lastValidPredicted = 0;
for (int k = 0; k < len; k++)
{
if (double.IsFinite(actual[k]) && Math.Abs(actual[k]) >= 1e-10) { lastValidActual = actual[k]; break; }
}
for (int k = 0; k < len; k++)
{
if (double.IsFinite(predicted[k])) { lastValidPredicted = predicted[k]; break; }
}
int bufferIndex = 0;
int i = 0;
int warmupEnd = Math.Min(period, len);
for (; i < warmupEnd; i++)
{
double act = actual[i];
double pred = predicted[i];
if (double.IsFinite(act) && Math.Abs(act) >= 1e-10) lastValidActual = act; else act = lastValidActual;
if (double.IsFinite(pred)) lastValidPredicted = pred; else pred = lastValidPredicted;
double divisor = Math.Abs(act) < 1e-10 ? 1e-10 : act;
double percentageError = 100.0 * Math.Abs((act - pred) / divisor);
sum += percentageError;
buffer[i] = percentageError;
output[i] = sum / (i + 1);
}
int tickCount = 0;
for (; i < len; i++)
{
double act = actual[i];
double pred = predicted[i];
if (double.IsFinite(act) && Math.Abs(act) >= 1e-10) lastValidActual = act; else act = lastValidActual;
if (double.IsFinite(pred)) lastValidPredicted = pred; else pred = lastValidPredicted;
double divisor = Math.Abs(act) < 1e-10 ? 1e-10 : act;
double percentageError = 100.0 * Math.Abs((act - pred) / divisor);
sum = sum - buffer[bufferIndex] + percentageError;
buffer[bufferIndex] = percentageError;
bufferIndex++;
if (bufferIndex >= period) bufferIndex = 0;
output[i] = sum / period;
tickCount++;
if (tickCount >= ResyncInterval)
{
tickCount = 0;
double recalcSum = 0;
for (int k = 0; k < period; k++) recalcSum += buffer[k];
sum = recalcSum;
}
}
}
}
+157
View File
@@ -0,0 +1,157 @@
# MAPE: Mean Absolute Percentage Error
> "The metric that lets you compare apples to oranges, as long as you don't have any zeros."
Mean Absolute Percentage Error (MAPE) measures the average absolute percentage difference between actual and predicted values. It expresses accuracy as a percentage, making it scale-independent and easy to interpret.
## Historical Context
MAPE has been widely used in forecasting and operations research since the mid-20th century. Its intuitive percentage-based interpretation makes it a favorite in business contexts where stakeholders need to understand prediction accuracy without domain expertise.
## Architecture & Physics
MAPE divides each absolute error by the actual value, converting errors to percentages. This makes it independent of the scale of the data but introduces asymmetry and problems with zero values.
### Properties
- **Scale-independent**: Expressed as percentage
- **Asymmetric**: Penalizes over-prediction more than under-prediction
- **Undefined at zero**: Cannot compute when actual value is zero
- **Non-negative**: MAPE ≥ 0, with 0 indicating perfect prediction
- **No upper bound**: Can exceed 100% for large errors
## Mathematical Foundation
### 1. Percentage Error
For each observation, calculate the absolute percentage error:
$$APE_i = 100 \times \left| \frac{y_i - \hat{y}_i}{y_i} \right|$$
Where:
- $y_i$ = actual value
- $\hat{y}_i$ = predicted value
### 2. Mean Calculation
Average the absolute percentage errors over the period:
$$MAPE = \frac{100}{n} \sum_{i=1}^{n} \left| \frac{y_i - \hat{y}_i}{y_i} \right|$$
### 3. Running Update (O(1))
QuanTAlib uses a ring buffer with running sum for O(1) updates:
$$S_{new} = S_{old} - APE_{oldest} + APE_{newest}$$
$$MAPE = \frac{S_{new}}{n}$$
## Implementation Details
### Usage Patterns
```csharp
// Streaming mode - update with each new observation
var mape = new Mape(period: 20);
var result = mape.Update(actualValue, predictedValue);
// Batch mode - calculate for entire series
var results = Mape.Calculate(actualSeries, predictedSeries, period: 20);
// Span mode - zero-allocation for high performance
Mape.Batch(actualSpan, predictedSpan, outputSpan, period: 20);
```
### Parameters
| Parameter | Type | Description |
| :--- | :--- | :--- |
| **period** | int | Lookback window for averaging (must be > 0) |
### Properties
| Property | Type | Description |
| :--- | :--- | :--- |
| **Last** | TValue | Most recent MAPE value (as percentage) |
| **IsHot** | bool | True when buffer is full |
| **Name** | string | Indicator name (e.g., "Mape(20)") |
| **WarmupPeriod** | int | Number of periods before valid output |
## Performance Profile
| Metric | Score | Notes |
| :--- | :--- | :--- |
| **Throughput** | ~12 ns/bar | O(1) update complexity |
| **Allocations** | 0 | Uses pre-allocated ring buffer |
| **Complexity** | O(1) | Constant time per update |
| **Accuracy** | 10/10 | Exact calculation |
| **Timeliness** | 9/10 | No lag beyond the period |
| **Smoothness** | 7/10 | Moderate smoothing |
## Interpretation
| MAPE Range | Interpretation |
| :--- | :--- |
| **< 10%** | Highly accurate |
| **10-20%** | Good accuracy |
| **20-50%** | Reasonable accuracy |
| **> 50%** | Poor accuracy |
## The Asymmetry Problem
MAPE is asymmetric because it divides by the actual value:
```csharp
var mape1 = new Mape(1);
var mape2 = new Mape(1);
// Under-prediction: actual=100, predicted=50
// |100-50|/100 = 50%
mape1.Update(100, 50); // Returns 50%
// Over-prediction: actual=50, predicted=100
// |50-100|/50 = 100%
mape2.Update(50, 100); // Returns 100%
```
Same absolute error (50), but over-prediction shows higher MAPE.
## Comparison with Other Metrics
| Metric | Scale | Handles Zero | Symmetric |
| :--- | :--- | :--- | :--- |
| **MAPE** | Percentage | No | No |
| **MAPD** | Percentage | No | No |
| **SMAPE** | Percentage | Partially | Yes |
| **MAE** | Original units | Yes | Yes |
| **MPE** | Percentage | No | Yes (signed) |
## Common Use Cases
1. **Demand Forecasting**: Inventory and supply chain planning
2. **Sales Prediction**: Revenue forecasting accuracy
3. **Financial Modeling**: Investment return predictions
4. **Operations**: Capacity planning and scheduling
## Limitations
1. **Zero Values**: Undefined when actual = 0 (QuanTAlib uses epsilon fallback)
2. **Asymmetry**: Biases toward under-prediction
3. **Scale Sensitivity**: Low values inflate MAPE disproportionately
4. **Outlier Impact**: Single large percentage error can dominate
## Edge Cases
- **Identical Values**: Returns 0% when actual equals predicted
- **Zero Actual**: Uses epsilon (1e-10) to avoid division by zero
- **NaN Handling**: Uses last valid value substitution
- **Single Input**: Not supported (requires two series)
- **Period = 1**: Returns current percentage error
## Related Indicators
- [MAPD](../mapd/Mapd.md) - Mean Absolute Percentage Deviation (divides by predicted)
- [SMAPE](../smape/Smape.md) - Symmetric Mean Absolute Percentage Error
- [MPE](../mpe/Mpe.md) - Mean Percentage Error (signed)
- [MAE](../mae/Mae.md) - Mean Absolute Error (same units)