rusty_backtest/features/

mod.rs

//! Advanced Microstructural Feature Engine
//!
//! High-performance implementation of microstructural features for HFT/MFT strategies.
//! Based on patterns from KRX A3B6G7 and Binance Tardis reference implementations.

pub mod asymmetry;
pub mod liquidity;
pub mod market_impact;
pub mod order_flow;
pub mod queue;
pub mod tardis_advanced;
pub mod tardis_features;
pub mod volatility;

#[cfg(target_arch = "x86_64")]
pub mod order_flow_simd;

#[cfg(target_arch = "x86_64")]
pub mod volatility_simd;

#[cfg(test)]
mod proptest_tests;

// Re-export key types
pub use asymmetry::{
    AsymmetryIndexCalculator, AsymmetryMetrics, MultiTimeframeAsymmetry, calculate_asymmetry_index,
};
pub use liquidity::{LiquidityAnalyzer, calculate_liquidity_shocks};
pub use market_impact::{KylesLambdaCalculator, MarketImpactAnalyzer, MarketImpactMetrics};
pub use order_flow::{OrderFlowAnalyzer, calculate_ofi, calculate_vpin};
pub use queue::{QueueAnalyzer, calculate_queue_imbalance};
pub use tardis_advanced::{
    HarmonicOscillator, HarmonicResult, OrderType, OrderTypeTracker, TardisAdvancedFeatures,
    TardisConfig, TardisFeatureVector,
};
pub use tardis_features::{
    AdvancedVPINCalculator, ExponentialDecayCalculator, PriceEntropyCalculator, PriceRunCalculator,
    RelativeTickVolumeCalculator, RollingPriceImpactCalculator, TradingBurstDetector,
    VolumePriceSensitivityCalculator, calculate_depth_weighted_ofi,
    calculate_liquidity_shock_ratio, calculate_multi_level_ofi_detailed,
    calculate_multi_level_queue_imbalance, calculate_order_book_depth, calculate_order_book_slope,
    calculate_order_cancel_rate, calculate_relative_spread, calculate_volume_weighted_ofi,
    calculate_weighted_order_imbalance,
};
pub use volatility::{VolatilityEstimator, calculate_realized_volatility};

use rust_decimal::Decimal;
use rust_decimal::prelude::ToPrimitive;
use rusty_common::collections::FxHashMap;
use smallvec::SmallVec;

/// Error type for feature calculation
#[derive(Debug, Clone)]
pub enum FeatureError {
    /// Decimal conversion failed
    DecimalConversion(&'static str),
    /// Invalid input data
    InvalidInput(&'static str),
    /// Insufficient data for calculation
    InsufficientData,
}

impl std::fmt::Display for FeatureError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            FeatureError::DecimalConversion(msg) => write!(f, "Decimal conversion error: {msg}"),
            FeatureError::InvalidInput(msg) => write!(f, "Invalid input: {msg}"),
            FeatureError::InsufficientData => write!(f, "Insufficient data for calculation"),
        }
    }
}

impl std::error::Error for FeatureError {}

/// Result type for feature calculations
pub type FeatureResult<T> = Result<T, FeatureError>;

/// Convert Decimal to f64 for statistical calculations
///
/// # Safety
/// This function returns NaN if the conversion fails, which should only happen
/// for extreme values that don't occur in normal market data. NaN will propagate
/// through calculations, making any errors visible rather than hidden.
#[inline]
pub(crate) fn decimal_to_f64_or_nan(d: Decimal) -> f64 {
    d.to_f64().unwrap_or_else(|| {
        #[cfg(debug_assertions)]
        eprintln!("Warning: Decimal to f64 conversion failed for value: {d}");
        f64::NAN
    })
}

/// L2 Level data for feature calculation
#[derive(Debug, Clone, Copy)]
pub struct Level {
    /// Price at this level
    pub price: Decimal,
    /// Total quantity available at this level
    pub quantity: Decimal,
    /// Number of orders at this level (if available from exchange)
    pub order_count: u32,
}

/// Multi-level order book snapshot
///
/// Cache-aligned for optimal memory access patterns
#[repr(align(64))]
#[derive(Debug, Clone)]
pub struct OrderBookSnapshot {
    /// Timestamp of this snapshot in nanoseconds
    pub timestamp_ns: u64,
    /// Symbol/instrument identifier
    pub symbol: String,
    /// Bid levels (best to worst, price descending)
    pub bids: SmallVec<[Level; 25]>,
    /// Ask levels (best to worst, price ascending)
    pub asks: SmallVec<[Level; 25]>,
}

impl OrderBookSnapshot {
    /// Calculate mid price from best bid and ask
    #[must_use]
    pub fn mid_price(&self) -> Decimal {
        if self.bids.is_empty() || self.asks.is_empty() {
            return Decimal::ZERO;
        }
        (self.bids[0].price + self.asks[0].price) / Decimal::TWO
    }
}

/// Trade tick for feature calculation
///
/// Cache-aligned for efficient processing in hot paths
#[repr(align(64))]
#[derive(Debug, Clone)]
pub struct TradeTick {
    /// Timestamp of the trade in nanoseconds
    pub timestamp_ns: u64,
    /// Symbol/instrument identifier
    pub symbol: String,
    /// Side of the aggressor/taker (buy or sell)
    pub side: TradeSide,
    /// Execution price of the trade
    pub price: Decimal,
    /// Trade quantity/volume
    pub quantity: Decimal,
}

/// Side of the trade indicating the aggressor/taker
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TradeSide {
    /// Buy trade (buyer was aggressor)
    Buy = 1,
    /// Sell trade (seller was aggressor)
    Sell = -1,
}

/// Combined microstructural features
///
/// Cache-aligned for efficient batch processing
#[repr(align(64))]
#[derive(Debug, Clone, Default)]
pub struct MicrostructuralFeatures {
    /// Timestamp when these features were calculated in nanoseconds
    pub timestamp_ns: u64,

    // Order Flow Features
    /// Basic order flow imbalance (level 1 only)
    pub ofi_basic: f64,
    /// Volume-weighted order flow imbalance across multiple levels
    pub ofi_weighted: f64,
    /// Volume-Synchronized Probability of Informed Trading
    pub vpin: f64,

    // Queue Features
    /// Imbalance between best bid and ask quantities
    pub queue_imbalance: f64,
    /// Queue imbalance weighted by depth levels
    pub weighted_queue_imbalance: f64,

    // Liquidity Features
    /// Total bid-side liquidity across tracked levels
    pub bid_liquidity: Decimal,
    /// Total ask-side liquidity across tracked levels
    pub ask_liquidity: Decimal,
    /// Ratio of bid to ask liquidity
    pub liquidity_ratio: f64,

    // Volatility Features
    /// Realized volatility over recent time window
    pub realized_volatility: f64,
    /// Estimated permanent price impact of trades
    pub price_impact: f64,

    // Market Structure
    /// Current bid-ask spread
    pub spread: Decimal,
    /// Current mid price (average of best bid and ask)
    pub mid_price: Decimal,
    /// Spread normalized by mid price (basis points)
    pub relative_spread: f64,
}

/// Snapshot of all calculated features at a point in time
#[derive(Debug, Clone, Default)]
pub struct FeatureSnapshot {
    /// Timestamp of this snapshot in nanoseconds
    pub timestamp: u64,

    // Trade-based features
    /// Volume-weighted average price over the window
    pub vwap: f64,
    /// Number of trades per unit time
    pub trade_intensity: f64,
    /// Ratio of buy trades to sell trades
    pub buy_sell_ratio: f64,
    /// Proportion of volume from large trades
    pub large_trade_ratio: f64,
    /// Net order flow (buy volume - sell volume) normalized
    pub order_flow_imbalance: f64,

    // Orderbook features
    /// Total quantity on bid side
    pub bid_liquidity: f64,
    /// Total quantity on ask side
    pub ask_liquidity: f64,
    /// Bid-ask spread in basis points
    pub spread_bps: f64,
    /// Current mid price
    pub mid_price: f64,
    /// Imbalance at best bid/ask levels
    pub queue_imbalance: f64,
    /// Ratio of bid depth to ask depth
    pub book_depth_ratio: f64,
    /// Rate of price decay on bid side
    pub bid_slope: f64,
    /// Rate of price increase on ask side
    pub ask_slope: f64,
    /// Asymmetry in order book shape
    pub orderbook_skew: f64,
    /// Overall bid-ask imbalance
    pub orderbook_imbalance: f64,
    /// Mid price weighted by best level quantities
    pub weighted_mid_price: f64,

    // Additional advanced features
    /// Volume-Synchronized Probability of Informed Trading
    pub vpin: f64,
    /// Multi-level weighted queue imbalance
    pub weighted_queue_imbalance: f64,
    /// Order Flow Imbalance
    pub ofi: f64,
    /// Volume-weighted OFI
    pub weighted_ofi: f64,
    /// Realized volatility estimate
    pub realized_volatility: f64,
    /// Kyle's lambda (price impact coefficient)
    pub kyles_lambda: f64,
    /// Market asymmetry index
    pub asymmetry_index: f64,
}

/// Trade-based features calculated from market trade data.
///
/// These features capture various aspects of trading activity and order flow dynamics
/// that are commonly used in HFT strategies for predicting short-term price movements.
#[derive(Debug, Clone, Default)]
pub struct TradeFeatures {
    /// Volume-weighted average price (VWAP) - The average price weighted by volume
    pub vwap: f64,
    /// Trade intensity - Number of trades per unit time, indicating market activity level
    pub trade_intensity: f64,
    /// Buy/sell ratio - Ratio of buy volume to sell volume, indicating directional pressure
    pub buy_sell_ratio: f64,
    /// Large trade ratio - Proportion of volume from large trades, indicating institutional activity
    pub large_trade_ratio: f64,
    /// Order flow imbalance - Net directional flow (buy volume - sell volume)
    pub order_flow_imbalance: f64,
}

/// Order book-based features calculated from limit order book snapshots.
///
/// These features capture the state and dynamics of the order book, providing insights
/// into liquidity, price pressure, and market microstructure that are essential for
/// high-frequency trading strategies.
#[derive(Debug, Clone, Default)]
pub struct OrderBookFeatures {
    /// Total bid-side liquidity - Sum of all bid quantities
    pub bid_liquidity: f64,
    /// Total ask-side liquidity - Sum of all ask quantities
    pub ask_liquidity: f64,
    /// Bid-ask spread in basis points - Relative spread normalized by mid price
    pub spread_bps: f64,
    /// Mid price - Average of best bid and best ask prices
    pub mid_price: f64,
    /// Queue imbalance - Imbalance between best bid and ask quantities
    pub queue_imbalance: f64,
    /// Book depth ratio - Ratio of bid to ask depth, indicating relative liquidity
    pub book_depth_ratio: f64,
    /// Bid slope - Rate of price decay on bid side, indicating buying pressure
    pub bid_slope: f64,
    /// Ask slope - Rate of price increase on ask side, indicating selling pressure
    pub ask_slope: f64,
    /// Order book skew - Asymmetry in the order book shape
    pub orderbook_skew: f64,
    /// Order book imbalance - Overall imbalance between bid and ask sides
    pub orderbook_imbalance: f64,
    /// Weighted mid price - Mid price weighted by best bid/ask quantities
    pub weighted_mid_price: f64,
    /// Basic order flow imbalance - Simple OFI calculation from order book changes
    pub ofi_basic: f64,
}

/// High-performance feature calculator
pub struct FeatureCalculator {
    window_size: usize,
    trade_buffer: Vec<TradeTick>,
    orderbook_buffer: Vec<OrderBookSnapshot>,
    #[allow(dead_code)]
    feature_cache: FxHashMap<u64, MicrostructuralFeatures>,
}

impl FeatureCalculator {
    /// Create new feature calculator
    #[must_use]
    pub fn new(window_size: usize) -> Self {
        Self {
            window_size,
            trade_buffer: Vec::with_capacity(window_size * 2),
            orderbook_buffer: Vec::with_capacity(window_size * 2),
            feature_cache: FxHashMap::default(),
        }
    }

    /// Add a trade tick to the internal buffer for feature calculation.
    ///
    /// This method maintains a sliding window of trades by:
    /// 1. Adding the new trade to the buffer
    /// 2. Keeping the buffer size at most 2x the window size
    /// 3. When the buffer exceeds 2x window size, it drains the oldest trades
    ///    (removes `window_size` trades from the beginning)
    ///
    /// This approach ensures we always have enough historical data for feature
    /// calculations while limiting memory usage. The 2x window size allows for
    /// efficient batch removal instead of removing one trade at a time.
    ///
    /// # Arguments
    /// * `trade` - The trade tick to add to the buffer
    pub fn add_trade(&mut self, trade: &TradeTick) {
        self.trade_buffer.push(trade.clone());

        // Maintain window size
        if self.trade_buffer.len() > self.window_size * 2 {
            let drain_count = self.window_size;
            self.trade_buffer.drain(0..drain_count);
        }
    }

    /// Add order book snapshot and calculate features
    pub fn add_orderbook(&mut self, snapshot: &OrderBookSnapshot) {
        self.orderbook_buffer.push(snapshot.clone());

        // Maintain window size
        if self.orderbook_buffer.len() > self.window_size * 2 {
            let drain_count = self.window_size;
            self.orderbook_buffer.drain(0..drain_count);
        }
    }

    /// Calculate trade-based features
    fn calculate_trade_features(&self) -> TradeFeatures {
        let recent_trades =
            &self.trade_buffer[self.trade_buffer.len().saturating_sub(self.window_size)..];

        let mut features = TradeFeatures::default();

        // Calculate VWAP
        if !recent_trades.is_empty() {
            let total_value: f64 = recent_trades
                .iter()
                .map(|t| decimal_to_f64_or_nan(t.price * t.quantity))
                .sum();
            let total_volume: f64 = recent_trades
                .iter()
                .map(|t| decimal_to_f64_or_nan(t.quantity))
                .sum();
            features.vwap = if total_volume > 0.0 {
                total_value / total_volume
            } else {
                0.0
            };

            // Trade intensity
            features.trade_intensity = recent_trades.len() as f64;

            // Buy/sell ratio
            let buy_count = recent_trades
                .iter()
                .filter(|t| t.side == TradeSide::Buy)
                .count() as f64;
            let sell_count = recent_trades
                .iter()
                .filter(|t| t.side == TradeSide::Sell)
                .count() as f64;
            features.buy_sell_ratio = if sell_count > 0.0 {
                buy_count / sell_count
            } else {
                buy_count
            };

            // Large trade ratio (placeholder - you can define "large" based on your criteria)
            features.large_trade_ratio = 0.0;

            // Order flow imbalance
            let buy_volume: f64 = recent_trades
                .iter()
                .filter(|t| t.side == TradeSide::Buy)
                .map(|t| decimal_to_f64_or_nan(t.quantity))
                .sum();
            let sell_volume: f64 = recent_trades
                .iter()
                .filter(|t| t.side == TradeSide::Sell)
                .map(|t| decimal_to_f64_or_nan(t.quantity))
                .sum();
            let total = buy_volume + sell_volume;
            features.order_flow_imbalance = if total > 0.0 {
                (buy_volume - sell_volume) / total
            } else {
                0.0
            };
        }

        features
    }

    /// Calculate order book based features
    fn calculate_orderbook_features(&self) -> OrderBookFeatures {
        let recent_books =
            &self.orderbook_buffer[self.orderbook_buffer.len().saturating_sub(self.window_size)..];

        let mut features = OrderBookFeatures::default();

        if let Some(current_book) = recent_books.last() {
            // Basic spread and mid price
            if let (Some(best_bid), Some(best_ask)) =
                (current_book.bids.first(), current_book.asks.first())
            {
                let spread = best_ask.price - best_bid.price;
                features.mid_price =
                    decimal_to_f64_or_nan((best_bid.price + best_ask.price) / Decimal::from(2));
                features.spread_bps = decimal_to_f64_or_nan(
                    spread / ((best_bid.price + best_ask.price) / Decimal::from(2))
                        * Decimal::from(10000),
                );
            }

            // Queue imbalance
            features.queue_imbalance =
                calculate_queue_imbalance(&current_book.bids, &current_book.asks);

            // OFI calculation
            if recent_books.len() >= 2 {
                let prev_book = &recent_books[recent_books.len() - 2];
                features.ofi_basic = calculate_ofi(prev_book, current_book);
            }

            // Calculate liquidity
            features.bid_liquidity = current_book
                .bids
                .iter()
                .take(10)
                .map(|l| decimal_to_f64_or_nan(l.quantity))
                .sum();
            features.ask_liquidity = current_book
                .asks
                .iter()
                .take(10)
                .map(|l| decimal_to_f64_or_nan(l.quantity))
                .sum();

            // Book depth ratio
            features.book_depth_ratio = if features.ask_liquidity > 0.0 {
                features.bid_liquidity / features.ask_liquidity
            } else {
                0.0
            };

            // Calculate slopes (simplified - you can implement more sophisticated slope calculation)
            features.bid_slope = 0.0;
            features.ask_slope = 0.0;

            // Orderbook skew
            features.orderbook_skew = if features.bid_liquidity + features.ask_liquidity > 0.0 {
                (features.bid_liquidity - features.ask_liquidity)
                    / (features.bid_liquidity + features.ask_liquidity)
            } else {
                0.0
            };

            // Orderbook imbalance (same as skew in this simplified version)
            features.orderbook_imbalance = features.orderbook_skew;

            // Weighted mid price (simplified - using top level only)
            if let (Some(best_bid), Some(best_ask)) =
                (current_book.bids.first(), current_book.asks.first())
            {
                let bid_weight = decimal_to_f64_or_nan(best_bid.quantity);
                let ask_weight = decimal_to_f64_or_nan(best_ask.quantity);
                let total_weight = bid_weight + ask_weight;
                if total_weight > 0.0 {
                    features.weighted_mid_price = (decimal_to_f64_or_nan(best_bid.price)
                        * ask_weight
                        + decimal_to_f64_or_nan(best_ask.price) * bid_weight)
                        / total_weight;
                } else {
                    features.weighted_mid_price = features.mid_price;
                }
            }
        }

        features
    }

    /// Alias for add_orderbook() to maintain backward compatibility
    #[must_use]
    pub fn on_orderbook(&mut self, snapshot: &OrderBookSnapshot) -> OrderBookFeatures {
        self.add_orderbook(snapshot);
        self.calculate_orderbook_features()
    }

    /// Alias for add_trade() to maintain backward compatibility
    #[must_use]
    pub fn on_trade(&mut self, trade: &TradeTick) -> Option<TradeFeatures> {
        self.add_trade(trade);
        if !self.trade_buffer.is_empty() {
            Some(self.calculate_trade_features())
        } else {
            None
        }
    }

    /// Get current features based on accumulated data
    pub fn get_features(&self) -> FeatureSnapshot {
        // Get latest features from both trade and orderbook data
        let trade_features = if !self.trade_buffer.is_empty() {
            self.calculate_trade_features()
        } else {
            TradeFeatures::default()
        };

        let orderbook_features = if !self.orderbook_buffer.is_empty() {
            self.calculate_orderbook_features()
        } else {
            OrderBookFeatures::default()
        };

        FeatureSnapshot {
            timestamp: self
                .orderbook_buffer
                .last()
                .map(|ob| ob.timestamp_ns)
                .or_else(|| self.trade_buffer.last().map(|t| t.timestamp_ns))
                .unwrap_or(0),
            // Flatten all features into the snapshot
            vwap: trade_features.vwap,
            trade_intensity: trade_features.trade_intensity,
            buy_sell_ratio: trade_features.buy_sell_ratio,
            large_trade_ratio: trade_features.large_trade_ratio,
            order_flow_imbalance: trade_features.order_flow_imbalance,
            bid_liquidity: orderbook_features.bid_liquidity,
            ask_liquidity: orderbook_features.ask_liquidity,
            spread_bps: orderbook_features.spread_bps,
            mid_price: orderbook_features.mid_price,
            queue_imbalance: orderbook_features.queue_imbalance,
            book_depth_ratio: orderbook_features.book_depth_ratio,
            bid_slope: orderbook_features.bid_slope,
            ask_slope: orderbook_features.ask_slope,
            orderbook_skew: orderbook_features.orderbook_skew,
            orderbook_imbalance: orderbook_features.orderbook_imbalance,
            weighted_mid_price: orderbook_features.weighted_mid_price,

            // Additional advanced features (set to default/0 for now)
            vpin: 0.0,
            weighted_queue_imbalance: 0.0,
            ofi: orderbook_features.ofi_basic,
            weighted_ofi: 0.0,
            realized_volatility: 0.0,
            kyles_lambda: 0.0,
            asymmetry_index: 0.0,
        }
    }
}