Working with Market Depth and Trades

Display 3-depth

[1]:
from numba import njit

@njit
def print_3depth(hbt):
    while hbt.elapse(60_000_000_000) == 0:
        print('current_timestamp:', hbt.current_timestamp)

        # Gets the market depth for the first asset, in the same order as when you created the backtest.
        depth = hbt.depth(0)

        # a key of bid_depth or ask_depth is price in ticks.
        # (integer) price_tick = rice / tick_size
        i = 0
        for price_tick in range(depth.best_ask_tick, depth.best_ask_tick + 100):
            qty = depth.ask_qty_at_tick(price_tick)
            if qty > 0:
                print(
                    'ask: ',
                    qty,
                    '@',
                    np.round(price_tick * depth.tick_size, 1)
                )

                i += 1
                if i == 3:
                    break
        i = 0
        for price_tick in range(depth.best_bid_tick, max(depth.best_bid_tick - 100, 0), -1):
            qty = depth.bid_qty_at_tick(price_tick)
            if qty > 0:
                print(
                    'bid: ',
                    qty,
                    '@',
                    np.round(price_tick * depth.tick_size, 1)
                )

                i += 1
                if i == 3:
                    break
    return True
[2]:
import numpy as np

btcusdt_20240809 = np.load('usdm/btcusdt_20240809.npz')['data']
btcusdt_20240808_eod = np.load('usdm/btcusdt_20240808_eod.npz')['data']
[3]:
from hftbacktest import BacktestAsset, HashMapMarketDepthBacktest

asset = (
    BacktestAsset()
        .data(btcusdt_20240809)
        .initial_snapshot(btcusdt_20240808_eod)
        .linear_asset(1.0)
        .constant_latency(10_000_000, 10_000_000)
        .risk_adverse_queue_model()
        .no_partial_fill_exchange()
        .trading_value_fee_model(0.0002, 0.0007)
        .tick_size(0.1)
        .lot_size(0.001)
)

hbt = HashMapMarketDepthBacktest([asset])

print_3depth(hbt)

_ = hbt.close()
current_timestamp: 1723161661500000000
ask:  1.759 @ 61594.2
ask:  0.006 @ 61594.4
ask:  0.114 @ 61595.2
bid:  3.526 @ 61594.1
bid:  0.016 @ 61594.0
bid:  0.002 @ 61593.9
current_timestamp: 1723161721500000000
ask:  2.575 @ 61576.6
ask:  0.004 @ 61576.7
ask:  0.455 @ 61577.0
bid:  2.558 @ 61576.5
bid:  0.002 @ 61576.0
bid:  0.515 @ 61575.5
current_timestamp: 1723161781500000000
ask:  0.131 @ 61629.7
ask:  0.005 @ 61630.1
ask:  0.005 @ 61630.5
bid:  5.742 @ 61629.6
bid:  0.247 @ 61629.4
bid:  0.034 @ 61629.3
current_timestamp: 1723161841500000000
ask:  0.202 @ 61621.6
ask:  0.002 @ 61622.5
ask:  0.003 @ 61622.6
bid:  3.488 @ 61621.5
bid:  0.86 @ 61620.0
bid:  0.248 @ 61619.6
current_timestamp: 1723161901500000000
ask:  1.397 @ 61584.0
ask:  0.832 @ 61585.1
ask:  0.132 @ 61586.0
bid:  3.307 @ 61583.9
bid:  0.01 @ 61583.8
bid:  0.002 @ 61582.0

Efficient Market Depth Access

ROIVectorMarketDepth provides more efficient market depth access through a vector that holds a limited price range of interest. The backtester using this feature can be created by ROIVectorMarketDepthBacktest.

[4]:
from numba import njit

@njit
def print_3depth_fast(hbt):
    roi_lb_tick = int(round(30000 / 0.1))
    roi_ub_tick = int(round(90000 / 0.1))

    while hbt.elapse(60_000_000_000) == 0:
        print('current_timestamp:', hbt.current_timestamp)

        # Gets the market depth for the first asset, in the same order as when you created the backtest.
        depth = hbt.depth(0)

        # a key of bid_depth or ask_depth is price in ticks.
        # (integer) price_tick = price / tick_size
        i = 0
        # for price_tick in range(depth.best_ask_tick, depth.best_ask_tick + 100):
        #     # depth.ask_depth returns the ask depth array, whose length is (roi_ub_tick + 1 - roi_lb_tick),
        #     # containing the quantities ranging from roi_lb_tick to roi_ub_tick.
        #     # Checks that the price_tick is in that range and adjust the index by subtracting roi_lb_tick.
        #     if price_tick < roi_lb_tick or price_tick > roi_ub_tick:
        #         continue
        #     t = price_tick - roi_lb_tick
        #     qty = depth.ask_depth[t]
        #     if qty > 0:
        #         print(
        #             'ask: ',
        #             qty,
        #             '@',
        #             np.round(price_tick * depth.tick_size, 1)
        #         )

        #         i += 1
        #         if i == 3:
        #             break
        # i = 0
        # for price_tick in range(depth.best_bid_tick, max(depth.best_bid_tick - 100, 0), -1):
        #     # depth.bid_depth returns the bid depth array, whose length is (roi_ub_tick + 1 - roi_lb_tick),
        #     # containing the quantities ranging from roi_lb_tick to roi_ub_tick.
        #     # Checks that the price_tick is in that range and adjust the index by subtracting roi_lb_tick.
        #     if price_tick < roi_lb_tick or price_tick > roi_ub_tick:
        #         continue
        #     t = price_tick - roi_lb_tick
        #     qty = depth.bid_depth[t]
        #     if qty > 0:
        #         print(
        #             'bid: ',
        #             qty,
        #             '@',
        #             np.round(price_tick * depth.tick_size, 1)
        #         )

        #         i += 1
        #         if i == 3:
        #             break
    return True
[5]:
from hftbacktest import ROIVectorMarketDepthBacktest

asset = (
    BacktestAsset()
        .data(btcusdt_20240809)
        .initial_snapshot(btcusdt_20240808_eod)
        .linear_asset(1.0)
        .constant_latency(10_000_000, 10_000_000)
        .risk_adverse_queue_model()
        .no_partial_fill_exchange()
        .trading_value_fee_model(0.0002, 0.0007)
        .tick_size(0.1)
        .lot_size(0.001)
        # Sets the lower bound price for the range of interest in the market depth.
        .roi_lb(30000)
        # Sets the upper bound price for the range of interest in the market depth.
        .roi_ub(90000)
)


[6]:
hbt = ROIVectorMarketDepthBacktest([asset])

print_3depth_fast(hbt)

#_ = hbt.close()
current_timestamp: 1723161661500000000
current_timestamp: 1723161721500000000
current_timestamp: 1723161781500000000
current_timestamp: 1723161841500000000
current_timestamp: 1723161901500000000
[6]:
True

Order Book Imbalance

[7]:
@njit
def orderbookimbalance(hbt, out):
    roi_lb_tick = int(round(30000 / 0.1))
    roi_ub_tick = int(round(90000 / 0.1))

    while hbt.elapse(10 * 1e9) == 0:
        depth = hbt.depth(0)

        mid_price = (depth.best_bid + depth.best_ask) / 2.0

        sum_ask_qty_50bp = 0.0
        sum_ask_qty = 0.0
        for price_tick in range(depth.best_ask_tick, roi_ub_tick + 1):
            if price_tick < roi_lb_tick or price_tick > roi_ub_tick:
                continue
            t = price_tick - roi_lb_tick

            ask_price = price_tick * depth.tick_size
            depth_from_mid = (ask_price - mid_price) / mid_price
            if depth_from_mid > 0.01:
                break
            sum_ask_qty += depth.ask_depth[t]

            if depth_from_mid <= 0.005:
                sum_ask_qty_50bp = sum_ask_qty


        sum_bid_qty_50bp = 0.0
        sum_bid_qty = 0.0
        for price_tick in range(depth.best_bid_tick, roi_lb_tick - 1, -1):
            if price_tick < roi_lb_tick or price_tick > roi_ub_tick:
                continue
            t = price_tick - roi_lb_tick

            bid_price = price_tick * depth.tick_size
            depth_from_mid = (mid_price - bid_price) / mid_price
            if depth_from_mid > 0.01:
                break
            sum_bid_qty += depth.bid_depth[t]

            if depth_from_mid <= 0.005:
                sum_bid_qty_50bp = sum_bid_qty

        imbalance_50bp = sum_bid_qty_50bp - sum_ask_qty_50bp
        imbalance_1pct = sum_bid_qty - sum_ask_qty
        imbalance_tob = depth.bid_depth[depth.best_bid_tick - roi_lb_tick] - depth.ask_depth[depth.best_ask_tick - roi_lb_tick]

        out.append((hbt.current_timestamp, imbalance_tob, imbalance_50bp, imbalance_1pct))
    return True
[8]:
from numba.typed import List
from numba.types import Tuple, float64

hbt = ROIVectorMarketDepthBacktest([asset])

tup_ty = Tuple((float64, float64, float64, float64))
out = List.empty_list(tup_ty, allocated=100_000)

orderbookimbalance(hbt, out)

_ = hbt.close()
[9]:
import polars as pl

df = pl.DataFrame(out).transpose()
df.columns = ['Local Timestamp', 'TOB Imbalance', '0.5% Imbalance', '1% Imbalance']
df = df.with_columns(
    pl.from_epoch('Local Timestamp', time_unit='ns')
)

df
[9]:
shape: (30, 4)
Local TimestampTOB Imbalance0.5% Imbalance1% Imbalance
datetime[ns]f64f64f64
2024-08-09 00:00:11.5002.729-1748.101-3908.736
2024-08-09 00:00:21.5004.623-1749.435-3512.845
2024-08-09 00:00:31.500-6.465-1259.897-3357.755
2024-08-09 00:00:41.500-7.922-1174.185-3471.955
2024-08-09 00:00:51.500-2.484-1147.597-3461.48
2024-08-09 00:04:21.5003.828-1186.236-3551.78
2024-08-09 00:04:31.500-1.35-1332.379-3517.854
2024-08-09 00:04:41.500-3.754-1166.521-2693.672
2024-08-09 00:04:51.500-2.525-1188.56-2716.914
2024-08-09 00:05:01.5001.91-594.991-2138.82
[10]:
import holoviews as hv

hv.extension('bokeh')

df.plot(x='Local Timestamp')
[10]:

Display last trades between the step

[11]:
from hftbacktest import BUY_EVENT

@njit
def print_trades(hbt):
    while hbt.elapse(60 * 1e9) == 0:
        print('-------------------------------------------------------------------------------')
        print('current_timestamp:', hbt.current_timestamp)

        # Gets the last trades occurring in the market, not the trades of our orders.
        last_trades = hbt.last_trades(0)

        num = 0
        for last_trade in last_trades:
            if num > 10:
                print('...')
                break
            print(
                'exch_timestamp:',
                last_trade.exch_ts,
                'buy' if (last_trade.ev & BUY_EVENT) == BUY_EVENT else 'sell',
                last_trade.qty,
                '@',
                last_trade.px
            )
            num += 1

        # To prevent accumulating all last trades, which may cause a slowdown,
        # clear_last_trades needs to be called.
        # After this, accessing `last_trades` will cause a crash.
        hbt.clear_last_trades(0)
    return True
[12]:
asset = (
    BacktestAsset()
        .data(btcusdt_20240809)
        .initial_snapshot(btcusdt_20240808_eod)
        .linear_asset(1.0)
        .constant_latency(10_000_000, 10_000_000)
        .risk_adverse_queue_model()
        .no_partial_fill_exchange()
        .trading_value_fee_model(0.0002, 0.0007)
        .tick_size(0.1)
        .lot_size(0.001)
        # To retrieve the last trades, `last_trades_capacity` should be set.
        .last_trades_capacity(1000)
        .roi_lb(30000)
        .roi_ub(90000)
)

hbt = ROIVectorMarketDepthBacktest([asset])

print_trades(hbt)

_ = hbt.close()
-------------------------------------------------------------------------------
current_timestamp: 1723161661500000000
exch_timestamp: 1723161602372000000 buy 0.489 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.198 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.006 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.002 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.003 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.011 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.238 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.007 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.005 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.003 @ 61659.8
exch_timestamp: 1723161602372000000 buy 0.002 @ 61659.8
...
-------------------------------------------------------------------------------
current_timestamp: 1723161721500000000
exch_timestamp: 1723161661697000000 sell 0.002 @ 61594.1
exch_timestamp: 1723161661724000000 sell 0.002 @ 61594.1
exch_timestamp: 1723161661751000000 buy 0.135 @ 61594.2
exch_timestamp: 1723161661806000000 sell 1.328 @ 61594.1
exch_timestamp: 1723161661806000000 sell 0.002 @ 61594.1
exch_timestamp: 1723161661806000000 sell 0.002 @ 61594.1
exch_timestamp: 1723161661806000000 sell 0.002 @ 61594.1
exch_timestamp: 1723161661806000000 sell 0.006 @ 61594.1
exch_timestamp: 1723161661806000000 sell 0.32 @ 61594.1
exch_timestamp: 1723161661806000000 sell 0.032 @ 61594.1
exch_timestamp: 1723161661806000000 sell 1.208 @ 61594.1
...
-------------------------------------------------------------------------------
current_timestamp: 1723161781500000000
exch_timestamp: 1723161721541000000 sell 0.002 @ 61576.5
exch_timestamp: 1723161721574000000 buy 0.012 @ 61576.6
exch_timestamp: 1723161721578000000 sell 0.003 @ 61576.5
exch_timestamp: 1723161721583000000 buy 0.275 @ 61576.6
exch_timestamp: 1723161721583000000 buy 0.469 @ 61576.6
exch_timestamp: 1723161721585000000 buy 0.095 @ 61576.6
exch_timestamp: 1723161721585000000 buy 0.102 @ 61576.6
exch_timestamp: 1723161721585000000 buy 0.197 @ 61576.6
exch_timestamp: 1723161721586000000 buy 0.13 @ 61576.6
exch_timestamp: 1723161721587000000 buy 0.425 @ 61576.6
exch_timestamp: 1723161721587000000 buy 0.324 @ 61576.6
...
-------------------------------------------------------------------------------
current_timestamp: 1723161841500000000
exch_timestamp: 1723161781628000000 sell 0.026 @ 61629.6
exch_timestamp: 1723161781727000000 buy 0.011 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.05 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.006 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.002 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.007 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.002 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.075 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.065 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.247 @ 61629.7
exch_timestamp: 1723161781727000000 buy 0.002 @ 61629.7
...
-------------------------------------------------------------------------------
current_timestamp: 1723161901500000000
exch_timestamp: 1723161841561000000 buy 0.01 @ 61621.6
exch_timestamp: 1723161841561000000 buy 0.006 @ 61621.6
exch_timestamp: 1723161841561000000 buy 0.002 @ 61621.6
exch_timestamp: 1723161841561000000 buy 0.022 @ 61621.6
exch_timestamp: 1723161841561000000 buy 0.097 @ 61621.6
exch_timestamp: 1723161841561000000 buy 0.024 @ 61621.6
exch_timestamp: 1723161841564000000 buy 0.024 @ 61621.6
exch_timestamp: 1723161841564000000 buy 0.014 @ 61621.6
exch_timestamp: 1723161841565000000 buy 0.003 @ 61621.6
exch_timestamp: 1723161841613000000 buy 0.002 @ 61622.5
exch_timestamp: 1723161841613000000 buy 0.003 @ 61622.6
...

Rolling Volume-Weighted Average Price

[13]:
@njit
def rolling_vwap(hbt, out):
    buy_amount_bin = np.zeros(100_000, np.float64)
    buy_qty_bin = np.zeros(100_000, np.float64)
    sell_amount_bin = np.zeros(100_000, np.float64)
    sell_qty_bin = np.zeros(100_000, np.float64)

    idx = 0
    last_trade_price = np.nan

    while hbt.elapse(10 * 1e9) == 0:
        last_trades = hbt.last_trades(0)

        for last_trade in last_trades:
            if (last_trade.ev & BUY_EVENT) == BUY_EVENT:
                buy_amount_bin[idx] += last_trade.px * last_trade.qty
                buy_qty_bin[idx] += last_trade.qty
            else:
                sell_amount_bin[idx] += last_trade.px * last_trade.qty
                sell_qty_bin[idx] += last_trade.qty

        hbt.clear_last_trades(0)
        idx += 1

        if idx >= 1:
            vwap10sec = np.divide(
                buy_amount_bin[idx - 1] + sell_amount_bin[idx - 1],
                buy_qty_bin[idx - 1] + sell_qty_bin[idx - 1]
            )
        else:
            vwap10sec = np.nan

        if idx >= 6:
            vwap1m = np.divide(
                np.sum(buy_amount_bin[idx - 6:idx]) + np.sum(sell_amount_bin[idx - 6:idx]),
                np.sum(buy_qty_bin[idx - 6:idx]) + np.sum(sell_qty_bin[idx - 6:idx])
            )
            buy_vwap1m = np.divide(np.sum(buy_amount_bin[idx - 6:idx]), np.sum(buy_qty_bin[idx - 6:idx]))
            sell_vwap1m = np.divide(np.sum(sell_amount_bin[idx - 6:idx]), np.sum(sell_qty_bin[idx - 6:idx]))
        else:
            vwap1m = np.nan
            buy_vwap1m = np.nan
            sell_vwap1m = np.nan

        out.append((hbt.current_timestamp, vwap10sec, vwap1m, buy_vwap1m, sell_vwap1m))
    return True
[14]:
hbt = ROIVectorMarketDepthBacktest([asset])

tup_ty = Tuple((float64, float64, float64, float64, float64))
out = List.empty_list(tup_ty, allocated=100_000)

rolling_vwap(hbt, out)

_ = hbt.close()
[15]:
df = pl.DataFrame(out).transpose()
df.columns = ['Local Timestamp', '10-sec VWAP', '1-min VWAP', '1-min Buy VWAP', '1-min Sell VWAP']
df = df.with_columns(
    pl.from_epoch('Local Timestamp', time_unit='ns')
)

df
[15]:
shape: (30, 5)
Local Timestamp10-sec VWAP1-min VWAP1-min Buy VWAP1-min Sell VWAP
datetime[ns]f64f64f64f64
2024-08-09 00:00:11.50061687.182976NaNNaNNaN
2024-08-09 00:00:21.50061709.337576NaNNaNNaN
2024-08-09 00:00:31.50061697.538054NaNNaNNaN
2024-08-09 00:00:41.50061663.958879NaNNaNNaN
2024-08-09 00:00:51.50061637.340621NaNNaNNaN
2024-08-09 00:04:21.50061643.00984761624.45901161626.49554261622.549429
2024-08-09 00:04:31.50061670.79568561635.87725161638.36231461632.48854
2024-08-09 00:04:41.50061643.10858261641.84648961648.67233761636.032054
2024-08-09 00:04:51.50061614.72356961640.49084161647.76984461634.372128
2024-08-09 00:05:01.50061584.69746761637.33410261642.20955161632.12064
[16]:
df.plot(x='Local Timestamp')
[16]: