From 3b519e6a31983d28f29d6293de338bfc90796f86 Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 19 Nov 2024 17:24:06 +0200 Subject: [PATCH 1/8] Add time-weighted median price calculation. To enable, set MEDIAN_DECAY env var (default recommended value 0.0001). --- src/data-feeds/ccxt-provider-service.ts | 74 +++++++++++++++++++++---- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/src/data-feeds/ccxt-provider-service.ts b/src/data-feeds/ccxt-provider-service.ts index 270e20d..5dfc6bb 100644 --- a/src/data-feeds/ccxt-provider-service.ts +++ b/src/data-feeds/ccxt-provider-service.ts @@ -1,5 +1,5 @@ import { Logger } from "@nestjs/common"; -import ccxt, { Exchange, Trade } from "ccxt"; +import ccxt, { Exchange, pro, Trade } from "ccxt"; import { readFileSync } from "fs"; import { FeedId, FeedValueData } from "../dto/provider-requests.dto"; import { BaseDataFeed } from "./base-feed"; @@ -27,12 +27,13 @@ interface FeedConfig { } interface PriceInfo { - price: number; + value: number; time: number; exchange: string; } const usdtToUsdFeedId: FeedId = { category: FeedCategory.Crypto.valueOf(), name: "USDT/USD" }; +const lambda = process.env.MEDIAN_DECAY ? parseFloat(process.env.MEDIAN_DECAY) : undefined; export class CcxtFeed implements BaseDataFeed { private readonly logger = new Logger(CcxtFeed.name); @@ -202,7 +203,7 @@ export class CcxtFeed implements BaseDataFeed { private setPrice(exchangeName: string, symbol: string, price: number, timestamp: number) { const prices = this.prices.get(symbol) || new Map(); prices.set(exchangeName, { - price: price, + value: price, time: timestamp, exchange: exchangeName, }); @@ -227,7 +228,7 @@ export class CcxtFeed implements BaseDataFeed { return price * usdtToUsd; }; - const prices: number[] = []; + const prices: PriceInfo[] = []; // Gather all available prices for (const source of config.sources) { @@ -235,13 +236,16 @@ export class CcxtFeed implements BaseDataFeed { // Skip if no price information is available if (!info) continue; - let price = info.price; + let price = info.value; price = source.symbol.endsWith("USDT") ? await convertToUsd(source.symbol, source.exchange, price) : price; if (price === undefined) continue; // Add the price to our list for median calculation - prices.push(price); + prices.push({ + ...info, + value: price, + }); } if (prices.length === 0) { @@ -249,24 +253,74 @@ export class CcxtFeed implements BaseDataFeed { return undefined; } + if (lambda === undefined) { + return this.median(prices); + } else { + return this.weightedMedian(prices); + } + } + + private median(prices: PriceInfo[]): number { // If single price found, return price if (prices.length === 1) { - return prices[0]; + return prices[0].value; } // Sort the prices in ascending order - prices.sort((a, b) => a - b); + prices.sort((a, b) => a.value - b.value); // Calculate the median const mid = Math.floor(prices.length / 2); const median = prices.length % 2 !== 0 - ? prices[mid] // Odd number of elements, take the middle one - : (prices[mid - 1] + prices[mid]) / 2; // Even number of elements, average the two middle ones + ? prices[mid].value // Odd number of elements, take the middle one + : (prices[mid - 1].value + prices[mid].value) / 2; // Even number of elements, average the two middle ones return median; } + private weightedMedian(prices: PriceInfo[]): number { + if (prices.length === 0) { + throw new Error("Price list cannot be empty."); + } + + prices.sort((a, b) => a.time - b.time); + + // Current time for weight calculation + const now = Date.now(); + + // Calculate exponential weights + const weights = prices.map(data => { + const timeDifference = now - data.time; + return Math.exp(-lambda * timeDifference); // Exponential decay + }); + + // Normalize weights to sum to 1 + const weightSum = weights.reduce((sum, weight) => sum + weight, 0); + const normalizedWeights = weights.map(weight => weight / weightSum); + + // Combine prices and weights + const weightedPrices = prices.map((data, i) => ({ + price: data.value, + weight: normalizedWeights[i], + })); + + // Sort prices by value for median calculation + weightedPrices.sort((a, b) => a.price - b.price); + + // Find the weighted median + let cumulativeWeight = 0; + for (let i = 0; i < weightedPrices.length; i++) { + cumulativeWeight += weightedPrices[i].weight; + if (cumulativeWeight >= 0.5) { + return weightedPrices[i].price; + } + } + + this.logger.warn("Unable to calculate weighted median"); + return undefined; + } + private loadConfig() { const network = process.env.NETWORK as networks; let configPath: string; From 7818e9cbcf883ae914eccebbcc850554c4984403 Mon Sep 17 00:00:00 2001 From: magurh Date: Tue, 19 Nov 2024 11:36:06 +0000 Subject: [PATCH 2/8] fix: update list of exchanges --- src/config/feeds.json | 118 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 23 deletions(-) diff --git a/src/config/feeds.json b/src/config/feeds.json index a92eb28..4462d8e 100644 --- a/src/config/feeds.json +++ b/src/config/feeds.json @@ -17,6 +17,22 @@ { "exchange": "okx", "symbol": "FLR/USDT" + }, + { + "exchange": "coinbase", + "symbol": "FLR/USD" + }, + { + "exchange": "kraken", + "symbol": "FLR/USD" + }, + { + "exchange": "bybit", + "symbol": "FLR/USDT" + }, + { + "exchange": "htx", + "symbol": "FLR/USDT" } ] }, @@ -34,6 +50,10 @@ { "exchange": "kraken", "symbol": "SGB/USD" + }, + { + "exchange": "mexc", + "symbol": "SGB/USDT" } ] }, @@ -67,6 +87,18 @@ { "exchange": "binance", "symbol": "XRP/USDT" + }, + { + "exchange": "coinbase", + "symbol": "XRP/USD" + }, + { + "exchange": "bybit", + "symbol": "XRP/USDT" + }, + { + "exchange": "kraken", + "symbol": "XRP/USD" } ] }, @@ -209,6 +241,10 @@ { "exchange": "okx", "symbol": "ADA/USDT" + }, + { + "exchange": "coinbase", + "symbol": "ADA/USD" } ] }, @@ -862,6 +898,14 @@ { "exchange": "okx", "symbol": "UNI/USDT" + }, + { + "exchange": "htx", + "symbol": "UNI/USDT" + }, + { + "exchange": "coinbase", + "symbol": "UNI/USD" } ] }, @@ -1362,19 +1406,19 @@ "feed": { "category": 1, "name": "SUI/USD" }, "sources": [ { - "exchange": "ascendex", - "symbol": "SUI/USDT" + "exchange": "coinbase", + "symbol": "SUI/USD" }, { - "exchange": "binanceus", + "exchange": "okx", "symbol": "SUI/USDT" }, { - "exchange": "bingx", + "exchange": "binanceus", "symbol": "SUI/USDT" }, { - "exchange": "bitfinex2", + "exchange": "bingx", "symbol": "SUI/USDT" }, { @@ -1388,6 +1432,14 @@ { "exchange": "bybit", "symbol": "SUI/USDT" + }, + { + "exchange": "okx", + "symbol": "SUI/USDT" + }, + { + "exchange": "mexc", + "symbol": "SUI/USDT" } ] }, @@ -1402,10 +1454,6 @@ "exchange": "bingx", "symbol": "PEPE/USDT" }, - { - "exchange": "bitfinex2", - "symbol": "PEPE/USDT" - }, { "exchange": "bitget", "symbol": "PEPE/USDT" @@ -1421,6 +1469,22 @@ { "exchange": "cryptocom", "symbol": "PEPE/USDT" + }, + { + "exchange": "coinbase", + "symbol": "PEPE/USD" + }, + { + "exchange": "gate", + "symbol": "PEPE/USDT" + }, + { + "exchange": "okx", + "symbol": "PEPE/USDT" + }, + { + "exchange": "htx", + "symbol": "PEPE/USDT" } ] }, @@ -1457,32 +1521,36 @@ "feed": { "category": 1, "name": "AAVE/USD" }, "sources": [ { - "exchange": "ascendex", + "exchange": "binanceus", "symbol": "AAVE/USDT" }, { - "exchange": "binanceus", + "exchange": "bingx", "symbol": "AAVE/USDT" }, { - "exchange": "bingx", + "exchange": "bitget", "symbol": "AAVE/USDT" }, { - "exchange": "bitfinex2", + "exchange": "bybit", "symbol": "AAVE/USDT" }, { - "exchange": "bitget", + "exchange": "htx", "symbol": "AAVE/USDT" }, { - "exchange": "bitmart", + "exchange": "gate", "symbol": "AAVE/USDT" }, { - "exchange": "bybit", + "exchange": "okx", "symbol": "AAVE/USDT" + }, + { + "exchange": "coinbase", + "symbol": "AAVE/USD" } ] }, @@ -1490,31 +1558,35 @@ "feed": { "category": 1, "name": "FTM/USD" }, "sources": [ { - "exchange": "ascendex", + "exchange": "binanceus", "symbol": "FTM/USDT" }, { - "exchange": "binanceus", + "exchange": "bingx", "symbol": "FTM/USDT" }, { - "exchange": "bingx", + "exchange": "bitget", "symbol": "FTM/USDT" }, { - "exchange": "bitfinex2", + "exchange": "bitmart", "symbol": "FTM/USDT" }, { - "exchange": "bitget", + "exchange": "bybit", "symbol": "FTM/USDT" }, { - "exchange": "bitmart", + "exchange": "kucoin", "symbol": "FTM/USDT" }, { - "exchange": "bybit", + "exchange": "okx", + "symbol": "FTM/USDT" + }, + { + "exchange": "gate", "symbol": "FTM/USDT" } ] From 24ee3f9067cd571c09d0312053a47aabb9e82a69 Mon Sep 17 00:00:00 2001 From: Dinesh Pinto Date: Wed, 20 Nov 2024 22:16:38 +0400 Subject: [PATCH 3/8] fix: pol source on okx --- src/config/feeds.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/feeds.json b/src/config/feeds.json index 4462d8e..0e8e37c 100644 --- a/src/config/feeds.json +++ b/src/config/feeds.json @@ -517,7 +517,7 @@ }, { "exchange": "okx", - "symbol": "POL/USDT" + "symbol": "POL/USDC" }, { "exchange": "binance", From 5b3d6e27cbeb45e5f75d12481976b7ea33788f63 Mon Sep 17 00:00:00 2001 From: Dinesh Pinto Date: Wed, 20 Nov 2024 22:29:05 +0400 Subject: [PATCH 4/8] fix: revert pol/usdc change --- src/config/feeds.json | 124 ++++++++---------------------------------- 1 file changed, 24 insertions(+), 100 deletions(-) diff --git a/src/config/feeds.json b/src/config/feeds.json index 0e8e37c..14e96c3 100644 --- a/src/config/feeds.json +++ b/src/config/feeds.json @@ -17,22 +17,6 @@ { "exchange": "okx", "symbol": "FLR/USDT" - }, - { - "exchange": "coinbase", - "symbol": "FLR/USD" - }, - { - "exchange": "kraken", - "symbol": "FLR/USD" - }, - { - "exchange": "bybit", - "symbol": "FLR/USDT" - }, - { - "exchange": "htx", - "symbol": "FLR/USDT" } ] }, @@ -50,10 +34,6 @@ { "exchange": "kraken", "symbol": "SGB/USD" - }, - { - "exchange": "mexc", - "symbol": "SGB/USDT" } ] }, @@ -87,18 +67,6 @@ { "exchange": "binance", "symbol": "XRP/USDT" - }, - { - "exchange": "coinbase", - "symbol": "XRP/USD" - }, - { - "exchange": "bybit", - "symbol": "XRP/USDT" - }, - { - "exchange": "kraken", - "symbol": "XRP/USD" } ] }, @@ -241,10 +209,6 @@ { "exchange": "okx", "symbol": "ADA/USDT" - }, - { - "exchange": "coinbase", - "symbol": "ADA/USD" } ] }, @@ -517,7 +481,7 @@ }, { "exchange": "okx", - "symbol": "POL/USDC" + "symbol": "POL/USDT" }, { "exchange": "binance", @@ -601,10 +565,6 @@ { "exchange": "cryptocom", "symbol": "USDT/USD" - }, - { - "exchange": "kraken", - "symbol": "USDT/USD" } ] }, @@ -898,14 +858,6 @@ { "exchange": "okx", "symbol": "UNI/USDT" - }, - { - "exchange": "htx", - "symbol": "UNI/USDT" - }, - { - "exchange": "coinbase", - "symbol": "UNI/USD" } ] }, @@ -1406,11 +1358,7 @@ "feed": { "category": 1, "name": "SUI/USD" }, "sources": [ { - "exchange": "coinbase", - "symbol": "SUI/USD" - }, - { - "exchange": "okx", + "exchange": "ascendex", "symbol": "SUI/USDT" }, { @@ -1422,23 +1370,19 @@ "symbol": "SUI/USDT" }, { - "exchange": "bitget", - "symbol": "SUI/USDT" - }, - { - "exchange": "bitmart", + "exchange": "bitfinex2", "symbol": "SUI/USDT" }, { - "exchange": "bybit", + "exchange": "bitget", "symbol": "SUI/USDT" }, { - "exchange": "okx", + "exchange": "bitmart", "symbol": "SUI/USDT" }, { - "exchange": "mexc", + "exchange": "bybit", "symbol": "SUI/USDT" } ] @@ -1454,6 +1398,10 @@ "exchange": "bingx", "symbol": "PEPE/USDT" }, + { + "exchange": "bitfinex2", + "symbol": "PEPE/USDT" + }, { "exchange": "bitget", "symbol": "PEPE/USDT" @@ -1469,22 +1417,6 @@ { "exchange": "cryptocom", "symbol": "PEPE/USDT" - }, - { - "exchange": "coinbase", - "symbol": "PEPE/USD" - }, - { - "exchange": "gate", - "symbol": "PEPE/USDT" - }, - { - "exchange": "okx", - "symbol": "PEPE/USDT" - }, - { - "exchange": "htx", - "symbol": "PEPE/USDT" } ] }, @@ -1521,36 +1453,32 @@ "feed": { "category": 1, "name": "AAVE/USD" }, "sources": [ { - "exchange": "binanceus", + "exchange": "ascendex", "symbol": "AAVE/USDT" }, { - "exchange": "bingx", + "exchange": "binanceus", "symbol": "AAVE/USDT" }, { - "exchange": "bitget", + "exchange": "bingx", "symbol": "AAVE/USDT" }, { - "exchange": "bybit", + "exchange": "bitfinex2", "symbol": "AAVE/USDT" }, { - "exchange": "htx", + "exchange": "bitget", "symbol": "AAVE/USDT" }, { - "exchange": "gate", + "exchange": "bitmart", "symbol": "AAVE/USDT" }, { - "exchange": "okx", + "exchange": "bybit", "symbol": "AAVE/USDT" - }, - { - "exchange": "coinbase", - "symbol": "AAVE/USD" } ] }, @@ -1558,35 +1486,31 @@ "feed": { "category": 1, "name": "FTM/USD" }, "sources": [ { - "exchange": "binanceus", - "symbol": "FTM/USDT" - }, - { - "exchange": "bingx", + "exchange": "ascendex", "symbol": "FTM/USDT" }, { - "exchange": "bitget", + "exchange": "binanceus", "symbol": "FTM/USDT" }, { - "exchange": "bitmart", + "exchange": "bingx", "symbol": "FTM/USDT" }, { - "exchange": "bybit", + "exchange": "bitfinex2", "symbol": "FTM/USDT" }, { - "exchange": "kucoin", + "exchange": "bitget", "symbol": "FTM/USDT" }, { - "exchange": "okx", + "exchange": "bitmart", "symbol": "FTM/USDT" }, { - "exchange": "gate", + "exchange": "bybit", "symbol": "FTM/USDT" } ] From da3329e43bcd1e36b5e89c16b891a3882fd8c685 Mon Sep 17 00:00:00 2001 From: Dinesh Pinto Date: Wed, 20 Nov 2024 22:46:49 +0400 Subject: [PATCH 5/8] fix: remove invalid exchange pairs --- src/config/feeds.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/config/feeds.json b/src/config/feeds.json index 14e96c3..68e648b 100644 --- a/src/config/feeds.json +++ b/src/config/feeds.json @@ -438,14 +438,6 @@ "exchange": "binance", "symbol": "MATIC/USDT" }, - { - "exchange": "cryptocom", - "symbol": "MATIC/USDT" - }, - { - "exchange": "cryptocom", - "symbol": "MATIC/USD" - }, { "exchange": "gate", "symbol": "MATIC/USDT" From fa539a89386b3742c8540dca407beb5f03861cca Mon Sep 17 00:00:00 2001 From: Dinesh Pinto Date: Wed, 20 Nov 2024 22:54:10 +0400 Subject: [PATCH 6/8] fix: remove deprecated pairs --- src/config/feeds.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/config/feeds.json b/src/config/feeds.json index 68e648b..a221ce0 100644 --- a/src/config/feeds.json +++ b/src/config/feeds.json @@ -437,18 +437,6 @@ { "exchange": "binance", "symbol": "MATIC/USDT" - }, - { - "exchange": "gate", - "symbol": "MATIC/USDT" - }, - { - "exchange": "kucoin", - "symbol": "MATIC/USDT" - }, - { - "exchange": "okx", - "symbol": "MATIC/USDT" } ] }, From e189965382a1d6461bc603e4340ae86e31a143b5 Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 26 Nov 2024 11:51:50 +0200 Subject: [PATCH 7/8] Fix timestamps, add debug logging --- src/data-feeds/ccxt-provider-service.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/data-feeds/ccxt-provider-service.ts b/src/data-feeds/ccxt-provider-service.ts index 5dfc6bb..3415a20 100644 --- a/src/data-feeds/ccxt-provider-service.ts +++ b/src/data-feeds/ccxt-provider-service.ts @@ -132,7 +132,7 @@ export class CcxtFeed implements BaseDataFeed { continue; } - this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp); + this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp ?? 0); } } else { throw new Error("Exchange does not support fetchTickers"); @@ -152,7 +152,7 @@ export class CcxtFeed implements BaseDataFeed { continue; } - this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp); + this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp ?? 0); } } } @@ -196,7 +196,7 @@ export class CcxtFeed implements BaseDataFeed { private processTrades(trades: Trade[], exchangeName: string) { trades.forEach(trade => { - this.setPrice(exchangeName, trade.symbol, trade.price, trade.timestamp); + this.setPrice(exchangeName, trade.symbol, trade.price, trade.timestamp ?? Date.now()); }); } @@ -253,6 +253,7 @@ export class CcxtFeed implements BaseDataFeed { return undefined; } + this.logger.debug(`Calculating results for ${JSON.stringify(feedId)}`); if (lambda === undefined) { return this.median(prices); } else { @@ -303,16 +304,24 @@ export class CcxtFeed implements BaseDataFeed { const weightedPrices = prices.map((data, i) => ({ price: data.value, weight: normalizedWeights[i], + exchange: data.exchange, + staleness: now - data.time, })); // Sort prices by value for median calculation weightedPrices.sort((a, b) => a.price - b.price); + this.logger.debug("Weighted prices:"); + for (const { price, weight, exchange, staleness: we } of weightedPrices) { + this.logger.debug(`Price: ${price}, weight: ${weight}, staleness ms: ${we}, exchange: ${exchange}`); + } + // Find the weighted median let cumulativeWeight = 0; for (let i = 0; i < weightedPrices.length; i++) { cumulativeWeight += weightedPrices[i].weight; if (cumulativeWeight >= 0.5) { + this.logger.debug(`Weighted median: ${weightedPrices[i].price}`); return weightedPrices[i].price; } } From 98b7a0385389d50ae6bf4725bf275c7338b2f309 Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 26 Nov 2024 15:24:25 +0200 Subject: [PATCH 8/8] Use weighted median by default --- src/data-feeds/ccxt-provider-service.ts | 36 +++++-------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/src/data-feeds/ccxt-provider-service.ts b/src/data-feeds/ccxt-provider-service.ts index 3415a20..836c93e 100644 --- a/src/data-feeds/ccxt-provider-service.ts +++ b/src/data-feeds/ccxt-provider-service.ts @@ -33,7 +33,8 @@ interface PriceInfo { } const usdtToUsdFeedId: FeedId = { category: FeedCategory.Crypto.valueOf(), name: "USDT/USD" }; -const lambda = process.env.MEDIAN_DECAY ? parseFloat(process.env.MEDIAN_DECAY) : undefined; +// Parameter for exponential decay in time-weighted median price calculation +const lambda = process.env.MEDIAN_DECAY ? parseFloat(process.env.MEDIAN_DECAY) : 0.00005; export class CcxtFeed implements BaseDataFeed { private readonly logger = new Logger(CcxtFeed.name); @@ -132,7 +133,7 @@ export class CcxtFeed implements BaseDataFeed { continue; } - this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp ?? 0); + this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp); } } else { throw new Error("Exchange does not support fetchTickers"); @@ -152,7 +153,7 @@ export class CcxtFeed implements BaseDataFeed { continue; } - this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp ?? 0); + this.setPrice(exchangeName, ticker.symbol, ticker.last, ticker.timestamp); } } } @@ -196,7 +197,7 @@ export class CcxtFeed implements BaseDataFeed { private processTrades(trades: Trade[], exchangeName: string) { trades.forEach(trade => { - this.setPrice(exchangeName, trade.symbol, trade.price, trade.timestamp ?? Date.now()); + this.setPrice(exchangeName, trade.symbol, trade.price, trade.timestamp); }); } @@ -204,7 +205,7 @@ export class CcxtFeed implements BaseDataFeed { const prices = this.prices.get(symbol) || new Map(); prices.set(exchangeName, { value: price, - time: timestamp, + time: timestamp ?? Date.now(), exchange: exchangeName, }); this.prices.set(symbol, prices); @@ -254,30 +255,7 @@ export class CcxtFeed implements BaseDataFeed { } this.logger.debug(`Calculating results for ${JSON.stringify(feedId)}`); - if (lambda === undefined) { - return this.median(prices); - } else { - return this.weightedMedian(prices); - } - } - - private median(prices: PriceInfo[]): number { - // If single price found, return price - if (prices.length === 1) { - return prices[0].value; - } - - // Sort the prices in ascending order - prices.sort((a, b) => a.value - b.value); - - // Calculate the median - const mid = Math.floor(prices.length / 2); - const median = - prices.length % 2 !== 0 - ? prices[mid].value // Odd number of elements, take the middle one - : (prices[mid - 1].value + prices[mid].value) / 2; // Even number of elements, average the two middle ones - - return median; + return this.weightedMedian(prices); } private weightedMedian(prices: PriceInfo[]): number {