diff --git a/typescript/infra/config/environments/mainnet3/balances/dailyRelayerBurn.json b/typescript/infra/config/environments/mainnet3/balances/dailyRelayerBurn.json new file mode 100644 index 0000000000..38104dd6c2 --- /dev/null +++ b/typescript/infra/config/environments/mainnet3/balances/dailyRelayerBurn.json @@ -0,0 +1,124 @@ +{ + "abstract": 0.00291, + "ancient8": 0.00373, + "alephzeroevmmainnet": 13.8, + "apechain": 3.56, + "appchain": 0.0236, + "arbitrum": 0.0402, + "arbitrumnova": 0.00102, + "artela": 191, + "arthera": 108, + "astar": 65.9, + "astarzkevm": 0.00369, + "aurora": 0.00102, + "flame": 1.95, + "avalanche": 0.0943, + "b3": 0.00427, + "base": 0.0712, + "bitlayer": 0.00202, + "blast": 0.00591, + "bob": 0.00352, + "boba": 0.00351, + "bsc": 0.167, + "bsquared": 0.00362, + "celo": 6.04, + "cheesechain": 2670, + "chilizmainnet": 100, + "conflux": 21.5, + "conwai": 889, + "coredao": 4.56, + "corn": 0.0000316, + "cyber": 0.00443, + "degenchain": 481, + "dogechain": 10.5, + "duckchain": 0.643, + "eclipsemainnet": 0.025, + "endurance": 2.23, + "ethereum": 0.556, + "everclear": 0.00626, + "evmos": 224, + "fantom": 6.61, + "flare": 133, + "flowmainnet": 5.13, + "form": 0.00355, + "fraxtal": 0.016, + "fusemainnet": 126, + "glue": 8.4, + "gnosis": 3.12, + "gravity": 122, + "guru": 280, + "harmony": 171, + "hemi": 0.00102, + "immutablezkevmmainnet": 4.14, + "inevm": 0.171, + "ink": 0.0113, + "injective": 0.171, + "kaia": 16.8, + "kroma": 0.00341, + "linea": 0.0782, + "lisk": 0.00781, + "lukso": 1.97, + "lumia": 3.21, + "lumiaprism": 4.09, + "mantapacific": 0.00371, + "mantle": 13.9, + "matchain": 0.0048, + "merlin": 0.000318, + "metal": 0.0102, + "metis": 0.1, + "mint": 0.00462, + "mode": 0.0135, + "molten": 9.51, + "moonbeam": 24.1, + "morph": 0.283, + "nero": 3.13, + "neutron": 11.8, + "oortmainnet": 36.7, + "optimism": 0.0354, + "orderly": 0.00471, + "osmosis": 8.28, + "polygon": 16.9, + "polygonzkevm": 0.0447, + "polynomialfi": 0.00436, + "prom": 2.92, + "proofofplay": 0.00102, + "rarichain": 0.00691, + "real": 0.00157, + "redstone": 0.00344, + "rivalz": 0.00102, + "rootstockmainnet": 0.00241, + "sanko": 0.158, + "scroll": 0.0169, + "sei": 10.4, + "shibarium": 9.34, + "snaxchain": 0.00412, + "solanamainnet": 5, + "soneium": 0.00719, + "sonic": 6.61, + "sonicsvm": 0.0138, + "soon": 0.00102, + "stride": 6.88, + "superseed": 0.00396, + "superpositionmainnet": 0.00152, + "swell": 0.00454, + "taiko": 0.00726, + "tangle": 3.13, + "telos": 22.7, + "torus": 5.82, + "treasure": 789, + "trumpchain": 0.12, + "unichain": 0.00102, + "unitzero": 6.11, + "vana": 0.372, + "viction": 9.34, + "worldchain": 0.00348, + "xai": 19.6, + "xlayer": 0.373, + "xpla": 41.3, + "zeronetwork": 0.00267, + "zetachain": 7.57, + "zircuit": 0.00379, + "zklink": 0.00102, + "zksync": 0.00284, + "zoramainnet": 0.00832 +} diff --git a/typescript/infra/config/environments/mainnet3/balances/desiredRelayerBalances.json b/typescript/infra/config/environments/mainnet3/balances/desiredRelayerBalances.json new file mode 100644 index 0000000000..88f37a8cee --- /dev/null +++ b/typescript/infra/config/environments/mainnet3/balances/desiredRelayerBalances.json @@ -0,0 +1,124 @@ +{ + "abstract": "0", + "alephzeroevmmainnet": "110", + "ancient8": "0.5", + "apechain": "50", + "appchain": "0.189", + "arbitrum": "0.5", + "arbitrumnova": "0.05", + "artela": "1530", + "arthera": "864", + "astar": "527", + "astarzkevm": "0.05", + "aurora": "0.05", + "avalanche": "5", + "b3": "0.05", + "base": "0.57", + "bitlayer": "0.0162", + "blast": "0.2", + "bob": "0.2", + "boba": "0.05", + "bsc": "5", + "bsquared": "0.029", + "celo": "48.3", + "cheesechain": "21400", + "chilizmainnet": "800", + "conflux": "172", + "conwai": "7110", + "coredao": "36.5", + "corn": "0.001", + "cyber": "0.05", + "degenchain": "3850", + "dogechain": "100", + "duckchain": "5.14", + "eclipsemainnet": "0", + "endurance": "20", + "ethereum": "4.45", + "everclear": "0.0501", + "evmos": "1790", + "fantom": "100", + "flame": "15.6", + "flare": "1060", + "flowmainnet": "41", + "form": "0.05", + "fraxtal": "0.2", + "fusemainnet": "1010", + "glue": "67.2", + "gnosis": "25", + "gravity": "976", + "guru": "2240", + "harmony": "1370", + "hemi": "0.05", + "immutablezkevmmainnet": "33.1", + "inevm": "3", + "injective": "0", + "ink": "0.0904", + "kaia": "250", + "kroma": "0.05", + "linea": "1", + "lisk": "0.0625", + "lukso": "20", + "lumia": "25.7", + "lumiaprism": "32.7", + "mantapacific": "0.2", + "mantle": "111", + "matchain": "0.05", + "merlin": "0.00254", + "metal": "0.0816", + "metis": "3", + "mint": "0.05", + "mode": "0.2", + "molten": "76.1", + "moonbeam": "193", + "morph": "2.26", + "nero": "25", + "neutron": "0", + "oortmainnet": "2000", + "optimism": "0.5", + "orderly": "0.05", + "osmosis": "0", + "polygon": "135", + "polygonzkevm": "0.5", + "polynomialfi": "0.05", + "prom": "23.4", + "proofofplay": "0.05", + "rarichain": "0.0553", + "real": "0.1", + "redstone": "0.2", + "rivalz": "0.05", + "rootstockmainnet": "0.0193", + "sanko": "2", + "scroll": "0.5", + "sei": "83.2", + "shibarium": "74.7", + "snaxchain": "0.05", + "solanamainnet": "0", + "soneium": "0.0575", + "sonic": "52.9", + "sonicsvm": "0", + "soon": "0", + "stride": "0", + "superpositionmainnet": "0.05", + "superseed": "0.05", + "swell": "0.05", + "taiko": "0.2", + "tangle": "25", + "telos": "182", + "torus": "46.6", + "treasure": "6310", + "trumpchain": "0.96", + "unichain": "0.05", + "unitzero": "50", + "vana": "2.98", + "viction": "74.7", + "worldchain": "0.2", + "xai": "157", + "xlayer": "2.98", + "xpla": "330", + "zeronetwork": "0.05", + "zetachain": "60.6", + "zircuit": "0.0303", + "zklink": "0.05", + "zksync": "0.05", + "zoramainnet": "0.2" +} diff --git a/typescript/infra/config/environments/mainnet3/balances/highUrgencyRelayerBalance.json b/typescript/infra/config/environments/mainnet3/balances/highUrgencyRelayerBalance.json new file mode 100644 index 0000000000..93f6db4f26 --- /dev/null +++ b/typescript/infra/config/environments/mainnet3/balances/highUrgencyRelayerBalance.json @@ -0,0 +1,122 @@ +{ + "alephzeroevmmainnet": "27.6", + "ancient8": "0.00746", + "apechain": "7.12", + "appchain": "0.0472", + "arbitrum": "0.0804", + "arbitrumnova": "0.00204", + "artela": "382", + "arthera": "216", + "astar": "132", + "astarzkevm": "0.00738", + "aurora": "0.00204", + "avalanche": "0.89", + "b3": "0.00854", + "base": "0.142", + "bitlayer": "0.00404", + "blast": "0.0118", + "bob": "0.00704", + "boba": "0.00702", + "bsc": "0.47", + "bsquared": "0.00724", + "celo": "12.1", + "cheesechain": "5340", + "chilizmainnet": "200", + "conflux": "43", + "conwai": "1780", + "coredao": "9.12", + "corn": "0.0000632", + "cyber": "0.00886", + "degenchain": "962", + "dogechain": "21", + "duckchain": "1.29", + "eclipsemainnet": "0.05", + "endurance": "4.46", + "ethereum": "1.11", + "everclear": "0.0125", + "evmos": "448", + "fantom": "13.2", + "flame": "3.9", + "flare": "266", + "flowmainnet": "10.3", + "form": "0.0071", + "fraxtal": "0.032", + "fusemainnet": "252", + "glue": "16.8", + "gnosis": "6.24", + "gravity": "244", + "guru": "560", + "harmony": "342", + "hemi": "0.00204", + "immutablezkevmmainnet": "8.28", + "inevm": "0.342", + "injective": "0.342", + "ink": "0.0226", + "kaia": "33.6", + "kroma": "0.00682", + "linea": "0.22", + "lisk": "0.0156", + "lukso": "3.94", + "lumia": "6.42", + "lumiaprism": "8.18", + "mantapacific": "0.02", + "mantle": "27.8", + "matchain": "0.0096", + "merlin": "0.000636", + "metal": "0.0204", + "metis": "1", + "mint": "0.00924", + "mode": "0.027", + "molten": "19", + "moonbeam": "48.2", + "morph": "0.566", + "nero": "6.26", + "neutron": "23.6", + "oortmainnet": "400", + "optimism": "0.0708", + "orderly": "0.00942", + "polygon": "33.8", + "polygonzkevm": "0.0894", + "polynomialfi": "0.00872", + "prom": "5.84", + "proofofplay": "0.00204", + "rarichain": "0.0138", + "real": "0.00314", + "redstone": "0.00688", + "rivalz": "0.00204", + "rootstockmainnet": "0.00482", + "sanko": "0.316", + "scroll": "0.0338", + "sei": "20.8", + "shibarium": "18.7", + "snaxchain": "0.00824", + "solanamainnet": "10", + "soneium": "0.0144", + "sonic": "13.2", + "sonicsvm": "0.0276", + "soon": "0.00204", + "stride": "13.8", + "superpositionmainnet": "0.00304", + "superseed": "0.00792", + "swell": "0.00908", + "taiko": "0.019", + "tangle": "6.26", + "telos": "45.4", + "torus": "11.6", + "treasure": "1580", + "trumpchain": "0.24", + "unichain": "0.00204", + "unitzero": "12.2", + "vana": "0.744", + "viction": "18.7", + "worldchain": "0.00696", + "xai": "39.2", + "xlayer": "0.746", + "xpla": "82.6", + "zeronetwork": "0.00534", + "zetachain": "15.1", + "zircuit": "0.0094", + "zklink": "0.00204", + "zksync": "0.00568", + "zoramainnet": "0.0166" +} diff --git a/typescript/infra/config/environments/mainnet3/balances/lowUrgencyKeyFunderBalance.json b/typescript/infra/config/environments/mainnet3/balances/lowUrgencyKeyFunderBalance.json new file mode 100644 index 0000000000..292cff8a0e --- /dev/null +++ b/typescript/infra/config/environments/mainnet3/balances/lowUrgencyKeyFunderBalance.json @@ -0,0 +1,122 @@ +{ + "alephzeroevmmainnet": "200", + "ancient8": "1", + "apechain": "100", + "appchain": "0.283", + "arbitrum": "1.2", + "arbitrumnova": "0.1", + "artela": "2290", + "arthera": "1300", + "astar": "791", + "astarzkevm": "0.1", + "aurora": "0.1", + "avalanche": "22", + "b3": "0.1", + "base": "1.1", + "bitlayer": "0.0242", + "blast": "0.4", + "bob": "0.4", + "boba": "0.1", + "bsc": "8", + "bsquared": "0.0434", + "celo": "308", + "cheesechain": "32000", + "chilizmainnet": "1200", + "conflux": "258", + "conwai": "10700", + "coredao": "54.7", + "corn": "0.002", + "cyber": "0.1", + "degenchain": "5770", + "dogechain": "200", + "duckchain": "10", + "eclipsemainnet": "0.3", + "endurance": "40", + "ethereum": "6.67", + "everclear": "0.1", + "evmos": "2690", + "fantom": "200", + "flame": "23.4", + "flare": "1600", + "flowmainnet": "61.6", + "form": "0.1", + "fraxtal": "0.4", + "fusemainnet": "1510", + "glue": "101", + "gnosis": "210", + "gravity": "1460", + "guru": "3360", + "harmony": "2050", + "hemi": "0.0122", + "immutablezkevmmainnet": "50", + "inevm": "6.1", + "injective": "2.05", + "ink": "0.136", + "kaia": "500", + "kroma": "0.1", + "linea": "2", + "lisk": "0.1", + "lukso": "40", + "lumia": "38.5", + "lumiaprism": "49.1", + "mantapacific": "0.4", + "mantle": "167", + "matchain": "0.0576", + "merlin": "0.00382", + "metal": "0.122", + "metis": "6", + "mint": "0.1", + "mode": "0.4", + "molten": "114", + "moonbeam": "700", + "morph": "3.4", + "nero": "37.6", + "neutron": "142", + "oortmainnet": "4000", + "optimism": "1.2", + "orderly": "0.1", + "polygon": "250", + "polygonzkevm": "1.1", + "polynomialfi": "0.1", + "prom": "36", + "proofofplay": "0.05", + "rarichain": "0.1", + "real": "0.2", + "redstone": "0.4", + "rivalz": "0.1", + "rootstockmainnet": "0.0289", + "sanko": "2", + "scroll": "1.1", + "sei": "125", + "shibarium": "112", + "snaxchain": "0.05", + "solanamainnet": "60", + "soneium": "0.1", + "sonic": "79.3", + "sonicsvm": "0.166", + "soon": "0.0122", + "stride": "82.6", + "superpositionmainnet": "0.1", + "superseed": "0.05", + "swell": "0.1", + "taiko": "0.4", + "tangle": "37.6", + "telos": "272", + "torus": "69.8", + "treasure": "9470", + "trumpchain": "1.44", + "unichain": "0.1", + "unitzero": "73.3", + "vana": "4.46", + "viction": "112", + "worldchain": "0.4", + "xai": "235", + "xlayer": "4.48", + "xpla": "496", + "zeronetwork": "0.1", + "zetachain": "90.8", + "zircuit": "0.0455", + "zklink": "0.05", + "zksync": "0.1", + "zoramainnet": "0.4" +} diff --git a/typescript/infra/package.json b/typescript/infra/package.json index ba2cc9a219..51cf0d2b49 100644 --- a/typescript/infra/package.json +++ b/typescript/infra/package.json @@ -28,6 +28,7 @@ "deep-object-diff": "^1.1.9", "dotenv": "^10.0.0", "json-stable-stringify": "^1.1.1", + "postgres": "^3.4.5", "prom-client": "^14.0.1", "prompts": "^2.4.2", "yaml": "2.4.5", diff --git a/typescript/infra/scripts/agent-utils.ts b/typescript/infra/scripts/agent-utils.ts index 260c51956b..5e0614591f 100644 --- a/typescript/infra/scripts/agent-utils.ts +++ b/typescript/infra/scripts/agent-utils.ts @@ -43,6 +43,8 @@ import { EnvironmentConfig, assertEnvironment, } from '../src/config/environment.js'; +import { BalanceThresholdType } from '../src/config/funding/balances.js'; +import { AlertType } from '../src/config/funding/grafanaAlerts.js'; import { Role } from '../src/roles.js'; import { assertContext, @@ -90,6 +92,12 @@ export function getArgs() { .alias('e', 'environment'); } +export function withBalanceThresholdConfig(args: Argv) { + return args + .describe('balanceThresholdConfig', 'balance threshold config') + .choices('balanceThresholdConfig', Object.values(BalanceThresholdType)); +} + export function withFork(args: Argv) { return args .describe('fork', 'network to fork') @@ -155,6 +163,13 @@ export function withChain(args: Argv) { .alias('c', 'chain'); } +export function withWrite(args: Argv) { + return args + .describe('write', 'Write output to file') + .boolean('write') + .default('write', false); +} + export function withChains(args: Argv, chainOptions?: ChainName[]) { return ( args @@ -200,6 +215,23 @@ export function withProtocol(args: Argv) { .demandOption('protocol'); } +export function withAlertType(args: Argv) { + return args + .describe('alertType', 'alert type') + .choices('alertType', Object.values(AlertType)); +} + +export function withAlertTypeRequired(args: Argv) { + return withAlertType(args).demandOption('alertType'); +} + +export function withConfirmAllChoices(args: Argv) { + return args + .describe('all', 'Confirm all choices') + .boolean('all') + .default('all', false); +} + export function withAgentRole(args: Argv) { return args .describe('role', 'agent role') diff --git a/typescript/infra/scripts/funding/calculate-relayer-daily-burn.ts b/typescript/infra/scripts/funding/calculate-relayer-daily-burn.ts new file mode 100644 index 0000000000..40c93e05ad --- /dev/null +++ b/typescript/infra/scripts/funding/calculate-relayer-daily-burn.ts @@ -0,0 +1,137 @@ +import postgres, { Sql } from 'postgres'; + +import { ChainMap } from '@hyperlane-xyz/sdk'; +import { rootLogger } from '@hyperlane-xyz/utils'; + +import rawDailyRelayerBurn from '../../config/environments/mainnet3/balances/dailyRelayerBurn.json'; +import { mainnet3SupportedChainNames } from '../../config/environments/mainnet3/supportedChainNames.js'; +import rawTokenPrices from '../../config/environments/mainnet3/tokenPrices.json'; +import { RELAYER_MIN_DOLLAR_BALANCE_PER_DAY } from '../../src/config/funding/balances.js'; +import { formatDailyRelayerBurn } from '../../src/funding/grafana.js'; +import { fetchLatestGCPSecret } from '../../src/utils/gcloud.js'; +import { writeJsonAtPath } from '../../src/utils/utils.js'; + +const tokenPrices: ChainMap = rawTokenPrices; +const currentDailyRelayerBurn: ChainMap = rawDailyRelayerBurn; + +const DAILY_BURN_PATH = + './config/environments/mainnet3/balances/dailyRelayerBurn.json'; + +const LOOK_BACK_DAYS = 10; // the number of days to look back for average destination tx costs +const MIN_NUMBER_OF_TXS = 200; // the minimum number of txs to consider for daily burn + +async function main() { + const chainsMissingInTokenPrices = mainnet3SupportedChainNames.filter( + (chain) => !(chain in tokenPrices), + ); + + if (chainsMissingInTokenPrices.length > 0) { + rootLogger.error( + `Token prices missing for chains: ${chainsMissingInTokenPrices.join( + ', ', + )} consider adding them to tokenPrices.json and running the script again.`, + ); + process.exit(1); + } + + const sql = await getReadOnlyScraperDb(); + let burnData: ChainMap; + try { + burnData = await getDailyRelayerBurn(sql); + } catch (err) { + rootLogger.error('Error fetching daily burn data:', err); + process.exit(1); + } finally { + await sql.end(); + } + + console.table(burnData); + + try { + rootLogger.info('Writing daily burn data to file..'); + writeJsonAtPath(DAILY_BURN_PATH, burnData); + rootLogger.info('Daily burn data written to file.'); + } catch (err) { + rootLogger.error('Error writing daily burn data to file:', err); + } +} + +async function getReadOnlyScraperDb() { + const credentialsUrl = await fetchLatestGCPSecret( + 'hyperlane-mainnet3-scraper3-db-read-only', + ); + return postgres(credentialsUrl); +} + +async function fetchDailyRelayerBurnData(sql: Sql) { + const results = await sql` + WITH + look_back_stats AS ( + SELECT + dest_domain.name AS domain_name, + COUNT(*) AS total_messages, + ( + SUM( + mv.destination_tx_gas_used * mv.destination_tx_effective_gas_price + ) / POWER(10, 18) + ) / COUNT(*) AS avg_tx_cost_native, + COUNT(*) / ${LOOK_BACK_DAYS} AS avg_daily_messages + FROM + message_view mv + LEFT JOIN DOMAIN dest_domain ON mv.destination_domain_id = dest_domain.id + WHERE + mv.send_occurred_at >= CURRENT_TIMESTAMP - (INTERVAL '1 day' * ${LOOK_BACK_DAYS}) + AND dest_domain.is_test_net IS FALSE + AND mv.destination_domain_id not in (1408864445, 1399811149) -- ignore sealevel chains as scraper does not capture all costs + AND mv.is_delivered IS TRUE + GROUP BY + dest_domain.name + ) + SELECT + domain_name as chain, + GREATEST( + avg_tx_cost_native * ${MIN_NUMBER_OF_TXS}, + avg_tx_cost_native * avg_daily_messages + ) as daily_burn + FROM + look_back_stats + ORDER BY + domain_name; + `; + return results; +} + +async function getDailyRelayerBurn(sql: Sql) { + const dailyRelayerBurnQueryResults = await fetchDailyRelayerBurnData(sql); + + const burn: Record = {}; + for (const chain of Object.keys(tokenPrices)) { + const row = dailyRelayerBurnQueryResults.find((row) => row.chain === chain); + + // minimum native balance required to maintain our desired minimum dollar balance in the relayer + const minNativeBalance = + RELAYER_MIN_DOLLAR_BALANCE_PER_DAY / parseFloat(tokenPrices[chain]); + + // some chains may have had no messages in the look back window so we set daily burn based on the minimum dollar balance + const proposedDailyRelayerBurn = + row === undefined + ? minNativeBalance + : Math.max(row.daily_burn, minNativeBalance); + + // only update the daily burn if the proposed daily burn is greater than the current daily burn + // add the chain to the daily burn if it doesn't exist + const newDailyRelayerBurn = + chain in currentDailyRelayerBurn + ? Math.max(proposedDailyRelayerBurn, currentDailyRelayerBurn[chain]) + : proposedDailyRelayerBurn; + + burn[chain] = formatDailyRelayerBurn(newDailyRelayerBurn); + } + + return burn; +} + +main().catch((err) => { + rootLogger.error('Error:', err); + process.exit(1); +}); diff --git a/typescript/infra/scripts/funding/read-alert-thresholds.ts b/typescript/infra/scripts/funding/read-alert-thresholds.ts new file mode 100644 index 0000000000..25d6ebe8fc --- /dev/null +++ b/typescript/infra/scripts/funding/read-alert-thresholds.ts @@ -0,0 +1,41 @@ +import yargs from 'yargs'; + +import { rootLogger } from '@hyperlane-xyz/utils'; + +import { THRESHOLD_CONFIG_PATH } from '../../src/config/funding/balances.js'; +import { alertConfigMapping } from '../../src/config/funding/grafanaAlerts.js'; +import { + getAlertThresholds, + sortThresholds, +} from '../../src/funding/grafana.js'; +import { writeJsonAtPath } from '../../src/utils/utils.js'; +import { withAlertTypeRequired, withWrite } from '../agent-utils.js'; + +async function main() { + const { alertType, write } = await withWrite( + withAlertTypeRequired(yargs(process.argv.slice(2))), + ).argv; + + const alertThresholds = await getAlertThresholds(alertType); + const sortedThresholds = sortThresholds(alertThresholds); + + console.table(sortedThresholds); + + if (write) { + rootLogger.info('Writing alert thresholds to file..'); + try { + writeJsonAtPath( + `${THRESHOLD_CONFIG_PATH}/${alertConfigMapping[alertType].configFileName}`, + sortedThresholds, + ); + rootLogger.info('Alert thresholds written to file.'); + } catch (e) { + rootLogger.error('Error writing alert thresholds to file:', e); + } + } +} + +main().catch((err) => { + rootLogger.error(err); + process.exit(1); +}); diff --git a/typescript/infra/scripts/funding/update-balance-threshold-config.ts b/typescript/infra/scripts/funding/update-balance-threshold-config.ts new file mode 100644 index 0000000000..cbc0e20239 --- /dev/null +++ b/typescript/infra/scripts/funding/update-balance-threshold-config.ts @@ -0,0 +1,111 @@ +import { checkbox } from '@inquirer/prompts'; +import yargs from 'yargs'; + +import { ChainMap } from '@hyperlane-xyz/sdk'; +import { rootLogger } from '@hyperlane-xyz/utils'; + +import rawDailyBurn from '../../config/environments/mainnet3/balances/dailyRelayerBurn.json'; +import { + BalanceThresholdType, + THRESHOLD_CONFIG_PATH, + balanceThresholdConfigMapping, +} from '../../src/config/funding/balances.js'; +import { + formatDailyRelayerBurn, + sortThresholds, +} from '../../src/funding/grafana.js'; +import { readJSONAtPath, writeJsonAtPath } from '../../src/utils/utils.js'; +import { + withBalanceThresholdConfig, + withConfirmAllChoices, +} from '../agent-utils.js'; + +const dailyBurn: ChainMap = rawDailyBurn; + +const exclusionList = ['osmosis']; + +async function main() { + const { balanceThresholdConfig, all } = await withConfirmAllChoices( + withBalanceThresholdConfig(yargs(process.argv.slice(2))), + ).argv; + + const configToUpdate: BalanceThresholdType[] = all + ? Object.values(BalanceThresholdType) + : balanceThresholdConfig + ? [balanceThresholdConfig] + : await checkbox({ + message: 'Select the balance threshold config to update', + choices: Object.values(BalanceThresholdType).map((config) => ({ + name: balanceThresholdConfigMapping[config].choiceLabel, + value: config, + checked: true, // default to all checked + })), + }); + + for (const config of configToUpdate) { + rootLogger.info(`Updating ${config} config`); + + let currentThresholds: ChainMap = {}; + const newThresholds: ChainMap = {}; + try { + currentThresholds = readJSONAtPath( + `${THRESHOLD_CONFIG_PATH}/${balanceThresholdConfigMapping[config].configFileName}`, + ); + } catch (e) { + rootLogger.error(`Error reading ${config} config: ${e}`); + } + + // Update the threshold for each chain, if it doesn't exist, create a new one + for (const chain in dailyBurn) { + if (!currentThresholds[chain]) { + // Skip chains in the exclusion list + if (exclusionList.includes(chain)) { + rootLogger.info(`Skipping ${chain} as it is in the exclusion list`); + continue; + } + + newThresholds[chain] = formatDailyRelayerBurn( + dailyBurn[chain] * + balanceThresholdConfigMapping[config].dailyRelayerBurnMultiplier, + ).toString(); + } else { + // This will ensure that chains where the desired threshold is 0 will be unchanged + if ( + config === BalanceThresholdType.RelayerBalance && + parseFloat(currentThresholds[chain]) === 0 + ) { + newThresholds[chain] = currentThresholds[chain]; + continue; + } + + newThresholds[chain] = Math.max( + formatDailyRelayerBurn( + dailyBurn[chain] * + balanceThresholdConfigMapping[config].dailyRelayerBurnMultiplier, + ), + parseFloat(currentThresholds[chain]), + ).toString(); + } + } + + const sortedThresholds = sortThresholds(newThresholds); + + try { + rootLogger.info(`Writing ${config} config to file..`); + writeJsonAtPath( + `${THRESHOLD_CONFIG_PATH}/${balanceThresholdConfigMapping[config].configFileName}`, + sortedThresholds, + ); + rootLogger.info(`Successfully updated ${config} config`); + } catch (e) { + rootLogger.error(`Error writing ${config} config: ${e}`); + } + } +} + +main() + .then() + .catch((e) => { + rootLogger.error(e); + process.exit(1); + }); diff --git a/typescript/infra/scripts/funding/write-alert.ts b/typescript/infra/scripts/funding/write-alert.ts new file mode 100644 index 0000000000..5245fd63fa --- /dev/null +++ b/typescript/infra/scripts/funding/write-alert.ts @@ -0,0 +1,72 @@ +import { checkbox } from '@inquirer/prompts'; +import yargs from 'yargs'; + +import { ChainMap } from '@hyperlane-xyz/sdk'; +import { rootLogger } from '@hyperlane-xyz/utils'; + +import { THRESHOLD_CONFIG_PATH } from '../../src/config/funding/balances.js'; +import { + AlertType, + alertConfigMapping, +} from '../../src/config/funding/grafanaAlerts.js'; +import { + fetchGrafanaAlert, + fetchServiceAccountToken, + generateQuery, + updateGrafanaAlert, +} from '../../src/funding/grafana.js'; +import { readJSONAtPath } from '../../src/utils/utils.js'; +import { withAlertType, withConfirmAllChoices } from '../agent-utils.js'; + +async function main() { + const { alertType, all } = await withConfirmAllChoices( + withAlertType(yargs(process.argv.slice(2))), + ).argv; + + const saToken = await fetchServiceAccountToken(); + + const alertsToUpdate: AlertType[] = all + ? Object.values(AlertType) + : alertType + ? [alertType] + : await checkbox({ + message: 'Select the alert type to update', + choices: Object.values(AlertType).map((alert) => ({ + name: alertConfigMapping[alert].choiceLabel, + value: alert, + checked: true, // default to all checked + })), + }); + + for (const alert of alertsToUpdate) { + // fetch alertRule config from Grafana + const alertRule = await fetchGrafanaAlert(alert, saToken); + + let thresholds: ChainMap = {}; + try { + thresholds = readJSONAtPath( + `${THRESHOLD_CONFIG_PATH}/${alertConfigMapping[alert].configFileName}`, + ); + } catch (e) { + rootLogger.error(`Error reading ${alert} config: ${e}`); + process.exit(1); + } + + const query = generateQuery(alert, thresholds); + + // only change the query + await updateGrafanaAlert( + alertConfigMapping[alert].grafanaAlertId, + alertRule.rawData, + query, + saToken, + ); + + rootLogger.info(`Updated ${alert} alert`); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/typescript/infra/src/agents/key-utils.ts b/typescript/infra/src/agents/key-utils.ts index 46262ed8ba..69392470b3 100644 --- a/typescript/infra/src/agents/key-utils.ts +++ b/typescript/infra/src/agents/key-utils.ts @@ -1,5 +1,4 @@ -import { dirname, join } from 'path'; -import { fileURLToPath } from 'url'; +import { join } from 'path'; import { ChainMap, ChainName } from '@hyperlane-xyz/sdk'; import { Address, objMap, rootLogger } from '@hyperlane-xyz/utils'; diff --git a/typescript/infra/src/config/funding/alert-query-templates.ts b/typescript/infra/src/config/funding/alert-query-templates.ts new file mode 100644 index 0000000000..1ddeabd1c4 --- /dev/null +++ b/typescript/infra/src/config/funding/alert-query-templates.ts @@ -0,0 +1,56 @@ +// alert queries currently need to support special cases i.e cross VM (sealevel and cosmos chains) and ata payer (sealevel). These special cases are hard coded for now. We aim to add cross VM support to the key and will be able to remove special casing in the future +export const LOW_URGENCY_KEY_FUNDER_HEADER = `# Note: use last_over_time(hyperlane_wallet_balance{}[1d]) to be resilient to gaps in the 'hyperlane_wallet_balance' +# that occur due to key funder only running every hour or so. + +min by (chain, wallet_address, wallet_name) ( + # Mainnets`; + +export const LOW_URGENCY_KEY_FUNDER_FOOTER = ` # Mainnets that don't use key-funder and all funds are stored in the relayer's wallet + # Eclipse + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", hyperlane_context="hyperlane", chain=~"eclipsemainnet"}[1d]) - 1 or + # Any ATA payer on Eclipse + last_over_time(hyperlane_wallet_balance{wallet_name=~".*/ata-payer", chain=~"eclipsemainnet"}[1d]) - 0.01 or + # SOL/eclipsemainnet-solanamainnet + last_over_time(hyperlane_wallet_balance{wallet_name=~"SOL/eclipsemainnet-solanamainnet/ata-payer | USDC/eclipsemainnet-ethereum-solanamainnet/ata-payer", chain=~"eclipsemainnet"}[1d]) - 0.1 or + + # Solana + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", hyperlane_context="hyperlane", chain=~"solanamainnet"}[1d]) - 27 or + # Any ATA payer on Solana + last_over_time(hyperlane_wallet_balance{wallet_name=~".*/ata-payer", chain=~"solanamainnet"}[1d]) - 0.2 or + # Any ATA payer on Solana + last_over_time(hyperlane_wallet_balance{wallet_name=~"USDC/eclipsemainnet-ethereum-solanamainnet/ata-payer", chain=~"solanamainnet"}[1d]) - 0.8 or + + # Neutron context + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="mantapacific", hyperlane_context="neutron"}[1d]) - 0.3 or + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="arbitrum", hyperlane_context="neutron"}[1d]) - 0.3 or + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="neutron", hyperlane_context="neutron"}[1d]) - 1500 or + + # Injective + (last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="injective", wallet_address!~"inj1ddw6pm84zmtpms0gpknfsejkk9v6t0ergjpl30|inj1ds32d5t26j7gauvtly86lk6uh06ear3jvqllaw"}[1d]) / 1000000000000) - 3 or + + # Stride + (last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="stride"}[1d])) - 10 +`; + +// TODO: add footer for LOW_URGENCY_ENG_KEY_FUNDER + +export const HIGH_URGENCY_RELAYER_HEADER = `min by (chain, wallet_address, wallet_name) ( + # Mainnets`; + +export const HIGH_URGENCY_RELAYER_FOOTER = ` # Special contexts already have hyperlane_context set correctly + + # Eclipse + last_over_time(hyperlane_wallet_balance{wallet_name=~".*/ata-payer", chain=~"eclipsemainnet"}[1d]) - 0.001 or + + # Solana + last_over_time(hyperlane_wallet_balance{wallet_name=~".*/ata-payer", chain=~"solanamainnet"}[1d]) - 0.1 or + + # SOON + # Any ATA payer on SOON + last_over_time(hyperlane_wallet_balance{wallet_name=~".*/ata-payer", chain=~"soon"}[1d]) - 0.005 or + + # Neutron context lines stay with neutron context + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="mantapacific", hyperlane_context="neutron"}[1d]) - 0.02 or + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="arbitrum", hyperlane_context="neutron"}[1d]) - 0.02 or + last_over_time(hyperlane_wallet_balance{wallet_name="relayer", chain="neutron", hyperlane_context="neutron"}[1d]) - 0.7 +`; diff --git a/typescript/infra/src/config/funding/balances.ts b/typescript/infra/src/config/funding/balances.ts new file mode 100644 index 0000000000..ee0dcda0e7 --- /dev/null +++ b/typescript/infra/src/config/funding/balances.ts @@ -0,0 +1,45 @@ +export enum BalanceThresholdType { + RelayerBalance = 'relayerBalance', + LowUrgencyKeyFunderBalance = 'lowUrgencyKeyFunderBalance', + LowUrgencyEngKeyFunderBalance = 'lowUrgencyEngKeyFunderBalance', + HighUrgencyRelayerBalance = 'highUrgencyRelayerBalance', +} + +interface BalanceThresholdConfig { + configFileName: string; + dailyRelayerBurnMultiplier: number; + choiceLabel: string; +} + +export const THRESHOLD_CONFIG_PATH = './config/environments/mainnet3/balances'; + +const RELAYER_BALANCE_TARGET_DAYS = 8; +const RELAYER_MIN_DOLLAR_BALANCE_TARGET = 25; +export const RELAYER_MIN_DOLLAR_BALANCE_PER_DAY = + RELAYER_MIN_DOLLAR_BALANCE_TARGET / RELAYER_BALANCE_TARGET_DAYS; + +export const balanceThresholdConfigMapping: Record< + BalanceThresholdType, + BalanceThresholdConfig +> = { + [BalanceThresholdType.RelayerBalance]: { + configFileName: 'desiredRelayerBalances.json', + dailyRelayerBurnMultiplier: RELAYER_BALANCE_TARGET_DAYS, + choiceLabel: 'Desired Relayer Balance', + }, + [BalanceThresholdType.LowUrgencyKeyFunderBalance]: { + configFileName: `${[BalanceThresholdType.LowUrgencyKeyFunderBalance]}.json`, + dailyRelayerBurnMultiplier: 12, + choiceLabel: 'Low Urgency Key Funder Balance', + }, + [BalanceThresholdType.LowUrgencyEngKeyFunderBalance]: { + configFileName: `${BalanceThresholdType.LowUrgencyEngKeyFunderBalance}.json`, + dailyRelayerBurnMultiplier: 6, + choiceLabel: 'Low Urgency Eng Key Funder Balance', + }, + [BalanceThresholdType.HighUrgencyRelayerBalance]: { + configFileName: `${BalanceThresholdType.HighUrgencyRelayerBalance}.json`, + dailyRelayerBurnMultiplier: 2, + choiceLabel: 'High Urgency Relayer Balance', + }, +}; diff --git a/typescript/infra/src/config/funding/grafanaAlerts.ts b/typescript/infra/src/config/funding/grafanaAlerts.ts new file mode 100644 index 0000000000..aee95273e6 --- /dev/null +++ b/typescript/infra/src/config/funding/grafanaAlerts.ts @@ -0,0 +1,166 @@ +import { + HIGH_URGENCY_RELAYER_FOOTER, + HIGH_URGENCY_RELAYER_HEADER, + LOW_URGENCY_KEY_FUNDER_FOOTER, + LOW_URGENCY_KEY_FUNDER_HEADER, +} from './alert-query-templates.js'; +import { + BalanceThresholdType, + balanceThresholdConfigMapping, +} from './balances.js'; + +export const GRAFANA_URL = 'https://abacusworks.grafana.net'; + +export enum AlertType { + LowUrgencyKeyFunderBalance = 'lowUrgencyKeyFunderBalance', + LowUrgencyEngKeyFunderBalance = 'lowUrgencyEngKeyFunderBalance', + HighUrgencyRelayerBalance = 'highUrgencyRelayerBalance', +} + +interface AlertConfig { + walletName: WalletName; + grafanaAlertId: string; + configFileName: string; + choiceLabel: string; + queryTemplate: { + header: string; + footer: string; + }; +} + +export enum WalletName { + KeyFunder = 'keyFunder', + Relayer = 'relayer', + // ATAPayer = 'ataPayer', +} + +export const walletNameQueryFormat: Record = { + [WalletName.KeyFunder]: 'key-funder', + [WalletName.Relayer]: 'relayer', + // [WalletName.ATAPayer]: '.*ata-payer +}; + +export const alertConfigMapping: Record = { + [AlertType.LowUrgencyKeyFunderBalance]: { + walletName: WalletName.KeyFunder, + grafanaAlertId: 'ae9z3blz6fj0gb', + configFileName: + balanceThresholdConfigMapping[ + BalanceThresholdType.LowUrgencyKeyFunderBalance + ].configFileName, + choiceLabel: + balanceThresholdConfigMapping[ + BalanceThresholdType.LowUrgencyKeyFunderBalance + ].choiceLabel, + queryTemplate: { + header: LOW_URGENCY_KEY_FUNDER_HEADER, + footer: LOW_URGENCY_KEY_FUNDER_FOOTER, + }, + }, + [AlertType.LowUrgencyEngKeyFunderBalance]: { + walletName: WalletName.KeyFunder, + grafanaAlertId: 'ceb9c63qs7fuoe', + configFileName: + balanceThresholdConfigMapping[ + BalanceThresholdType.LowUrgencyEngKeyFunderBalance + ].configFileName, + choiceLabel: + balanceThresholdConfigMapping[ + BalanceThresholdType.LowUrgencyEngKeyFunderBalance + ].choiceLabel, + queryTemplate: { + header: LOW_URGENCY_KEY_FUNDER_HEADER, + footer: LOW_URGENCY_KEY_FUNDER_FOOTER, + }, + }, + [AlertType.HighUrgencyRelayerBalance]: { + walletName: WalletName.Relayer, + grafanaAlertId: 'beb9c2jwhacqoe', + configFileName: + balanceThresholdConfigMapping[ + BalanceThresholdType.HighUrgencyRelayerBalance + ].configFileName, + choiceLabel: + balanceThresholdConfigMapping[ + BalanceThresholdType.HighUrgencyRelayerBalance + ].choiceLabel, + queryTemplate: { + header: HIGH_URGENCY_RELAYER_HEADER, + footer: HIGH_URGENCY_RELAYER_FOOTER, + }, + }, +}; + +interface NotificationSettings { + receiver: string; + group_by: string[]; +} + +interface AlertQueryModel { + editorMode?: string; + exemplar?: boolean; + expr: string; + instant?: boolean; + intervalMs: number; + legendFormat?: string; + maxDataPoints: number; + range?: boolean; + refId: string; + conditions?: Array<{ + evaluator: { + params: number[]; + type: string; + }; + operator: { + type: string; + }; + query: { + params: any[]; + }; + reducer: { + params: any[]; + type: string; + }; + type: string; + }>; + datasource?: { + name?: string; + type: string; + uid: string; + }; + expression?: string; + type?: string; +} + +interface AlertQuery { + refId: string; + queryType: string; + relativeTimeRange: { + from: number; + to: number; + }; + datasourceUid: string; + model: AlertQueryModel; +} + +// interface defined based on documentation at https://grafana.com/docs/grafana/latest/developers/http_api/alerting_provisioning/#span-idprovisioned-alert-rulespan-provisionedalertrule +export interface ProvisionedAlertRule { + id: number; + uid: string; + orgID: number; + folderUID: string; + ruleGroup: string; + title: string; + condition: string; + data: AlertQuery[]; + noDataState: string; + execErrState: string; + + updated: string; + for: string; + + annotations?: Record; + labels?: Record; + isPaused?: boolean; + notification_settings?: NotificationSettings; +} diff --git a/typescript/infra/src/funding/grafana.ts b/typescript/infra/src/funding/grafana.ts new file mode 100644 index 0000000000..2f1763396f --- /dev/null +++ b/typescript/infra/src/funding/grafana.ts @@ -0,0 +1,207 @@ +import { ChainMap } from '@hyperlane-xyz/sdk'; +import { rootLogger } from '@hyperlane-xyz/utils'; + +import { + AlertType, + GRAFANA_URL, + ProvisionedAlertRule, + WalletName, + alertConfigMapping, + walletNameQueryFormat, +} from '../config/funding/grafanaAlerts.js'; +import { fetchGCPSecret } from '../utils/gcloud.js'; + +export const logger = rootLogger.child({ module: 'grafana' }); + +export function formatDailyRelayerBurn(dailyRelayerBurn: number): number { + return Number(dailyRelayerBurn.toPrecision(3)); +} + +export async function fetchGrafanaAlert(alertType: AlertType, saToken: string) { + const response = await fetch( + `${GRAFANA_URL}/api/v1/provisioning/alert-rules/${alertConfigMapping[alertType].grafanaAlertId}`, + { + headers: { + Authorization: `Bearer ${saToken}`, + 'Content-Type': 'application/json', + }, + }, + ); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = (await response.json()) as ProvisionedAlertRule; + + const queries = data.data.map((d) => d.model.expr); + + return { + title: data.title, + queries, + rawData: data, + }; +} + +export async function exportGrafanaAlert( + alertType: AlertType, + saToken: string, + format: string = 'json', +) { + const response = await fetch( + `${GRAFANA_URL}/api/v1/provisioning/alert-rules/${alertConfigMapping[alertType].grafanaAlertId}/export?format=${format}`, + { + headers: { + Authorization: `Bearer ${saToken}`, + 'Content-Type': 'application/json', + }, + }, + ); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return response; +} + +function parsePromQLQuery( + query: string, + walletName: WalletName, +): ChainMap { + const balances: ChainMap = {}; + const alertRegex = getAlertRegex(walletName); + + // Get all matches + const matches = Array.from(query.matchAll(alertRegex)); + for (const match of matches) { + const [_, chain, balanceStr] = match; + const minBalance = balanceStr; + + balances[chain] = minBalance; + } + + return Object.fromEntries(Object.entries(balances).sort()); +} + +function getAlertRegex(walletName: WalletName): RegExp { + switch (walletName) { + case WalletName.KeyFunder: + return /wallet_name="key-funder", chain="([^"]+)"[^-]+ - ([0-9.]+)/g; + case WalletName.Relayer: + return /wallet_name="relayer", chain="([^"]+)"[^-]+ - ([0-9.]+)/g; + default: + throw new Error(`Unknown wallet name: ${walletName}`); + } +} + +export async function getAlertThresholds( + alertType: AlertType, +): Promise> { + const saToken = await fetchServiceAccountToken(); + const alert = await fetchGrafanaAlert(alertType, saToken); + const alertQuery = alert.queries[0]; + const walletName = alertConfigMapping[alertType].walletName; + return parsePromQLQuery(alertQuery, walletName); +} + +export async function fetchServiceAccountToken(): Promise { + let saToken: string | undefined; + + try { + saToken = (await fetchGCPSecret( + 'grafana-balance-alert-thresholds-token', + false, + )) as string; + } catch (error) { + logger.error( + 'Error fetching grafana service account token from GCP secrets:', + error, + ); + throw error; + } + + return saToken; +} + +export async function updateGrafanaAlert( + alertUid: string, + existingAlert: ProvisionedAlertRule, + newQuery: string, + saToken: string, +) { + // Create the updated rule based on the existing one + const updatedRule: ProvisionedAlertRule = { + ...existingAlert, + data: existingAlert.data.map((d) => ({ + ...d, + model: { + ...d.model, + expr: newQuery, + }, + })), + }; + + const response = await fetch( + `${GRAFANA_URL}/api/v1/provisioning/alert-rules/${alertUid}`, + { + method: 'PUT', + headers: { + Authorization: `Bearer ${saToken}`, + 'Content-Type': 'application/json', + 'X-Disable-Provenance': 'true', + }, + body: JSON.stringify(updatedRule), + }, + ); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error( + `Failed to update alert: ${response.status} ${JSON.stringify(errorData)}`, + ); + } + + return response.json(); +} + +export function generateQuery( + alertType: AlertType, + thresholds: ChainMap, +): string { + const config = alertConfigMapping[alertType]; + const walletQueryName = walletNameQueryFormat[config.walletName]; + + // TODO: abstract away special handling for relayer queries that need hyperlane_context + const needsHyperlaneContext = config.walletName === WalletName.Relayer; + + const queryFragments = Object.entries(thresholds).map( + ([chain, minBalance]) => { + const labels = [`wallet_name="${walletQueryName}"`, `chain="${chain}"`]; + if (needsHyperlaneContext) { + labels.push('hyperlane_context="hyperlane"'); + } + return `last_over_time(hyperlane_wallet_balance{${labels.join( + ', ', + )}}[1d]) - ${minBalance} or`; + }, + ); + + return `${config.queryTemplate.header} + ${queryFragments.join('\n ')} + +${config.queryTemplate.footer} +)`; +} + +export function sortThresholds( + newThresholds: ChainMap, +): ChainMap { + const orderedThresholds: ChainMap = {}; + Object.keys(newThresholds) + .sort() + .forEach((key) => { + orderedThresholds[key] = newThresholds[key]; + }); + return orderedThresholds; +} diff --git a/yarn.lock b/yarn.lock index 75ff7cba40..7e68e2e31a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7503,6 +7503,7 @@ __metadata: hardhat: "npm:^2.22.2" json-stable-stringify: "npm:^1.1.1" mocha: "npm:^10.2.0" + postgres: "npm:^3.4.5" prettier: "npm:^2.8.8" prom-client: "npm:^14.0.1" prompts: "npm:^2.4.2" @@ -30747,6 +30748,13 @@ __metadata: languageName: node linkType: hard +"postgres@npm:^3.4.5": + version: 3.4.5 + resolution: "postgres@npm:3.4.5" + checksum: 10/b51341a8640bd63c42caf2a866c8b6bcac0d4cc1b85985f01eb3fd941eb7bf3fe5200bcf6983eaa1d42ae7555d85f4ff51d35916b5543b95aecfe98b76213611 + languageName: node + linkType: hard + "preact@npm:^10.16.0, preact@npm:^10.24.2": version: 10.24.3 resolution: "preact@npm:10.24.3"