From 9a22c7a29e97a131e4f88526738baa8916bf36d3 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Mon, 30 Dec 2024 04:06:12 +0100 Subject: [PATCH 01/29] Poll failed invoices with visibility timeout --- api/resolvers/wallet.js | 31 ++++++++++++++++++ api/typeDefs/wallet.js | 1 + fragments/wallet.js | 9 ++++++ lib/apollo.js | 6 ++++ .../migration.sql | 2 ++ prisma/schema.prisma | 1 + wallets/index.js | 32 +++++++++++++++++-- worker/index.js | 4 ++- worker/wallet.js | 4 +++ 9 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 prisma/migrations/20251201010948_invoice_locked_at/migration.sql diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index c71749ac0..de378ff5c 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -456,6 +456,37 @@ const resolvers = { cursor: nextCursor, entries: logs } + }, + failedInvoices: async (parent, args, { me, models }) => { + if (!me) { + throw new GqlAuthenticationError() + } + // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED + return await models.$queryRaw` + WITH failed AS ( + UPDATE "Invoice" + SET "lockedAt" = now() + WHERE id IN ( + SELECT id FROM "Invoice" + WHERE "userId" = ${me.id} + AND "actionState" = 'FAILED' + AND "userCancel" = false + AND "lockedAt" IS NULL + ORDER BY id DESC + FOR UPDATE SKIP LOCKED + ) + RETURNING * + ), + _ AS ( + INSERT INTO pgboss.job (name, data, startafter, keepuntil) + SELECT + 'unlockInvoice', + jsonb_build_object('id', id), + now() + interval '10 minutes', + now() + interval '15 minutes' + FROM failed + ) + SELECT * FROM failed` } }, Wallet: { diff --git a/api/typeDefs/wallet.js b/api/typeDefs/wallet.js index 3006fc7a4..efc49130b 100644 --- a/api/typeDefs/wallet.js +++ b/api/typeDefs/wallet.js @@ -72,6 +72,7 @@ const typeDefs = ` wallet(id: ID!): Wallet walletByType(type: String!): Wallet walletLogs(type: String, from: String, to: String, cursor: String): WalletLog! + failedInvoices: [Invoice!]! } extend type Mutation { diff --git a/fragments/wallet.js b/fragments/wallet.js index f75d6547e..b4d34d30a 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql` } } ` + +export const FAILED_INVOICES = gql` + ${INVOICE_FIELDS} + query FailedInvoices { + failedInvoices { + ...InvoiceFields + } + } +` diff --git a/lib/apollo.js b/lib/apollo.js index c3b7a6f5b..ecfd53f9e 100644 --- a/lib/apollo.js +++ b/lib/apollo.js @@ -266,6 +266,12 @@ function getClient (uri) { facts: [...(existing?.facts || []), ...incoming.facts] } } + }, + failedInvoices: { + keyArgs: [], + merge (existing, incoming) { + return incoming + } } } }, diff --git a/prisma/migrations/20251201010948_invoice_locked_at/migration.sql b/prisma/migrations/20251201010948_invoice_locked_at/migration.sql new file mode 100644 index 000000000..439fc394f --- /dev/null +++ b/prisma/migrations/20251201010948_invoice_locked_at/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Invoice" ADD COLUMN "lockedAt" TIMESTAMP(3); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8e0e64eab..c89e302d8 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -927,6 +927,7 @@ model Invoice { cancelled Boolean @default(false) cancelledAt DateTime? userCancel Boolean? + lockedAt DateTime? msatsRequested BigInt msatsReceived BigInt? desc String? diff --git a/wallets/index.js b/wallets/index.js index bdf00083b..993be4d69 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,6 +1,6 @@ import { useMe } from '@/components/me' -import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' -import { SSR } from '@/lib/constants' +import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' +import { FAST_POLL_INTERVAL, SSR } from '@/lib/constants' import { useApolloClient, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common' @@ -204,7 +204,9 @@ export function WalletsProvider ({ children }) { removeLocalWallets }} > - {children} + + {children} + ) } @@ -225,3 +227,27 @@ export function useSendWallets () { .filter(w => !w.def.isAvailable || w.def.isAvailable()) .filter(w => w.config?.enabled && canSend(w)) } + +function RetryHandler ({ children }) { + const failedInvoices = useFailedInvoices() + + useEffect(() => { + // TODO: retry + }, [failedInvoices]) + + return children +} + +function useFailedInvoices () { + const wallets = useSendWallets() + + // TODO: use longer poll interval in prod? + const { data } = useQuery(FAILED_INVOICES, { + pollInterval: FAST_POLL_INTERVAL, + fetchPolicy: 'no-cache', + nextFetchPolicy: 'no-cache', + skip: wallets.length === 0 + }) + + return data?.failedInvoices ?? [] +} diff --git a/worker/index.js b/worker/index.js index dfb6c4f97..07dce5b8e 100644 --- a/worker/index.js +++ b/worker/index.js @@ -5,7 +5,8 @@ import createPrisma from '@/lib/create-prisma' import { checkInvoice, checkPendingDeposits, checkPendingWithdrawals, checkWithdrawal, - finalizeHodlInvoice, subscribeToWallet + finalizeHodlInvoice, subscribeToWallet, + unlockInvoice } from './wallet' import { repin } from './repin' import { trust } from './trust' @@ -102,6 +103,7 @@ async function work () { await boss.work('autoWithdraw', jobWrapper(autoWithdraw)) await boss.work('checkInvoice', jobWrapper(checkInvoice)) await boss.work('checkWithdrawal', jobWrapper(checkWithdrawal)) + await boss.work('unlockInvoice', jobWrapper(unlockInvoice)) // paidAction jobs await boss.work('paidActionForwarding', jobWrapper(paidActionForwarding)) await boss.work('paidActionForwarded', jobWrapper(paidActionForwarded)) diff --git a/worker/wallet.js b/worker/wallet.js index ac09c7ac4..6264bce6e 100644 --- a/worker/wallet.js +++ b/worker/wallet.js @@ -284,3 +284,7 @@ export async function checkPendingWithdrawals (args) { } } } + +export async function unlockInvoice ({ data: { id }, models }) { + await models.invoice.update({ where: { id }, data: { lockedAt: null } }) +} From 0dbdd22fd3361eed27d0363ae25b69ac80f50a8d Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 2 Jan 2025 16:24:03 +0100 Subject: [PATCH 02/29] Don't return intermediate failed invoices --- api/resolvers/wallet.js | 4 +++- lib/constants.js | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index de378ff5c..320a19c43 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -9,7 +9,8 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib import { USER_ID, INVOICE_RETENTION_DAYS, PAID_ACTION_PAYMENT_METHODS, - WALLET_CREATE_INVOICE_TIMEOUT_MS + WALLET_CREATE_INVOICE_TIMEOUT_MS, + WALLET_RETRY_AFTER_MS } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' import assertGofacYourself from './ofac' @@ -471,6 +472,7 @@ const resolvers = { WHERE "userId" = ${me.id} AND "actionState" = 'FAILED' AND "userCancel" = false + AND "cancelledAt" < now() - interval '${WALLET_RETRY_AFTER_MS} milliseconds' AND "lockedAt" IS NULL ORDER BY id DESC FOR UPDATE SKIP LOCKED diff --git a/lib/constants.js b/lib/constants.js index 48cf0cda0..c81481233 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -197,5 +197,9 @@ export const ZAP_UNDO_DELAY_MS = 5_000 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 +// When should failed invoices be returned to a client to retry? +// This must be high enough such that intermediate failed invoices that will be retried +// by the client due to sender or receiver fallbacks are not returned to the client. +export const WALLET_RETRY_AFTER_MS = 60_000 export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l' From c8970610d386b68f735011a68a28d7dac3bcd105 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Mon, 6 Jan 2025 23:19:40 +0100 Subject: [PATCH 03/29] Don't retry too old invoices --- api/resolvers/wallet.js | 19 ++++++++++++++----- lib/constants.js | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index 320a19c43..8bcae59b8 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -10,7 +10,8 @@ import { USER_ID, INVOICE_RETENTION_DAYS, PAID_ACTION_PAYMENT_METHODS, WALLET_CREATE_INVOICE_TIMEOUT_MS, - WALLET_RETRY_AFTER_MS + WALLET_RETRY_AFTER_MS, + WALLET_RETRY_BEFORE_MS } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' import assertGofacYourself from './ofac' @@ -463,16 +464,21 @@ const resolvers = { throw new GqlAuthenticationError() } // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED - return await models.$queryRaw` + // we use queryRawUnsafe because Prisma does not support tsrange + // see https://www.prisma.io/docs/orm/overview/databases/postgresql + return await models.$queryRawUnsafe(` WITH failed AS ( UPDATE "Invoice" SET "lockedAt" = now() WHERE id IN ( SELECT id FROM "Invoice" - WHERE "userId" = ${me.id} + WHERE "userId" = $1 AND "actionState" = 'FAILED' AND "userCancel" = false - AND "cancelledAt" < now() - interval '${WALLET_RETRY_AFTER_MS} milliseconds' + AND now()::timestamp <@ tsrange( + "cancelledAt" + $2::interval, + "cancelledAt" + $3::interval + ) AND "lockedAt" IS NULL ORDER BY id DESC FOR UPDATE SKIP LOCKED @@ -488,7 +494,10 @@ const resolvers = { now() + interval '15 minutes' FROM failed ) - SELECT * FROM failed` + SELECT * FROM failed`, + me.id, + `${WALLET_RETRY_AFTER_MS} milliseconds`, + `${WALLET_RETRY_BEFORE_MS} milliseconds`) } }, Wallet: { diff --git a/lib/constants.js b/lib/constants.js index c81481233..e6ca9f020 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -201,5 +201,6 @@ export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 // This must be high enough such that intermediate failed invoices that will be retried // by the client due to sender or receiver fallbacks are not returned to the client. export const WALLET_RETRY_AFTER_MS = 60_000 +export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l' From 073c5b2ea70a697af85ad3bc2f9f30e218b57552 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Tue, 7 Jan 2025 02:57:33 +0100 Subject: [PATCH 04/29] Retry invoices on client --- components/use-invoice.js | 6 +++--- wallets/index.js | 23 ++++++++++++++++------- wallets/payment.js | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/components/use-invoice.js b/components/use-invoice.js index 1cbe94dfd..534426358 100644 --- a/components/use-invoice.js +++ b/components/use-invoice.js @@ -1,5 +1,5 @@ import { useApolloClient, useMutation } from '@apollo/client' -import { useCallback } from 'react' +import { useCallback, useMemo } from 'react' import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors' import { RETRY_PAID_ACTION } from '@/fragments/paidAction' import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet' @@ -42,7 +42,7 @@ export default function useInvoice () { return data.cancelInvoice }, [cancelInvoice]) - const retry = useCallback(async ({ id, hash, hmac }, { update }) => { + const retry = useCallback(async ({ id, hash, hmac }, { update } = {}) => { console.log('retrying invoice:', hash) const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update }) if (error) throw error @@ -53,5 +53,5 @@ export default function useInvoice () { return newInvoice }, [retryPaidAction]) - return { cancel, retry, isInvoice } + return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice]) } diff --git a/wallets/index.js b/wallets/index.js index 993be4d69..b2822b963 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -7,6 +7,8 @@ import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigur import useVault from '@/components/vault/use-vault' import walletDefs from '@/wallets/client' import { generateMutation } from './graphql' +import { useWalletPayment } from './payment' +import useInvoice from '@/components/use-invoice' const WalletsContext = createContext({ wallets: [] @@ -223,17 +225,25 @@ export function useWallet (name) { export function useSendWallets () { const { wallets } = useWallets() // return all enabled wallets that are available and can send - return wallets + return useMemo(() => wallets .filter(w => !w.def.isAvailable || w.def.isAvailable()) - .filter(w => w.config?.enabled && canSend(w)) + .filter(w => w.config?.enabled && canSend(w)), [wallets]) } function RetryHandler ({ children }) { const failedInvoices = useFailedInvoices() + const waitForWalletPayment = useWalletPayment() + const invoiceHelper = useInvoice() useEffect(() => { - // TODO: retry - }, [failedInvoices]) + (async () => { + for (const invoice of failedInvoices) { + // TODO: don't retry forever + const newInvoice = await invoiceHelper.retry(invoice) + waitForWalletPayment(newInvoice).catch(console.error) + } + })() + }, [failedInvoices, invoiceHelper, waitForWalletPayment]) return children } @@ -244,9 +254,8 @@ function useFailedInvoices () { // TODO: use longer poll interval in prod? const { data } = useQuery(FAILED_INVOICES, { pollInterval: FAST_POLL_INTERVAL, - fetchPolicy: 'no-cache', - nextFetchPolicy: 'no-cache', - skip: wallets.length === 0 + skip: wallets.length === 0, + notifyOnNetworkStatusChange: true }) return data?.failedInvoices ?? [] diff --git a/wallets/payment.js b/wallets/payment.js index 043d57f89..7cb0bec64 100644 --- a/wallets/payment.js +++ b/wallets/payment.js @@ -17,7 +17,7 @@ export function useWalletPayment () { const loggerFactory = useWalletLoggerFactory() const invoiceHelper = useInvoice() - return useCallback(async (invoice, { waitFor, updateOnFallback }) => { + return useCallback(async (invoice, { waitFor, updateOnFallback } = {}) => { let aggregateError = new WalletAggregateError([]) let latestInvoice = invoice From 6e48349b04a62d73513e205e17cf7a490d48fcdf Mon Sep 17 00:00:00 2001 From: ekzyis Date: Tue, 7 Jan 2025 10:12:19 +0100 Subject: [PATCH 05/29] Only attempt payment 3 times --- api/paidAction/index.js | 7 +++++-- api/resolvers/notifications.js | 2 ++ api/resolvers/paidAction.js | 6 +++++- api/resolvers/wallet.js | 7 +++++-- lib/constants.js | 2 ++ .../migration.sql | 2 ++ prisma/schema.prisma | 1 + wallets/server.js | 15 ++++++++++----- 8 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 prisma/migrations/20250107084543_invoice_retry_count/migration.sql diff --git a/api/paidAction/index.js b/api/paidAction/index.js index ad067a3ca..473d4bc13 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -328,7 +328,9 @@ export async function retryPaidAction (actionType, args, incomingContext) { me: await models.user.findUnique({ where: { id: parseInt(me.id) } }), cost: BigInt(msatsRequested), actionId, - predecessorId: failedInvoice.id + predecessorId: failedInvoice.id, + // a locked invoice means we're retrying a payment from the beginning with all sender and receiver wallets + retry: failedInvoice.lockedAt ? failedInvoice.retry + 1 : failedInvoice.retry } let invoiceArgs @@ -419,7 +421,7 @@ async function createSNInvoice (actionType, args, context) { } async function createDbInvoice (actionType, args, context) { - const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context + const { me, models, tx, cost, optimistic, actionId, invoiceArgs, retry, predecessorId } = context const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs const db = tx ?? models @@ -445,6 +447,7 @@ async function createDbInvoice (actionType, args, context) { actionArgs: args, expiresAt, actionId, + retry, predecessorId } diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 26e8c4872..7ed345da4 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate' import { replyToSubscription } from '@/lib/webPush' import { getSub } from './sub' import { GqlAuthenticationError, GqlInputError } from '@/lib/error' +import { WALLET_MAX_RETRIES } from '@/lib/constants' export default { Query: { @@ -350,6 +351,7 @@ export default { WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' + AND "Invoice"."retry" >= ${WALLET_MAX_RETRIES} AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 2b993c1f0..bc2f5f50c 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -1,5 +1,5 @@ import { retryPaidAction } from '../paidAction' -import { USER_ID } from '@/lib/constants' +import { USER_ID, WALLET_MAX_RETRIES } from '@/lib/constants' function paidActionType (actionType) { switch (actionType) { @@ -67,6 +67,10 @@ export default { throw new Error(`Invoice is not in failed state: ${invoice.actionState}`) } + if (invoice.retry >= WALLET_MAX_RETRIES) { + throw new Error('Payment has been retried too many times') + } + const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd }) return { diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index 8bcae59b8..daf8441d2 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -11,7 +11,8 @@ import { PAID_ACTION_PAYMENT_METHODS, WALLET_CREATE_INVOICE_TIMEOUT_MS, WALLET_RETRY_AFTER_MS, - WALLET_RETRY_BEFORE_MS + WALLET_RETRY_BEFORE_MS, + WALLET_MAX_RETRIES } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' import assertGofacYourself from './ofac' @@ -480,6 +481,7 @@ const resolvers = { "cancelledAt" + $3::interval ) AND "lockedAt" IS NULL + AND "retry" < $4 ORDER BY id DESC FOR UPDATE SKIP LOCKED ) @@ -497,7 +499,8 @@ const resolvers = { SELECT * FROM failed`, me.id, `${WALLET_RETRY_AFTER_MS} milliseconds`, - `${WALLET_RETRY_BEFORE_MS} milliseconds`) + `${WALLET_RETRY_BEFORE_MS} milliseconds`, + WALLET_MAX_RETRIES) } }, Wallet: { diff --git a/lib/constants.js b/lib/constants.js index e6ca9f020..5ed604245 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -202,5 +202,7 @@ export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 // by the client due to sender or receiver fallbacks are not returned to the client. export const WALLET_RETRY_AFTER_MS = 60_000 export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours +// we want to attempt a payment three times so we retry two times +export const WALLET_MAX_RETRIES = 2 export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l' diff --git a/prisma/migrations/20250107084543_invoice_retry_count/migration.sql b/prisma/migrations/20250107084543_invoice_retry_count/migration.sql new file mode 100644 index 000000000..3a9765862 --- /dev/null +++ b/prisma/migrations/20250107084543_invoice_retry_count/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Invoice" ADD COLUMN "retry" INTEGER NOT NULL DEFAULT 0; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c89e302d8..e9c4af15f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -928,6 +928,7 @@ model Invoice { cancelledAt DateTime? userCancel Boolean? lockedAt DateTime? + retry Int @default(0) msatsRequested BigInt msatsReceived BigInt? desc String? diff --git a/wallets/server.js b/wallets/server.js index bcf1078c1..9d49e9863 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -24,9 +24,13 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln] const MAX_PENDING_INVOICES_PER_WALLET = 25 -export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) { +export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { retry, predecessorId, models }) { // get the wallets in order of priority - const wallets = await getInvoiceableWallets(userId, { predecessorId, models }) + const wallets = await getInvoiceableWallets(userId, { + retry, + predecessorId, + models + }) msats = toPositiveNumber(msats) @@ -79,7 +83,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa export async function createWrappedInvoice (userId, { msats, feePercent, description, descriptionHash, expiry = 360 }, - { predecessorId, models, me, lnd }) { + { retry, predecessorId, models, me, lnd }) { let logger, bolt11 try { const { invoice, wallet } = await createInvoice(userId, { @@ -88,7 +92,7 @@ export async function createWrappedInvoice (userId, description, descriptionHash, expiry - }, { predecessorId, models }) + }, { retry, predecessorId, models }) logger = walletLogger({ wallet, models }) bolt11 = invoice @@ -108,7 +112,7 @@ export async function createWrappedInvoice (userId, } } -export async function getInvoiceableWallets (userId, { predecessorId, models }) { +export async function getInvoiceableWallets (userId, { retry, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it // so it has not been updated yet. @@ -141,6 +145,7 @@ export async function getInvoiceableWallets (userId, { predecessorId, models }) FROM "Invoice" JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId" WHERE "Invoice"."actionState" = 'RETRYING' + AND "Invoice"."retry" = ${retry} ) SELECT "InvoiceForward"."walletId" From c101b6068407752896db4b4487a7d4e3cee97ae1 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 8 Jan 2025 20:11:39 +0100 Subject: [PATCH 06/29] Fix fallbacks during last retry --- api/paidAction/index.js | 4 +--- api/resolvers/paidAction.js | 7 +++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/paidAction/index.js b/api/paidAction/index.js index 473d4bc13..5539bd0b6 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -328,9 +328,7 @@ export async function retryPaidAction (actionType, args, incomingContext) { me: await models.user.findUnique({ where: { id: parseInt(me.id) } }), cost: BigInt(msatsRequested), actionId, - predecessorId: failedInvoice.id, - // a locked invoice means we're retrying a payment from the beginning with all sender and receiver wallets - retry: failedInvoice.lockedAt ? failedInvoice.retry + 1 : failedInvoice.retry + predecessorId: failedInvoice.id } let invoiceArgs diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index bc2f5f50c..986bc53c2 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -67,11 +67,14 @@ export default { throw new Error(`Invoice is not in failed state: ${invoice.actionState}`) } - if (invoice.retry >= WALLET_MAX_RETRIES) { + // a locked invoice means we want to retry a payment from the beginning + // with all sender and receiver wallets so we need to increment the retry count + const retry = invoice.lockedAt ? invoice.retry + 1 : invoice.retry + if (retry > WALLET_MAX_RETRIES) { throw new Error('Payment has been retried too many times') } - const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd }) + const result = await retryPaidAction(invoice.actionType, { invoice }, { retry, models, me, lnd }) return { ...result, From 64e3efca4b18d4ad8adc30aa97f5dbecec66e2a5 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 8 Jan 2025 20:20:47 +0100 Subject: [PATCH 07/29] Rename retry column to paymentAttempt --- api/paidAction/index.js | 4 ++-- api/resolvers/notifications.js | 2 +- api/resolvers/paidAction.js | 6 +++--- api/resolvers/wallet.js | 2 +- .../20250107084543_automated_retries/migration.sql | 3 +++ .../20250107084543_invoice_retry_count/migration.sql | 2 -- .../20251201010948_invoice_locked_at/migration.sql | 2 -- prisma/schema.prisma | 2 +- wallets/server.js | 12 ++++++------ 9 files changed, 17 insertions(+), 18 deletions(-) create mode 100644 prisma/migrations/20250107084543_automated_retries/migration.sql delete mode 100644 prisma/migrations/20250107084543_invoice_retry_count/migration.sql delete mode 100644 prisma/migrations/20251201010948_invoice_locked_at/migration.sql diff --git a/api/paidAction/index.js b/api/paidAction/index.js index 5539bd0b6..8feabc306 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) { } async function createDbInvoice (actionType, args, context) { - const { me, models, tx, cost, optimistic, actionId, invoiceArgs, retry, predecessorId } = context + const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs const db = tx ?? models @@ -445,7 +445,7 @@ async function createDbInvoice (actionType, args, context) { actionArgs: args, expiresAt, actionId, - retry, + paymentAttempt, predecessorId } diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 7ed345da4..eec24f60a 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -351,7 +351,7 @@ export default { WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' - AND "Invoice"."retry" >= ${WALLET_MAX_RETRIES} + AND "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 986bc53c2..7a5184181 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -69,12 +69,12 @@ export default { // a locked invoice means we want to retry a payment from the beginning // with all sender and receiver wallets so we need to increment the retry count - const retry = invoice.lockedAt ? invoice.retry + 1 : invoice.retry - if (retry > WALLET_MAX_RETRIES) { + const paymentAttempt = invoice.lockedAt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt + if (paymentAttempt > WALLET_MAX_RETRIES) { throw new Error('Payment has been retried too many times') } - const result = await retryPaidAction(invoice.actionType, { invoice }, { retry, models, me, lnd }) + const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) return { ...result, diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index daf8441d2..0eb36dfd4 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -481,7 +481,7 @@ const resolvers = { "cancelledAt" + $3::interval ) AND "lockedAt" IS NULL - AND "retry" < $4 + AND "paymentAttempt" < $4 ORDER BY id DESC FOR UPDATE SKIP LOCKED ) diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql new file mode 100644 index 000000000..338aae76e --- /dev/null +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "Invoice" ADD COLUMN "lockedAt" TIMESTAMP(3); diff --git a/prisma/migrations/20250107084543_invoice_retry_count/migration.sql b/prisma/migrations/20250107084543_invoice_retry_count/migration.sql deleted file mode 100644 index 3a9765862..000000000 --- a/prisma/migrations/20250107084543_invoice_retry_count/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "Invoice" ADD COLUMN "retry" INTEGER NOT NULL DEFAULT 0; diff --git a/prisma/migrations/20251201010948_invoice_locked_at/migration.sql b/prisma/migrations/20251201010948_invoice_locked_at/migration.sql deleted file mode 100644 index 439fc394f..000000000 --- a/prisma/migrations/20251201010948_invoice_locked_at/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "Invoice" ADD COLUMN "lockedAt" TIMESTAMP(3); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e9c4af15f..027622549 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -928,7 +928,7 @@ model Invoice { cancelledAt DateTime? userCancel Boolean? lockedAt DateTime? - retry Int @default(0) + paymentAttempt Int @default(0) msatsRequested BigInt msatsReceived BigInt? desc String? diff --git a/wallets/server.js b/wallets/server.js index 9d49e9863..58c00c11b 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -24,10 +24,10 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln] const MAX_PENDING_INVOICES_PER_WALLET = 25 -export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { retry, predecessorId, models }) { +export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { paymentAttempt, predecessorId, models }) { // get the wallets in order of priority const wallets = await getInvoiceableWallets(userId, { - retry, + paymentAttempt, predecessorId, models }) @@ -83,7 +83,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa export async function createWrappedInvoice (userId, { msats, feePercent, description, descriptionHash, expiry = 360 }, - { retry, predecessorId, models, me, lnd }) { + { paymentAttempt, predecessorId, models, me, lnd }) { let logger, bolt11 try { const { invoice, wallet } = await createInvoice(userId, { @@ -92,7 +92,7 @@ export async function createWrappedInvoice (userId, description, descriptionHash, expiry - }, { retry, predecessorId, models }) + }, { paymentAttempt, predecessorId, models }) logger = walletLogger({ wallet, models }) bolt11 = invoice @@ -112,7 +112,7 @@ export async function createWrappedInvoice (userId, } } -export async function getInvoiceableWallets (userId, { retry, predecessorId, models }) { +export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it // so it has not been updated yet. @@ -145,7 +145,7 @@ export async function getInvoiceableWallets (userId, { retry, predecessorId, mod FROM "Invoice" JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId" WHERE "Invoice"."actionState" = 'RETRYING' - AND "Invoice"."retry" = ${retry} + AND "Invoice"."paymentAttempt" = ${paymentAttempt} ) SELECT "InvoiceForward"."walletId" From dc7295400a15583e46083aa8afab50cf322a2f99 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 9 Jan 2025 01:37:09 +0100 Subject: [PATCH 08/29] Fix no index used --- api/resolvers/wallet.js | 18 ++++++------------ .../migration.sql | 1 + prisma/schema.prisma | 1 + 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index 0eb36dfd4..e60d2e00f 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -467,21 +467,19 @@ const resolvers = { // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED // we use queryRawUnsafe because Prisma does not support tsrange // see https://www.prisma.io/docs/orm/overview/databases/postgresql - return await models.$queryRawUnsafe(` + return await models.$queryRaw` WITH failed AS ( UPDATE "Invoice" SET "lockedAt" = now() WHERE id IN ( SELECT id FROM "Invoice" - WHERE "userId" = $1 + WHERE "userId" = ${me.id} AND "actionState" = 'FAILED' AND "userCancel" = false - AND now()::timestamp <@ tsrange( - "cancelledAt" + $2::interval, - "cancelledAt" + $3::interval - ) + AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval + AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval AND "lockedAt" IS NULL - AND "paymentAttempt" < $4 + AND "paymentAttempt" < ${WALLET_MAX_RETRIES} ORDER BY id DESC FOR UPDATE SKIP LOCKED ) @@ -496,11 +494,7 @@ const resolvers = { now() + interval '15 minutes' FROM failed ) - SELECT * FROM failed`, - me.id, - `${WALLET_RETRY_AFTER_MS} milliseconds`, - `${WALLET_RETRY_BEFORE_MS} milliseconds`, - WALLET_MAX_RETRIES) + SELECT * FROM failed` } }, Wallet: { diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql index 338aae76e..17770e9e3 100644 --- a/prisma/migrations/20250107084543_automated_retries/migration.sql +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -1,3 +1,4 @@ -- AlterTable ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; ALTER TABLE "Invoice" ADD COLUMN "lockedAt" TIMESTAMP(3); +CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 027622549..da186ade9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -957,6 +957,7 @@ model Invoice { @@index([confirmedIndex], map: "Invoice.confirmedIndex_index") @@index([isHeld]) @@index([confirmedAt]) + @@index([cancelledAt]) @@index([actionType]) @@index([actionState]) } From 670777732340bb44120f35358b2ee3474cc71613 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 9 Jan 2025 02:28:14 +0100 Subject: [PATCH 09/29] Resolve TODOs --- wallets/index.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wallets/index.js b/wallets/index.js index b2822b963..83c6c13a0 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,6 +1,6 @@ import { useMe } from '@/components/me' import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' -import { FAST_POLL_INTERVAL, SSR } from '@/lib/constants' +import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants' import { useApolloClient, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common' @@ -238,7 +238,6 @@ function RetryHandler ({ children }) { useEffect(() => { (async () => { for (const invoice of failedInvoices) { - // TODO: don't retry forever const newInvoice = await invoiceHelper.retry(invoice) waitForWalletPayment(newInvoice).catch(console.error) } @@ -251,9 +250,8 @@ function RetryHandler ({ children }) { function useFailedInvoices () { const wallets = useSendWallets() - // TODO: use longer poll interval in prod? const { data } = useQuery(FAILED_INVOICES, { - pollInterval: FAST_POLL_INTERVAL, + pollInterval: NORMAL_POLL_INTERVAL, skip: wallets.length === 0, notifyOnNetworkStatusChange: true }) From d448a1a9dbbaed552cd4456e6f2a5568053623f2 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Mon, 13 Jan 2025 11:44:17 +0100 Subject: [PATCH 10/29] Use expiring locks --- api/resolvers/wallet.js | 43 ++++++++++++++--------------------------- lib/constants.js | 1 + worker/index.js | 4 +--- worker/wallet.js | 4 ---- 4 files changed, 17 insertions(+), 35 deletions(-) diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index e60d2e00f..7271a406c 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -12,6 +12,7 @@ import { WALLET_CREATE_INVOICE_TIMEOUT_MS, WALLET_RETRY_AFTER_MS, WALLET_RETRY_BEFORE_MS, + WALLET_VISIBILITY_TIMEOUT_MS, WALLET_MAX_RETRIES } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' @@ -465,36 +466,22 @@ const resolvers = { throw new GqlAuthenticationError() } // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED - // we use queryRawUnsafe because Prisma does not support tsrange - // see https://www.prisma.io/docs/orm/overview/databases/postgresql return await models.$queryRaw` - WITH failed AS ( - UPDATE "Invoice" - SET "lockedAt" = now() - WHERE id IN ( - SELECT id FROM "Invoice" - WHERE "userId" = ${me.id} - AND "actionState" = 'FAILED' - AND "userCancel" = false - AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval - AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval - AND "lockedAt" IS NULL - AND "paymentAttempt" < ${WALLET_MAX_RETRIES} - ORDER BY id DESC - FOR UPDATE SKIP LOCKED - ) - RETURNING * - ), - _ AS ( - INSERT INTO pgboss.job (name, data, startafter, keepuntil) - SELECT - 'unlockInvoice', - jsonb_build_object('id', id), - now() + interval '10 minutes', - now() + interval '15 minutes' - FROM failed + UPDATE "Invoice" + SET "lockedAt" = now() + WHERE id IN ( + SELECT id FROM "Invoice" + WHERE "userId" = ${me.id} + AND "actionState" = 'FAILED' + AND "userCancel" = false + AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval + AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval + AND ("lockedAt" IS NULL OR "lockedAt" < now() - ${`${WALLET_VISIBILITY_TIMEOUT_MS} milliseconds`}::interval) + AND "paymentAttempt" < ${WALLET_MAX_RETRIES} + ORDER BY id DESC + FOR UPDATE SKIP LOCKED ) - SELECT * FROM failed` + RETURNING *` } }, Wallet: { diff --git a/lib/constants.js b/lib/constants.js index 5ed604245..afbac77a5 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -202,6 +202,7 @@ export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 // by the client due to sender or receiver fallbacks are not returned to the client. export const WALLET_RETRY_AFTER_MS = 60_000 export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours +export const WALLET_VISIBILITY_TIMEOUT_MS = 60_000 // 1 minute // we want to attempt a payment three times so we retry two times export const WALLET_MAX_RETRIES = 2 diff --git a/worker/index.js b/worker/index.js index 07dce5b8e..dfb6c4f97 100644 --- a/worker/index.js +++ b/worker/index.js @@ -5,8 +5,7 @@ import createPrisma from '@/lib/create-prisma' import { checkInvoice, checkPendingDeposits, checkPendingWithdrawals, checkWithdrawal, - finalizeHodlInvoice, subscribeToWallet, - unlockInvoice + finalizeHodlInvoice, subscribeToWallet } from './wallet' import { repin } from './repin' import { trust } from './trust' @@ -103,7 +102,6 @@ async function work () { await boss.work('autoWithdraw', jobWrapper(autoWithdraw)) await boss.work('checkInvoice', jobWrapper(checkInvoice)) await boss.work('checkWithdrawal', jobWrapper(checkWithdrawal)) - await boss.work('unlockInvoice', jobWrapper(unlockInvoice)) // paidAction jobs await boss.work('paidActionForwarding', jobWrapper(paidActionForwarding)) await boss.work('paidActionForwarded', jobWrapper(paidActionForwarded)) diff --git a/worker/wallet.js b/worker/wallet.js index 6264bce6e..ac09c7ac4 100644 --- a/worker/wallet.js +++ b/worker/wallet.js @@ -284,7 +284,3 @@ export async function checkPendingWithdrawals (args) { } } } - -export async function unlockInvoice ({ data: { id }, models }) { - await models.invoice.update({ where: { id }, data: { lockedAt: null } }) -} From 7c108595c2b79925ff08fd37348b7aec1d7bc6e3 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Fri, 17 Jan 2025 18:39:54 +0100 Subject: [PATCH 11/29] Better comments for constants --- lib/constants.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/constants.js b/lib/constants.js index afbac77a5..6de217ed2 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -197,11 +197,14 @@ export const ZAP_UNDO_DELAY_MS = 5_000 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 -// When should failed invoices be returned to a client to retry? -// This must be high enough such that intermediate failed invoices that will be retried -// by the client due to sender or receiver fallbacks are not returned to the client. -export const WALLET_RETRY_AFTER_MS = 60_000 + +// interval between which failed invoices are returned to a client for automated retries. +// retry-after must be high enough such that intermediate failed invoices that will already +// be retried by the client due to sender or receiver fallbacks are not returned to the client. +export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours +// timeout after which we give up on waiting for the retry of a previously returned invoice +// and thus we allow returning an invoice to a client again export const WALLET_VISIBILITY_TIMEOUT_MS = 60_000 // 1 minute // we want to attempt a payment three times so we retry two times export const WALLET_MAX_RETRIES = 2 From 5c81a69b30fea3b1f403b860e1ab9c78cb071155 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Tue, 21 Jan 2025 13:12:55 +0100 Subject: [PATCH 12/29] Acquire lock during retry --- api/paidAction/README.md | 9 ++- api/paidAction/index.js | 3 +- api/resolvers/paidAction.js | 43 +++++++----- api/resolvers/wallet.js | 24 +++---- api/typeDefs/paidAction.js | 2 +- components/use-invoice.js | 4 +- fragments/paidAction.js | 4 +- lib/constants.js | 3 - .../migration.sql | 6 +- prisma/schema.prisma | 2 +- wallets/index.js | 68 ++++++++++++++----- wallets/server.js | 6 +- 12 files changed, 104 insertions(+), 70 deletions(-) diff --git a/api/paidAction/README.md b/api/paidAction/README.md index a32588076..08be33720 100644 --- a/api/paidAction/README.md +++ b/api/paidAction/README.md @@ -15,8 +15,9 @@ stateDiagram-v2 PENDING --> FAILED PAID --> [*] CANCELING --> FAILED - FAILED --> RETRYING + FAILED --> RETRY_PENDING FAILED --> [*] + RETRY_PENDING --> RETRYING RETRYING --> [*] [*] --> PENDING_HELD PENDING_HELD --> HELD @@ -59,8 +60,9 @@ stateDiagram-v2 PENDING --> FAILED PAID --> [*] CANCELING --> FAILED - FAILED --> RETRYING + FAILED --> RETRY_PENDING FAILED --> [*] + RETRY_PENDING --> RETRYING RETRYING --> [*] ``` @@ -121,8 +123,9 @@ This works by requesting an invoice from the recipient's wallet and reusing the stateDiagram-v2 PAID --> [*] CANCELING --> FAILED - FAILED --> RETRYING + FAILED --> RETRY_PENDING FAILED --> [*] + RETRY_PENDING --> RETRYING RETRYING --> [*] [*] --> PENDING_HELD PENDING_HELD --> FORWARDING diff --git a/api/paidAction/index.js b/api/paidAction/index.js index 8feabc306..5e4a34566 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -363,11 +363,10 @@ export async function retryPaidAction (actionType, args, incomingContext) { return await models.$transaction(async tx => { const context = { ...retryContext, tx, invoiceArgs } - // update the old invoice to RETRYING, so that it's not confused with FAILED await tx.invoice.update({ where: { id: failedInvoice.id, - actionState: 'FAILED' + actionState: 'RETRY_PENDING' }, data: { actionState: 'RETRYING' diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 7a5184181..958da4c00 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -50,35 +50,44 @@ export default { } }, Mutation: { - retryPaidAction: async (parent, { invoiceId }, { models, me, lnd }) => { + retryPaidAction: async (parent, { invoiceId, newAttempt }, { models, me, lnd }) => { if (!me) { throw new Error('You must be logged in') } - const invoice = await models.invoice.findUnique({ where: { id: invoiceId, userId: me.id } }) + // make sure only one client at a time can retry by immediately transitioning to an intermediate state + const [invoice] = await models.$queryRaw` + UPDATE "Invoice" + SET "actionState" = 'RETRY_PENDING' + WHERE id in ( + SELECT id FROM "Invoice" + WHERE id = ${invoiceId} AND "userId" = ${me.id} AND "actionState" = 'FAILED' + FOR UPDATE + ) + RETURNING *` if (!invoice) { throw new Error('Invoice not found') } - if (invoice.actionState !== 'FAILED') { - if (invoice.actionState === 'PAID') { - throw new Error('Invoice is already paid') - } - throw new Error(`Invoice is not in failed state: ${invoice.actionState}`) - } - - // a locked invoice means we want to retry a payment from the beginning - // with all sender and receiver wallets so we need to increment the retry count - const paymentAttempt = invoice.lockedAt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt + // do we want to retry a payment from the beginning with all sender and receiver wallets? + const paymentAttempt = newAttempt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt if (paymentAttempt > WALLET_MAX_RETRIES) { throw new Error('Payment has been retried too many times') } - const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) - - return { - ...result, - type: paidActionType(invoice.actionType) + try { + const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) + return { + ...result, + type: paidActionType(invoice.actionType) + } + } catch (err) { + // revert state transition to allow new retry for this invoice + await models.invoice.update({ + where: { id: invoiceId, actionState: 'RETRY_PENDING' }, + data: { actionState: 'FAILED' } + }) + throw err } } }, diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index 7271a406c..8e6f3f0e7 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -12,7 +12,6 @@ import { WALLET_CREATE_INVOICE_TIMEOUT_MS, WALLET_RETRY_AFTER_MS, WALLET_RETRY_BEFORE_MS, - WALLET_VISIBILITY_TIMEOUT_MS, WALLET_MAX_RETRIES } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' @@ -467,21 +466,14 @@ const resolvers = { } // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED return await models.$queryRaw` - UPDATE "Invoice" - SET "lockedAt" = now() - WHERE id IN ( - SELECT id FROM "Invoice" - WHERE "userId" = ${me.id} - AND "actionState" = 'FAILED' - AND "userCancel" = false - AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval - AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval - AND ("lockedAt" IS NULL OR "lockedAt" < now() - ${`${WALLET_VISIBILITY_TIMEOUT_MS} milliseconds`}::interval) - AND "paymentAttempt" < ${WALLET_MAX_RETRIES} - ORDER BY id DESC - FOR UPDATE SKIP LOCKED - ) - RETURNING *` + SELECT * FROM "Invoice" + WHERE "userId" = ${me.id} + AND "actionState" = 'FAILED' + AND "userCancel" = false + AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval + AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval + AND "paymentAttempt" < ${WALLET_MAX_RETRIES} + ORDER BY id DESC` } }, Wallet: { diff --git a/api/typeDefs/paidAction.js b/api/typeDefs/paidAction.js index 45c66c397..38a592090 100644 --- a/api/typeDefs/paidAction.js +++ b/api/typeDefs/paidAction.js @@ -7,7 +7,7 @@ extend type Query { } extend type Mutation { - retryPaidAction(invoiceId: Int!): PaidAction! + retryPaidAction(invoiceId: Int!, newAttempt: Boolean): PaidAction! } enum PaymentMethod { diff --git a/components/use-invoice.js b/components/use-invoice.js index 534426358..f59cd1db3 100644 --- a/components/use-invoice.js +++ b/components/use-invoice.js @@ -42,9 +42,9 @@ export default function useInvoice () { return data.cancelInvoice }, [cancelInvoice]) - const retry = useCallback(async ({ id, hash, hmac }, { update } = {}) => { + const retry = useCallback(async ({ id, hash, hmac, newAttempt = false }, { update } = {}) => { console.log('retrying invoice:', hash) - const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update }) + const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id), newAttempt }, update }) if (error) throw error const newInvoice = data.retryPaidAction.invoice diff --git a/fragments/paidAction.js b/fragments/paidAction.js index 5b33d1cba..29a92d874 100644 --- a/fragments/paidAction.js +++ b/fragments/paidAction.js @@ -91,8 +91,8 @@ export const RETRY_PAID_ACTION = gql` ${PAID_ACTION} ${ITEM_PAID_ACTION_FIELDS} ${ITEM_ACT_PAID_ACTION_FIELDS} - mutation retryPaidAction($invoiceId: Int!) { - retryPaidAction(invoiceId: $invoiceId) { + mutation retryPaidAction($invoiceId: Int!, $newAttempt: Boolean) { + retryPaidAction(invoiceId: $invoiceId, newAttempt: $newAttempt) { __typename ...PaidActionFields ... on ItemPaidAction { diff --git a/lib/constants.js b/lib/constants.js index 6de217ed2..35d9e669c 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -203,9 +203,6 @@ export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 // be retried by the client due to sender or receiver fallbacks are not returned to the client. export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours -// timeout after which we give up on waiting for the retry of a previously returned invoice -// and thus we allow returning an invoice to a client again -export const WALLET_VISIBILITY_TIMEOUT_MS = 60_000 // 1 minute // we want to attempt a payment three times so we retry two times export const WALLET_MAX_RETRIES = 2 diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql index 17770e9e3..2cbaf7f16 100644 --- a/prisma/migrations/20250107084543_automated_retries/migration.sql +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -1,4 +1,6 @@ -- AlterTable ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; -ALTER TABLE "Invoice" ADD COLUMN "lockedAt" TIMESTAMP(3); -CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); \ No newline at end of file +CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); + +-- AlterEnum +ALTER TYPE "InvoiceActionState" ADD VALUE 'RETRY_PENDING'; \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index da186ade9..4f6e63d48 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -893,6 +893,7 @@ enum InvoiceActionState { FORWARDING FORWARDED FAILED_FORWARD + RETRY_PENDING RETRYING CANCELING } @@ -927,7 +928,6 @@ model Invoice { cancelled Boolean @default(false) cancelledAt DateTime? userCancel Boolean? - lockedAt DateTime? paymentAttempt Int @default(0) msatsRequested BigInt msatsReceived BigInt? diff --git a/wallets/index.js b/wallets/index.js index 83c6c13a0..42fb55bee 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,7 +1,7 @@ import { useMe } from '@/components/me' import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants' -import { useApolloClient, useMutation, useQuery } from '@apollo/client' +import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common' import useVault from '@/components/vault/use-vault' @@ -231,30 +231,62 @@ export function useSendWallets () { } function RetryHandler ({ children }) { - const failedInvoices = useFailedInvoices() + const wallets = useSendWallets() const waitForWalletPayment = useWalletPayment() const invoiceHelper = useInvoice() + const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' }) + + const retry = useCallback(async (invoice) => { + const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true }) + await waitForWalletPayment(newInvoice) + }, [invoiceHelper, waitForWalletPayment]) useEffect(() => { - (async () => { - for (const invoice of failedInvoices) { - const newInvoice = await invoiceHelper.retry(invoice) - waitForWalletPayment(newInvoice).catch(console.error) + if (wallets.length === 0) return + + const retryPoll = async () => { + let failedInvoices + try { + const { data, error } = await getFailedInvoices() + if (error) throw error + failedInvoices = data.failedInvoices + } catch (err) { + console.error('failed to fetch invoices to retry:', err) + return } - })() - }, [failedInvoices, invoiceHelper, waitForWalletPayment]) - return children -} + for (const inv of failedInvoices) { + try { + await retry(inv) + } catch (err) { + // some retries are expected to fail since only one client at a time is allowed to retry + // these should show up as 'invoice not found' errors + console.error('retry failed:', err) + } + } + } -function useFailedInvoices () { - const wallets = useSendWallets() + let timeout, stopped + const queuePoll = () => { + timeout = setTimeout(async () => { + try { + await retryPoll() + } catch (err) { + // every error should already be handled in retryPoll + // but this catch is a safety net to not trigger an unhandled promise rejection + console.error('retry poll failed:', err) + } + if (!stopped) queuePoll() + }, NORMAL_POLL_INTERVAL) + } + const stopPolling = () => { + stopped = true + clearTimeout(timeout) + } - const { data } = useQuery(FAILED_INVOICES, { - pollInterval: NORMAL_POLL_INTERVAL, - skip: wallets.length === 0, - notifyOnNetworkStatusChange: true - }) + queuePoll() + return stopPolling + }, [wallets, getFailedInvoices, retry]) - return data?.failedInvoices ?? [] + return children } diff --git a/wallets/server.js b/wallets/server.js index 58c00c11b..3e71fc5f0 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -114,8 +114,8 @@ export async function createWrappedInvoice (userId, export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. - // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it - // so it has not been updated yet. + // the current predecessor invoice is in state 'RETRY_PENDING' and not in state 'RETRYING' + // because we are currently retrying it so it has not been updated yet. // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out. const wallets = await models.$queryRaw` SELECT @@ -135,7 +135,7 @@ export async function getInvoiceableWallets (userId, { paymentAttempt, predecess -- this failed invoice will be used to start the recursion SELECT "Invoice"."id", "Invoice"."predecessorId" FROM "Invoice" - WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'FAILED' + WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'RETRY_PENDING' UNION ALL From 1bbd465cd4f3aa207007ff1705ce436348c9cb71 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Tue, 4 Feb 2025 19:21:34 +0100 Subject: [PATCH 13/29] Use expiring lock in retry mutation --- api/paidAction/README.md | 9 ++--- api/paidAction/index.js | 2 +- api/resolvers/paidAction.js | 35 +++++++------------ lib/constants.js | 2 ++ .../migration.sql | 4 +-- prisma/schema.prisma | 2 +- wallets/server.js | 4 +-- 7 files changed, 23 insertions(+), 35 deletions(-) diff --git a/api/paidAction/README.md b/api/paidAction/README.md index 08be33720..a32588076 100644 --- a/api/paidAction/README.md +++ b/api/paidAction/README.md @@ -15,9 +15,8 @@ stateDiagram-v2 PENDING --> FAILED PAID --> [*] CANCELING --> FAILED - FAILED --> RETRY_PENDING + FAILED --> RETRYING FAILED --> [*] - RETRY_PENDING --> RETRYING RETRYING --> [*] [*] --> PENDING_HELD PENDING_HELD --> HELD @@ -60,9 +59,8 @@ stateDiagram-v2 PENDING --> FAILED PAID --> [*] CANCELING --> FAILED - FAILED --> RETRY_PENDING + FAILED --> RETRYING FAILED --> [*] - RETRY_PENDING --> RETRYING RETRYING --> [*] ``` @@ -123,9 +121,8 @@ This works by requesting an invoice from the recipient's wallet and reusing the stateDiagram-v2 PAID --> [*] CANCELING --> FAILED - FAILED --> RETRY_PENDING + FAILED --> RETRYING FAILED --> [*] - RETRY_PENDING --> RETRYING RETRYING --> [*] [*] --> PENDING_HELD PENDING_HELD --> FORWARDING diff --git a/api/paidAction/index.js b/api/paidAction/index.js index 5e4a34566..5b907dd2b 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -366,7 +366,7 @@ export async function retryPaidAction (actionType, args, incomingContext) { await tx.invoice.update({ where: { id: failedInvoice.id, - actionState: 'RETRY_PENDING' + actionState: 'FAILED' }, data: { actionState: 'RETRYING' diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 958da4c00..00b5c4778 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -1,5 +1,5 @@ import { retryPaidAction } from '../paidAction' -import { USER_ID, WALLET_MAX_RETRIES } from '@/lib/constants' +import { USER_ID, WALLET_MAX_RETRIES, WALLET_RETRY_TIMEOUT_MS } from '@/lib/constants' function paidActionType (actionType) { switch (actionType) { @@ -55,18 +55,18 @@ export default { throw new Error('You must be logged in') } - // make sure only one client at a time can retry by immediately transitioning to an intermediate state + // make sure only one client at a time can retry by acquiring a lock that expires const [invoice] = await models.$queryRaw` UPDATE "Invoice" - SET "actionState" = 'RETRY_PENDING' - WHERE id in ( - SELECT id FROM "Invoice" - WHERE id = ${invoiceId} AND "userId" = ${me.id} AND "actionState" = 'FAILED' - FOR UPDATE - ) + SET "retryPendingSince" = CURRENT_TIMESTAMP + WHERE + id = ${invoiceId} AND + "userId" = ${me.id} AND + "actionState" = 'FAILED' AND + ("retryPendingSince" IS NULL OR "retryPendingSince" < now() - ${`${WALLET_RETRY_TIMEOUT_MS} milliseconds`}::interval) RETURNING *` if (!invoice) { - throw new Error('Invoice not found') + throw new Error('Invoice not found or retry pending') } // do we want to retry a payment from the beginning with all sender and receiver wallets? @@ -75,19 +75,10 @@ export default { throw new Error('Payment has been retried too many times') } - try { - const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) - return { - ...result, - type: paidActionType(invoice.actionType) - } - } catch (err) { - // revert state transition to allow new retry for this invoice - await models.invoice.update({ - where: { id: invoiceId, actionState: 'RETRY_PENDING' }, - data: { actionState: 'FAILED' } - }) - throw err + const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) + return { + ...result, + type: paidActionType(invoice.actionType) } } }, diff --git a/lib/constants.js b/lib/constants.js index 35d9e669c..9ec202453 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -205,5 +205,7 @@ export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours // we want to attempt a payment three times so we retry two times export const WALLET_MAX_RETRIES = 2 +// when a pending retry for an invoice should be considered expired and can be attempted again +export const WALLET_RETRY_TIMEOUT_MS = 60_000 // 1 minute export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l' diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql index 2cbaf7f16..e60e2e9fe 100644 --- a/prisma/migrations/20250107084543_automated_retries/migration.sql +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -1,6 +1,4 @@ -- AlterTable ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "Invoice" ADD COLUMN "retryPendingSince" TIMESTAMP(3); CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); - --- AlterEnum -ALTER TYPE "InvoiceActionState" ADD VALUE 'RETRY_PENDING'; \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4f6e63d48..535456163 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -893,7 +893,6 @@ enum InvoiceActionState { FORWARDING FORWARDED FAILED_FORWARD - RETRY_PENDING RETRYING CANCELING } @@ -929,6 +928,7 @@ model Invoice { cancelledAt DateTime? userCancel Boolean? paymentAttempt Int @default(0) + retryPendingSince DateTime? msatsRequested BigInt msatsReceived BigInt? desc String? diff --git a/wallets/server.js b/wallets/server.js index 3e71fc5f0..e21199b92 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -114,7 +114,7 @@ export async function createWrappedInvoice (userId, export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. - // the current predecessor invoice is in state 'RETRY_PENDING' and not in state 'RETRYING' + // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' // because we are currently retrying it so it has not been updated yet. // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out. const wallets = await models.$queryRaw` @@ -135,7 +135,7 @@ export async function getInvoiceableWallets (userId, { paymentAttempt, predecess -- this failed invoice will be used to start the recursion SELECT "Invoice"."id", "Invoice"."predecessorId" FROM "Invoice" - WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'RETRY_PENDING' + WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'FAILED' UNION ALL From 0363043afc3d431b0b4a01b3abefd8867aa7795c Mon Sep 17 00:00:00 2001 From: ekzyis Date: Tue, 4 Feb 2025 20:09:06 +0100 Subject: [PATCH 14/29] Use now() instead of CURRENT_TIMESTAMP --- api/resolvers/paidAction.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 00b5c4778..6c5e7426b 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -58,7 +58,7 @@ export default { // make sure only one client at a time can retry by acquiring a lock that expires const [invoice] = await models.$queryRaw` UPDATE "Invoice" - SET "retryPendingSince" = CURRENT_TIMESTAMP + SET "retryPendingSince" = now() WHERE id = ${invoiceId} AND "userId" = ${me.id} AND From 3603d571c3aed96df51a127c24f469cf70346dc3 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 6 Feb 2025 03:39:54 +0100 Subject: [PATCH 15/29] Cosmetic changes --- api/paidAction/index.js | 1 + api/resolvers/paidAction.js | 1 + api/resolvers/wallet.js | 1 - wallets/index.js | 1 + wallets/server.js | 4 ++-- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/paidAction/index.js b/api/paidAction/index.js index 5b907dd2b..8feabc306 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -363,6 +363,7 @@ export async function retryPaidAction (actionType, args, incomingContext) { return await models.$transaction(async tx => { const context = { ...retryContext, tx, invoiceArgs } + // update the old invoice to RETRYING, so that it's not confused with FAILED await tx.invoice.update({ where: { id: failedInvoice.id, diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 6c5e7426b..089d94c4b 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -76,6 +76,7 @@ export default { } const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) + return { ...result, type: paidActionType(invoice.actionType) diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index 8e6f3f0e7..c9a31c262 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -464,7 +464,6 @@ const resolvers = { if (!me) { throw new GqlAuthenticationError() } - // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED return await models.$queryRaw` SELECT * FROM "Invoice" WHERE "userId" = ${me.id} diff --git a/wallets/index.js b/wallets/index.js index 42fb55bee..f29ce1402 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -279,6 +279,7 @@ function RetryHandler ({ children }) { if (!stopped) queuePoll() }, NORMAL_POLL_INTERVAL) } + const stopPolling = () => { stopped = true clearTimeout(timeout) diff --git a/wallets/server.js b/wallets/server.js index e21199b92..58c00c11b 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -114,8 +114,8 @@ export async function createWrappedInvoice (userId, export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. - // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' - // because we are currently retrying it so it has not been updated yet. + // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it + // so it has not been updated yet. // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out. const wallets = await models.$queryRaw` SELECT From dbe6f1c507218d495d4a05a8bd1312c9058a6389 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Sun, 9 Feb 2025 15:19:40 +0100 Subject: [PATCH 16/29] Immediately show failed post payments in notifications --- api/resolvers/notifications.js | 4 +++- api/resolvers/wallet.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index eec24f60a..4ae8eaaa8 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -351,7 +351,9 @@ export default { WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' - AND "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} + -- we want to show failed payments for posts in /notifications immediately and not wait for retries. + -- also, retries would never happen if the user has no wallet attached. + AND ("Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE') AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index c9a31c262..a88d80bc0 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -468,7 +468,7 @@ const resolvers = { SELECT * FROM "Invoice" WHERE "userId" = ${me.id} AND "actionState" = 'FAILED' - AND "userCancel" = false + AND ("userCancel" = false OR "actionType" = 'ITEM_CREATE') AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval AND "paymentAttempt" < ${WALLET_MAX_RETRIES} From f3f67c1d68b81ffac2f1bc2a7c1c496f15866e6b Mon Sep 17 00:00:00 2001 From: ekzyis Date: Sun, 9 Feb 2025 17:07:13 +0100 Subject: [PATCH 17/29] Update hasNewNotes --- api/resolvers/user.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/api/resolvers/user.js b/api/resolvers/user.js index f131fa929..7cd499002 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -4,7 +4,7 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor' import { msatsToSats } from '@/lib/format' import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate' import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item' -import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES } from '@/lib/constants' +import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES } from '@/lib/constants' import { viewGroup } from './growth' import { timeUnitForRange, whenRange } from '@/lib/time' import assertApiKeyNotPermitted from './apiKey' @@ -543,7 +543,17 @@ export default { actionType: { in: INVOICE_ACTION_NOTIFICATION_TYPES }, - actionState: 'FAILED' + actionState: 'FAILED', + OR: [ + { + paymentAttempt: { + gte: WALLET_MAX_RETRIES + } + }, + { + actionType: 'ITEM_CREATE' + } + ] } }) From a25242e8c94ec55d4621b814e7030eff3a8f4aa3 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 12 Feb 2025 17:58:14 +0100 Subject: [PATCH 18/29] Never retry on user cancel For a consistent UX and less mental overhead, I decided to remove the exception for ITEM_CREATE where it would still retry in the background even though we want to show the payment failure immediately in notifications. --- api/resolvers/wallet.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index a88d80bc0..e6f00c9e3 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -468,7 +468,8 @@ const resolvers = { SELECT * FROM "Invoice" WHERE "userId" = ${me.id} AND "actionState" = 'FAILED' - AND ("userCancel" = false OR "actionType" = 'ITEM_CREATE') + -- never retry if user has cancelled the invoice manually + AND "userCancel" = false AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval AND "paymentAttempt" < ${WALLET_MAX_RETRIES} From 4a4658d3fa28cc18cb00fa9ebd7e1c414b89f88d Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 12 Feb 2025 18:25:35 +0100 Subject: [PATCH 19/29] Fix notifications without pending retries missing if no send wallets If a stacker has no send wallets, they would miss notifications about failed payments because they would never get retried. This commit fixes this by making the notifications query aware if the stacker has send wallets. This way, it can tell if a notification will be retried or not. --- api/resolvers/notifications.js | 7 +++-- api/resolvers/user.js | 31 +++++++++++++------ api/resolvers/wallet.js | 2 ++ api/typeDefs/user.js | 1 + fragments/wallet.js | 6 ++++ .../migration.sql | 2 ++ prisma/schema.prisma | 1 + wallets/index.js | 11 ++++++- 8 files changed, 48 insertions(+), 13 deletions(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 4ae8eaaa8..af48db9d2 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -351,9 +351,10 @@ export default { WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' - -- we want to show failed payments for posts in /notifications immediately and not wait for retries. - -- also, retries would never happen if the user has no wallet attached. - AND ("Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE') + -- we want to show notifications only if no more automated retries will be attempted. + -- automated retries depend on if the user has wallets or not. + -- failed posts are an exception where we want to show them immediately and thus never automatically retry. + ${meFull.sendWallets ? `AND ("Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE')` : ''} AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/user.js b/api/resolvers/user.js index 7cd499002..a9fbaf013 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -544,16 +544,21 @@ export default { in: INVOICE_ACTION_NOTIFICATION_TYPES }, actionState: 'FAILED', - OR: [ - { - paymentAttempt: { - gte: WALLET_MAX_RETRIES + ...(user.sendWallets + ? { + OR: [ + { + paymentAttempt: { + gte: WALLET_MAX_RETRIES + } + }, + { + actionType: 'ITEM_CREATE' + } + ] } - }, - { - actionType: 'ITEM_CREATE' - } - ] + : {}) + } }) @@ -873,6 +878,14 @@ export default { await models.user.update({ where: { id: me.id }, data: { hideWelcomeBanner: true } }) return true + }, + setSendWallets: async (parent, { sendWallets }, { me, models }) => { + if (!me) { + throw new GqlAuthenticationError() + } + + await models.user.update({ where: { id: me.id }, data: { sendWallets } }) + return sendWallets } }, diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index e6f00c9e3..e7bb93907 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -473,6 +473,8 @@ const resolvers = { AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval AND "paymentAttempt" < ${WALLET_MAX_RETRIES} + -- never retry failed posts because we always immediately show them in notifications + AND "actionType" <> 'ITEM_CREATE' ORDER BY id DESC` } }, diff --git a/api/typeDefs/user.js b/api/typeDefs/user.js index bfefe7e3d..cd2a0b36a 100644 --- a/api/typeDefs/user.js +++ b/api/typeDefs/user.js @@ -44,6 +44,7 @@ export default gql` generateApiKey(id: ID!): String deleteApiKey(id: ID!): User disableFreebies: Boolean + setSendWallets(sendWallets: Boolean!): Boolean } type User { diff --git a/fragments/wallet.js b/fragments/wallet.js index b4d34d30a..aa191cb92 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -240,3 +240,9 @@ export const FAILED_INVOICES = gql` } } ` + +export const SET_SEND_WALLETS = gql` + mutation SetSendWallets($sendWallets: Boolean!) { + setSendWallets(sendWallets: $sendWallets) + } +` diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql index e60e2e9fe..c64fe648b 100644 --- a/prisma/migrations/20250107084543_automated_retries/migration.sql +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -2,3 +2,5 @@ ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; ALTER TABLE "Invoice" ADD COLUMN "retryPendingSince" TIMESTAMP(3); CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); + +ALTER TABLE "users" ADD COLUMN "sendWallets" BOOLEAN NOT NULL DEFAULT false; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 535456163..ce47d33a3 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -74,6 +74,7 @@ model User { nostrCrossposting Boolean @default(false) slashtagId String? @unique(map: "users.slashtagId_unique") noteCowboyHat Boolean @default(true) + sendWallets Boolean @default(false) streak Int? gunStreak Int? horseStreak Int? diff --git a/wallets/index.js b/wallets/index.js index f29ce1402..b0d3bc6ef 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,5 +1,5 @@ import { useMe } from '@/components/me' -import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' +import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS, SET_SEND_WALLETS } from '@/fragments/wallet' import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants' import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' @@ -235,6 +235,7 @@ function RetryHandler ({ children }) { const waitForWalletPayment = useWalletPayment() const invoiceHelper = useInvoice() const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' }) + const [setSendWallets] = useMutation(SET_SEND_WALLETS) const retry = useCallback(async (invoice) => { const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true }) @@ -289,5 +290,13 @@ function RetryHandler ({ children }) { return stopPolling }, [wallets, getFailedInvoices, retry]) + // save on server if user has send wallets so we can render notifications on the server + useEffect(() => { + setSendWallets({ variables: { sendWallets: wallets.length > 0 } }) + .catch(err => { + console.error('setSendWallets mutation failed:', err) + }) + }, [wallets.length]) + return children } From f95d71b3e8776dbdafcc7cce5724e76f19e0a557 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 12 Feb 2025 18:51:38 +0100 Subject: [PATCH 20/29] Stop hiding userCancel in notifications As mentioned in a previous commit, I want to show anything that will not be attempted anymore in notifications. Before, I wanted to hide manually cancelled invoices but to not change experience unnecessarily and to decrease mental overhead, I changed my mind. --- api/resolvers/notifications.js | 4 +++- api/resolvers/user.js | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index af48db9d2..03924339b 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -354,7 +354,9 @@ export default { -- we want to show notifications only if no more automated retries will be attempted. -- automated retries depend on if the user has wallets or not. -- failed posts are an exception where we want to show them immediately and thus never automatically retry. - ${meFull.sendWallets ? `AND ("Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE')` : ''} + ${meFull.sendWallets + ? `AND ("Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."userCancel" = true)` + : ''} AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/user.js b/api/resolvers/user.js index a9fbaf013..d2e2091c2 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -554,6 +554,9 @@ export default { }, { actionType: 'ITEM_CREATE' + }, + { + userCancel: true } ] } From a90eb5dcbbe5f0c25ac4dad042a9e1f597776694 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 12 Feb 2025 19:14:34 +0100 Subject: [PATCH 21/29] Also consider invoice.cancelledAt in notifications --- api/resolvers/notifications.js | 11 ++++++++--- api/resolvers/user.js | 9 +++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 03924339b..c91dd30f9 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -5,7 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate' import { replyToSubscription } from '@/lib/webPush' import { getSub } from './sub' import { GqlAuthenticationError, GqlInputError } from '@/lib/error' -import { WALLET_MAX_RETRIES } from '@/lib/constants' +import { WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants' export default { Query: { @@ -353,9 +353,14 @@ export default { AND "Invoice"."actionState" = 'FAILED' -- we want to show notifications only if no more automated retries will be attempted. -- automated retries depend on if the user has wallets or not. - -- failed posts are an exception where we want to show them immediately and thus never automatically retry. ${meFull.sendWallets - ? `AND ("Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."userCancel" = true)` + ? `AND ( + -- this is the inverse of the filter for automated retries + "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} + OR "Invoice"."actionType" = 'ITEM_CREATE' + OR "Invoice"."userCancel" = true + OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + )` : ''} AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR diff --git a/api/resolvers/user.js b/api/resolvers/user.js index d2e2091c2..1dab1e09d 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -4,9 +4,9 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor' import { msatsToSats } from '@/lib/format' import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate' import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item' -import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES } from '@/lib/constants' +import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants' import { viewGroup } from './growth' -import { timeUnitForRange, whenRange } from '@/lib/time' +import { datePivot, timeUnitForRange, whenRange } from '@/lib/time' import assertApiKeyNotPermitted from './apiKey' import { hashEmail } from '@/lib/crypto' import { isMuted } from '@/lib/user' @@ -557,6 +557,11 @@ export default { }, { userCancel: true + }, + { + cancelledAt: { + lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) + } } ] } From dea2e7a21b5383796ab044ade22546063f7a687f Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 12 Feb 2025 23:18:09 +0100 Subject: [PATCH 22/29] Always retry failed payments, even without send wallets --- api/resolvers/notifications.js | 19 ++++---- api/resolvers/user.js | 48 +++++++------------ api/typeDefs/user.js | 1 - fragments/wallet.js | 6 --- .../migration.sql | 2 - prisma/schema.prisma | 1 - wallets/index.js | 26 +++++----- 7 files changed, 40 insertions(+), 63 deletions(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index c91dd30f9..75594b1a9 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -351,17 +351,14 @@ export default { WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' - -- we want to show notifications only if no more automated retries will be attempted. - -- automated retries depend on if the user has wallets or not. - ${meFull.sendWallets - ? `AND ( - -- this is the inverse of the filter for automated retries - "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} - OR "Invoice"."actionType" = 'ITEM_CREATE' - OR "Invoice"."userCancel" = true - OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' - )` - : ''} + AND ( + -- this is the inverse of the filter for automated retries + "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} + OR "Invoice"."actionType" = 'ITEM_CREATE' + OR "Invoice"."userCancel" = true + -- TODO: test this since invoice is not updated after retry-before + OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + ) AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/user.js b/api/resolvers/user.js index 1dab1e09d..27d6329d4 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -544,29 +544,25 @@ export default { in: INVOICE_ACTION_NOTIFICATION_TYPES }, actionState: 'FAILED', - ...(user.sendWallets - ? { - OR: [ - { - paymentAttempt: { - gte: WALLET_MAX_RETRIES - } - }, - { - actionType: 'ITEM_CREATE' - }, - { - userCancel: true - }, - { - cancelledAt: { - lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) - } - } - ] + OR: [ + { + paymentAttempt: { + gte: WALLET_MAX_RETRIES } - : {}) - + }, + { + actionType: 'ITEM_CREATE' + }, + { + userCancel: true + }, + { + // TODO: test this since invoice is not updated after retry-before + cancelledAt: { + lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) + } + } + ] } }) @@ -886,14 +882,6 @@ export default { await models.user.update({ where: { id: me.id }, data: { hideWelcomeBanner: true } }) return true - }, - setSendWallets: async (parent, { sendWallets }, { me, models }) => { - if (!me) { - throw new GqlAuthenticationError() - } - - await models.user.update({ where: { id: me.id }, data: { sendWallets } }) - return sendWallets } }, diff --git a/api/typeDefs/user.js b/api/typeDefs/user.js index cd2a0b36a..bfefe7e3d 100644 --- a/api/typeDefs/user.js +++ b/api/typeDefs/user.js @@ -44,7 +44,6 @@ export default gql` generateApiKey(id: ID!): String deleteApiKey(id: ID!): User disableFreebies: Boolean - setSendWallets(sendWallets: Boolean!): Boolean } type User { diff --git a/fragments/wallet.js b/fragments/wallet.js index aa191cb92..b4d34d30a 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -240,9 +240,3 @@ export const FAILED_INVOICES = gql` } } ` - -export const SET_SEND_WALLETS = gql` - mutation SetSendWallets($sendWallets: Boolean!) { - setSendWallets(sendWallets: $sendWallets) - } -` diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql index c64fe648b..e60e2e9fe 100644 --- a/prisma/migrations/20250107084543_automated_retries/migration.sql +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -2,5 +2,3 @@ ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; ALTER TABLE "Invoice" ADD COLUMN "retryPendingSince" TIMESTAMP(3); CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); - -ALTER TABLE "users" ADD COLUMN "sendWallets" BOOLEAN NOT NULL DEFAULT false; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index ce47d33a3..535456163 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -74,7 +74,6 @@ model User { nostrCrossposting Boolean @default(false) slashtagId String? @unique(map: "users.slashtagId_unique") noteCowboyHat Boolean @default(true) - sendWallets Boolean @default(false) streak Int? gunStreak Int? horseStreak Int? diff --git a/wallets/index.js b/wallets/index.js index b0d3bc6ef..d72227b57 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,5 +1,5 @@ import { useMe } from '@/components/me' -import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS, SET_SEND_WALLETS } from '@/fragments/wallet' +import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants' import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' @@ -9,6 +9,7 @@ import walletDefs from '@/wallets/client' import { generateMutation } from './graphql' import { useWalletPayment } from './payment' import useInvoice from '@/components/use-invoice' +import { WalletConfigurationError } from './errors' const WalletsContext = createContext({ wallets: [] @@ -235,15 +236,24 @@ function RetryHandler ({ children }) { const waitForWalletPayment = useWalletPayment() const invoiceHelper = useInvoice() const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' }) - const [setSendWallets] = useMutation(SET_SEND_WALLETS) const retry = useCallback(async (invoice) => { const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true }) - await waitForWalletPayment(newInvoice) + + try { + await waitForWalletPayment(newInvoice) + } catch (err) { + if (err instanceof WalletConfigurationError) { + // consume attempt by canceling invoice + await invoiceHelper.cancel(newInvoice) + } + throw err + } }, [invoiceHelper, waitForWalletPayment]) useEffect(() => { - if (wallets.length === 0) return + // we always retry failed invoices, even if the user has no wallets on any client + // to make sure that failed payments will always show up in notifications eventually const retryPoll = async () => { let failedInvoices @@ -290,13 +300,5 @@ function RetryHandler ({ children }) { return stopPolling }, [wallets, getFailedInvoices, retry]) - // save on server if user has send wallets so we can render notifications on the server - useEffect(() => { - setSendWallets({ variables: { sendWallets: wallets.length > 0 } }) - .catch(err => { - console.error('setSendWallets mutation failed:', err) - }) - }, [wallets.length]) - return children } From ddc115ef5072ff171fffad48718001b77a901931 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 13 Feb 2025 02:55:48 +0100 Subject: [PATCH 23/29] Fix notification indicator on retry timeout --- api/resolvers/notifications.js | 1 - api/resolvers/user.js | 1 - worker/index.js | 3 ++- worker/paidAction.js | 14 ++++++++++++-- worker/wallet.js | 4 ++++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 75594b1a9..34d3fe246 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -356,7 +356,6 @@ export default { "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} OR "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."userCancel" = true - -- TODO: test this since invoice is not updated after retry-before OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' ) AND ( diff --git a/api/resolvers/user.js b/api/resolvers/user.js index 27d6329d4..fadab34c6 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -557,7 +557,6 @@ export default { userCancel: true }, { - // TODO: test this since invoice is not updated after retry-before cancelledAt: { lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) } diff --git a/worker/index.js b/worker/index.js index dfb6c4f97..85019f8f7 100644 --- a/worker/index.js +++ b/worker/index.js @@ -5,7 +5,7 @@ import createPrisma from '@/lib/create-prisma' import { checkInvoice, checkPendingDeposits, checkPendingWithdrawals, checkWithdrawal, - finalizeHodlInvoice, subscribeToWallet + finalizeHodlInvoice, retryTimeout, subscribeToWallet } from './wallet' import { repin } from './repin' import { trust } from './trust' @@ -101,6 +101,7 @@ async function work () { await boss.work('autoDropBolt11s', jobWrapper(autoDropBolt11s)) await boss.work('autoWithdraw', jobWrapper(autoWithdraw)) await boss.work('checkInvoice', jobWrapper(checkInvoice)) + await boss.work('retryTimeout', jobWrapper(retryTimeout)) await boss.work('checkWithdrawal', jobWrapper(checkWithdrawal)) // paidAction jobs await boss.work('paidActionForwarding', jobWrapper(paidActionForwarding)) diff --git a/worker/paidAction.js b/worker/paidAction.js index 9b3ecb5a6..4723352de 100644 --- a/worker/paidAction.js +++ b/worker/paidAction.js @@ -1,7 +1,7 @@ import { getPaymentFailureStatus, hodlInvoiceCltvDetails, getPaymentOrNotSent } from '@/api/lnd' import { paidActions } from '@/api/paidAction' import { walletLogger } from '@/api/resolvers/wallet' -import { LND_PATHFINDING_TIME_PREF_PPM, LND_PATHFINDING_TIMEOUT_MS, PAID_ACTION_TERMINAL_STATES } from '@/lib/constants' +import { LND_PATHFINDING_TIME_PREF_PPM, LND_PATHFINDING_TIMEOUT_MS, PAID_ACTION_TERMINAL_STATES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants' import { formatMsats, formatSats, msatsToSats, toPositiveNumber } from '@/lib/format' import { datePivot } from '@/lib/time' import { Prisma } from '@prisma/client' @@ -466,9 +466,19 @@ export async function paidActionFailed ({ data: { invoiceId, ...args }, models, await paidActions[dbInvoice.actionType].onFail?.({ invoice: dbInvoice }, { models, tx, lnd }) + const cancelledAt = new Date() + + // XXX update invoice after retry timeout for notification indicator + await models.$executeRaw` + INSERT INTO pgboss.job (name, data, retrylimit, retrybackoff, startafter, keepuntil, priority) + VALUES ('retryTimeout', + jsonb_build_object('hash', ${dbInvoice.hash}::TEXT), 21, true, + ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval, + ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${2 * WALLET_RETRY_BEFORE_MS} milliseconds`}::interval, 100)` + return { cancelled: true, - cancelledAt: new Date() + cancelledAt } }, ...args diff --git a/worker/wallet.js b/worker/wallet.js index ac09c7ac4..f60581887 100644 --- a/worker/wallet.js +++ b/worker/wallet.js @@ -161,6 +161,10 @@ export async function checkInvoice ({ data: { hash, invoice }, boss, models, lnd } } +export async function retryTimeout ({ data: { hash }, models, lnd, boss }) { + await models.invoice.update({ where: { hash }, data: { updatedAt: new Date() } }) +} + async function subscribeToWithdrawals (args) { const { lnd } = args From a485faf2a80ad8dc417272a3752dca7dab230d8a Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 13 Feb 2025 19:09:15 +0100 Subject: [PATCH 24/29] Set invoice.updated_at to date slightly in the future --- worker/wallet.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/worker/wallet.js b/worker/wallet.js index f60581887..a5d881bb9 100644 --- a/worker/wallet.js +++ b/worker/wallet.js @@ -3,7 +3,7 @@ import { subscribeToInvoices, subscribeToPayments, subscribeToInvoice } from 'ln-service' import { getPaymentOrNotSent } from '@/api/lnd' -import { sleep } from '@/lib/time' +import { datePivot, sleep } from '@/lib/time' import retry from 'async-retry' import { paidActionPaid, paidActionForwarded, @@ -162,7 +162,14 @@ export async function checkInvoice ({ data: { hash, invoice }, boss, models, lnd } export async function retryTimeout ({ data: { hash }, models, lnd, boss }) { - await models.invoice.update({ where: { hash }, data: { updatedAt: new Date() } }) + // This job exists to make sure the notification indicator shows up when + // 1) not enough automated retries have been attempted + // and + // 2) no more automated retries will be attempted because invoice is too old + // since the notification indicator only checks invoices that have been updated since the last poll. + // We set the date slightly in future to avoid possibly possible (= not verified) + // race conditions between the notification indicator poll and this update. + await models.invoice.update({ where: { hash }, data: { updatedAt: datePivot(new Date(), { seconds: 5 }) } }) } async function subscribeToWithdrawals (args) { From 8fc36ba4dce9a5868304b0b5db72e46c383256f5 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 13 Feb 2025 22:12:04 +0100 Subject: [PATCH 25/29] Use default job priority --- worker/paidAction.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/paidAction.js b/worker/paidAction.js index 4723352de..07cedf38f 100644 --- a/worker/paidAction.js +++ b/worker/paidAction.js @@ -470,11 +470,11 @@ export async function paidActionFailed ({ data: { invoiceId, ...args }, models, // XXX update invoice after retry timeout for notification indicator await models.$executeRaw` - INSERT INTO pgboss.job (name, data, retrylimit, retrybackoff, startafter, keepuntil, priority) + INSERT INTO pgboss.job (name, data, retrylimit, retrybackoff, startafter, keepuntil) VALUES ('retryTimeout', jsonb_build_object('hash', ${dbInvoice.hash}::TEXT), 21, true, ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval, - ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${2 * WALLET_RETRY_BEFORE_MS} milliseconds`}::interval, 100)` + ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${2 * WALLET_RETRY_BEFORE_MS} milliseconds`}::interval)` return { cancelled: true, From fa26f731bbf43a32e2b7c59ceb74d5948e4f9f17 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Thu, 13 Feb 2025 22:14:35 +0100 Subject: [PATCH 26/29] Stop retrying after one hour --- lib/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/constants.js b/lib/constants.js index 9ec202453..1d0ce0126 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -202,7 +202,7 @@ export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 // retry-after must be high enough such that intermediate failed invoices that will already // be retried by the client due to sender or receiver fallbacks are not returned to the client. export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute -export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours +export const WALLET_RETRY_BEFORE_MS = 3_600_000 // 1 hour // we want to attempt a payment three times so we retry two times export const WALLET_MAX_RETRIES = 2 // when a pending retry for an invoice should be considered expired and can be attempted again From d971229d7e8ca88373dd5cbafad8ee789aa8b4a4 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Fri, 14 Feb 2025 01:21:17 +0100 Subject: [PATCH 27/29] Remove special case for ITEM_CREATE --- api/resolvers/notifications.js | 1 - api/resolvers/user.js | 3 --- api/resolvers/wallet.js | 2 -- 3 files changed, 6 deletions(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index c9c1dc21c..fab517fa6 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -354,7 +354,6 @@ export default { AND ( -- this is the inverse of the filter for automated retries "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} - OR "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."userCancel" = true OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' ) diff --git a/api/resolvers/user.js b/api/resolvers/user.js index fadab34c6..9dc318673 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -550,9 +550,6 @@ export default { gte: WALLET_MAX_RETRIES } }, - { - actionType: 'ITEM_CREATE' - }, { userCancel: true }, diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index e7bb93907..e6f00c9e3 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -473,8 +473,6 @@ const resolvers = { AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval AND "paymentAttempt" < ${WALLET_MAX_RETRIES} - -- never retry failed posts because we always immediately show them in notifications - AND "actionType" <> 'ITEM_CREATE' ORDER BY id DESC` } }, From f57c37d5c6c6ed6ec6a3f6aaa2fc3ac59df0f3cd Mon Sep 17 00:00:00 2001 From: ekzyis Date: Fri, 14 Feb 2025 02:34:09 +0100 Subject: [PATCH 28/29] Replace retryTimeout job with notification indicator query --- api/resolvers/user.js | 30 +++++++++++++++++++++++++----- worker/index.js | 3 +-- worker/paidAction.js | 14 ++------------ worker/wallet.js | 13 +------------ 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/api/resolvers/user.js b/api/resolvers/user.js index 9dc318673..782380756 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -552,11 +552,6 @@ export default { }, { userCancel: true - }, - { - cancelledAt: { - lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) - } } ] } @@ -567,6 +562,31 @@ export default { return true } + const invoiceActionFailed2 = await models.invoice.findFirst({ + where: { + userId: me.id, + updatedAt: { + gt: datePivot(lastChecked, { milliseconds: -WALLET_RETRY_BEFORE_MS }) + }, + actionType: { + in: INVOICE_ACTION_NOTIFICATION_TYPES + }, + actionState: 'FAILED', + paymentAttempt: { + lt: WALLET_MAX_RETRIES + }, + userCancel: false, + cancelledAt: { + lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) + } + } + }) + + if (invoiceActionFailed2) { + foundNotes() + return true + } + // update checkedNotesAt to prevent rechecking same time period models.user.update({ where: { id: me.id }, diff --git a/worker/index.js b/worker/index.js index 85019f8f7..dfb6c4f97 100644 --- a/worker/index.js +++ b/worker/index.js @@ -5,7 +5,7 @@ import createPrisma from '@/lib/create-prisma' import { checkInvoice, checkPendingDeposits, checkPendingWithdrawals, checkWithdrawal, - finalizeHodlInvoice, retryTimeout, subscribeToWallet + finalizeHodlInvoice, subscribeToWallet } from './wallet' import { repin } from './repin' import { trust } from './trust' @@ -101,7 +101,6 @@ async function work () { await boss.work('autoDropBolt11s', jobWrapper(autoDropBolt11s)) await boss.work('autoWithdraw', jobWrapper(autoWithdraw)) await boss.work('checkInvoice', jobWrapper(checkInvoice)) - await boss.work('retryTimeout', jobWrapper(retryTimeout)) await boss.work('checkWithdrawal', jobWrapper(checkWithdrawal)) // paidAction jobs await boss.work('paidActionForwarding', jobWrapper(paidActionForwarding)) diff --git a/worker/paidAction.js b/worker/paidAction.js index 07cedf38f..9b3ecb5a6 100644 --- a/worker/paidAction.js +++ b/worker/paidAction.js @@ -1,7 +1,7 @@ import { getPaymentFailureStatus, hodlInvoiceCltvDetails, getPaymentOrNotSent } from '@/api/lnd' import { paidActions } from '@/api/paidAction' import { walletLogger } from '@/api/resolvers/wallet' -import { LND_PATHFINDING_TIME_PREF_PPM, LND_PATHFINDING_TIMEOUT_MS, PAID_ACTION_TERMINAL_STATES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants' +import { LND_PATHFINDING_TIME_PREF_PPM, LND_PATHFINDING_TIMEOUT_MS, PAID_ACTION_TERMINAL_STATES } from '@/lib/constants' import { formatMsats, formatSats, msatsToSats, toPositiveNumber } from '@/lib/format' import { datePivot } from '@/lib/time' import { Prisma } from '@prisma/client' @@ -466,19 +466,9 @@ export async function paidActionFailed ({ data: { invoiceId, ...args }, models, await paidActions[dbInvoice.actionType].onFail?.({ invoice: dbInvoice }, { models, tx, lnd }) - const cancelledAt = new Date() - - // XXX update invoice after retry timeout for notification indicator - await models.$executeRaw` - INSERT INTO pgboss.job (name, data, retrylimit, retrybackoff, startafter, keepuntil) - VALUES ('retryTimeout', - jsonb_build_object('hash', ${dbInvoice.hash}::TEXT), 21, true, - ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval, - ${cancelledAt}::TIMESTAMP WITH TIME ZONE + ${`${2 * WALLET_RETRY_BEFORE_MS} milliseconds`}::interval)` - return { cancelled: true, - cancelledAt + cancelledAt: new Date() } }, ...args diff --git a/worker/wallet.js b/worker/wallet.js index a5d881bb9..ac09c7ac4 100644 --- a/worker/wallet.js +++ b/worker/wallet.js @@ -3,7 +3,7 @@ import { subscribeToInvoices, subscribeToPayments, subscribeToInvoice } from 'ln-service' import { getPaymentOrNotSent } from '@/api/lnd' -import { datePivot, sleep } from '@/lib/time' +import { sleep } from '@/lib/time' import retry from 'async-retry' import { paidActionPaid, paidActionForwarded, @@ -161,17 +161,6 @@ export async function checkInvoice ({ data: { hash, invoice }, boss, models, lnd } } -export async function retryTimeout ({ data: { hash }, models, lnd, boss }) { - // This job exists to make sure the notification indicator shows up when - // 1) not enough automated retries have been attempted - // and - // 2) no more automated retries will be attempted because invoice is too old - // since the notification indicator only checks invoices that have been updated since the last poll. - // We set the date slightly in future to avoid possibly possible (= not verified) - // race conditions between the notification indicator poll and this update. - await models.invoice.update({ where: { hash }, data: { updatedAt: datePivot(new Date(), { seconds: 5 }) } }) -} - async function subscribeToWithdrawals (args) { const { lnd } = args From e63650e35bf986f76a6f126856c7565bb1d87abe Mon Sep 17 00:00:00 2001 From: ekzyis Date: Fri, 14 Feb 2025 03:41:11 +0100 Subject: [PATCH 29/29] Fix sortTime --- api/resolvers/notifications.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index fab517fa6..42cb8963b 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -346,7 +346,15 @@ export default { ) queries.push( - `(SELECT "Invoice".id::text, "Invoice"."updated_at" AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type + `(SELECT "Invoice".id::text, + CASE + WHEN + "Invoice"."paymentAttempt" < ${WALLET_MAX_RETRIES} + AND "Invoice"."userCancel" = false + AND "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + THEN "Invoice"."cancelledAt" + interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + ELSE "Invoice"."updated_at" + END AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type FROM "Invoice" WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2