diff --git a/api/paidAction/index.js b/api/paidAction/index.js index ad067a3ca..8feabc306 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) { } async function createDbInvoice (actionType, args, context) { - const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context + const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs const db = tx ?? models @@ -445,6 +445,7 @@ async function createDbInvoice (actionType, args, context) { actionArgs: args, expiresAt, actionId, + paymentAttempt, predecessorId } diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 33e587022..42cb8963b 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate' import { replyToSubscription } from '@/lib/webPush' import { getSub } from './sub' import { GqlAuthenticationError, GqlInputError } from '@/lib/error' +import { WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants' export default { Query: { @@ -345,11 +346,25 @@ export default { ) queries.push( - `(SELECT "Invoice".id::text, "Invoice"."updated_at" AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type + `(SELECT "Invoice".id::text, + CASE + WHEN + "Invoice"."paymentAttempt" < ${WALLET_MAX_RETRIES} + AND "Invoice"."userCancel" = false + AND "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + THEN "Invoice"."cancelledAt" + interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + ELSE "Invoice"."updated_at" + END AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type FROM "Invoice" WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' + AND ( + -- this is the inverse of the filter for automated retries + "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} + OR "Invoice"."userCancel" = true + OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}' + ) AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 2b993c1f0..089d94c4b 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -1,5 +1,5 @@ import { retryPaidAction } from '../paidAction' -import { USER_ID } from '@/lib/constants' +import { USER_ID, WALLET_MAX_RETRIES, WALLET_RETRY_TIMEOUT_MS } from '@/lib/constants' function paidActionType (actionType) { switch (actionType) { @@ -50,24 +50,32 @@ export default { } }, Mutation: { - retryPaidAction: async (parent, { invoiceId }, { models, me, lnd }) => { + retryPaidAction: async (parent, { invoiceId, newAttempt }, { models, me, lnd }) => { if (!me) { throw new Error('You must be logged in') } - const invoice = await models.invoice.findUnique({ where: { id: invoiceId, userId: me.id } }) + // make sure only one client at a time can retry by acquiring a lock that expires + const [invoice] = await models.$queryRaw` + UPDATE "Invoice" + SET "retryPendingSince" = now() + WHERE + id = ${invoiceId} AND + "userId" = ${me.id} AND + "actionState" = 'FAILED' AND + ("retryPendingSince" IS NULL OR "retryPendingSince" < now() - ${`${WALLET_RETRY_TIMEOUT_MS} milliseconds`}::interval) + RETURNING *` if (!invoice) { - throw new Error('Invoice not found') + throw new Error('Invoice not found or retry pending') } - if (invoice.actionState !== 'FAILED') { - if (invoice.actionState === 'PAID') { - throw new Error('Invoice is already paid') - } - throw new Error(`Invoice is not in failed state: ${invoice.actionState}`) + // do we want to retry a payment from the beginning with all sender and receiver wallets? + const paymentAttempt = newAttempt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt + if (paymentAttempt > WALLET_MAX_RETRIES) { + throw new Error('Payment has been retried too many times') } - const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd }) + const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) return { ...result, diff --git a/api/resolvers/user.js b/api/resolvers/user.js index f131fa929..782380756 100644 --- a/api/resolvers/user.js +++ b/api/resolvers/user.js @@ -4,9 +4,9 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor' import { msatsToSats } from '@/lib/format' import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate' import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item' -import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES } from '@/lib/constants' +import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants' import { viewGroup } from './growth' -import { timeUnitForRange, whenRange } from '@/lib/time' +import { datePivot, timeUnitForRange, whenRange } from '@/lib/time' import assertApiKeyNotPermitted from './apiKey' import { hashEmail } from '@/lib/crypto' import { isMuted } from '@/lib/user' @@ -543,7 +543,17 @@ export default { actionType: { in: INVOICE_ACTION_NOTIFICATION_TYPES }, - actionState: 'FAILED' + actionState: 'FAILED', + OR: [ + { + paymentAttempt: { + gte: WALLET_MAX_RETRIES + } + }, + { + userCancel: true + } + ] } }) @@ -552,6 +562,31 @@ export default { return true } + const invoiceActionFailed2 = await models.invoice.findFirst({ + where: { + userId: me.id, + updatedAt: { + gt: datePivot(lastChecked, { milliseconds: -WALLET_RETRY_BEFORE_MS }) + }, + actionType: { + in: INVOICE_ACTION_NOTIFICATION_TYPES + }, + actionState: 'FAILED', + paymentAttempt: { + lt: WALLET_MAX_RETRIES + }, + userCancel: false, + cancelledAt: { + lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS }) + } + } + }) + + if (invoiceActionFailed2) { + foundNotes() + return true + } + // update checkedNotesAt to prevent rechecking same time period models.user.update({ where: { id: me.id }, diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index c71749ac0..e6f00c9e3 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib import { USER_ID, INVOICE_RETENTION_DAYS, PAID_ACTION_PAYMENT_METHODS, - WALLET_CREATE_INVOICE_TIMEOUT_MS + WALLET_CREATE_INVOICE_TIMEOUT_MS, + WALLET_RETRY_AFTER_MS, + WALLET_RETRY_BEFORE_MS, + WALLET_MAX_RETRIES } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' import assertGofacYourself from './ofac' @@ -456,6 +459,21 @@ const resolvers = { cursor: nextCursor, entries: logs } + }, + failedInvoices: async (parent, args, { me, models }) => { + if (!me) { + throw new GqlAuthenticationError() + } + return await models.$queryRaw` + SELECT * FROM "Invoice" + WHERE "userId" = ${me.id} + AND "actionState" = 'FAILED' + -- never retry if user has cancelled the invoice manually + AND "userCancel" = false + AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval + AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval + AND "paymentAttempt" < ${WALLET_MAX_RETRIES} + ORDER BY id DESC` } }, Wallet: { diff --git a/api/typeDefs/paidAction.js b/api/typeDefs/paidAction.js index 45c66c397..38a592090 100644 --- a/api/typeDefs/paidAction.js +++ b/api/typeDefs/paidAction.js @@ -7,7 +7,7 @@ extend type Query { } extend type Mutation { - retryPaidAction(invoiceId: Int!): PaidAction! + retryPaidAction(invoiceId: Int!, newAttempt: Boolean): PaidAction! } enum PaymentMethod { diff --git a/api/typeDefs/wallet.js b/api/typeDefs/wallet.js index 3006fc7a4..efc49130b 100644 --- a/api/typeDefs/wallet.js +++ b/api/typeDefs/wallet.js @@ -72,6 +72,7 @@ const typeDefs = ` wallet(id: ID!): Wallet walletByType(type: String!): Wallet walletLogs(type: String, from: String, to: String, cursor: String): WalletLog! + failedInvoices: [Invoice!]! } extend type Mutation { diff --git a/components/use-invoice.js b/components/use-invoice.js index 1cbe94dfd..f59cd1db3 100644 --- a/components/use-invoice.js +++ b/components/use-invoice.js @@ -1,5 +1,5 @@ import { useApolloClient, useMutation } from '@apollo/client' -import { useCallback } from 'react' +import { useCallback, useMemo } from 'react' import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors' import { RETRY_PAID_ACTION } from '@/fragments/paidAction' import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet' @@ -42,9 +42,9 @@ export default function useInvoice () { return data.cancelInvoice }, [cancelInvoice]) - const retry = useCallback(async ({ id, hash, hmac }, { update }) => { + const retry = useCallback(async ({ id, hash, hmac, newAttempt = false }, { update } = {}) => { console.log('retrying invoice:', hash) - const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update }) + const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id), newAttempt }, update }) if (error) throw error const newInvoice = data.retryPaidAction.invoice @@ -53,5 +53,5 @@ export default function useInvoice () { return newInvoice }, [retryPaidAction]) - return { cancel, retry, isInvoice } + return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice]) } diff --git a/fragments/paidAction.js b/fragments/paidAction.js index 5b33d1cba..29a92d874 100644 --- a/fragments/paidAction.js +++ b/fragments/paidAction.js @@ -91,8 +91,8 @@ export const RETRY_PAID_ACTION = gql` ${PAID_ACTION} ${ITEM_PAID_ACTION_FIELDS} ${ITEM_ACT_PAID_ACTION_FIELDS} - mutation retryPaidAction($invoiceId: Int!) { - retryPaidAction(invoiceId: $invoiceId) { + mutation retryPaidAction($invoiceId: Int!, $newAttempt: Boolean) { + retryPaidAction(invoiceId: $invoiceId, newAttempt: $newAttempt) { __typename ...PaidActionFields ... on ItemPaidAction { diff --git a/fragments/wallet.js b/fragments/wallet.js index f75d6547e..b4d34d30a 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql` } } ` + +export const FAILED_INVOICES = gql` + ${INVOICE_FIELDS} + query FailedInvoices { + failedInvoices { + ...InvoiceFields + } + } +` diff --git a/lib/apollo.js b/lib/apollo.js index 1d127792f..c95af0af2 100644 --- a/lib/apollo.js +++ b/lib/apollo.js @@ -283,6 +283,12 @@ function getClient (uri) { facts: [...(existing?.facts || []), ...incoming.facts] } } + }, + failedInvoices: { + keyArgs: [], + merge (existing, incoming) { + return incoming + } } } }, diff --git a/lib/constants.js b/lib/constants.js index 48cf0cda0..1d0ce0126 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -198,4 +198,14 @@ export const ZAP_UNDO_DELAY_MS = 5_000 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 +// interval between which failed invoices are returned to a client for automated retries. +// retry-after must be high enough such that intermediate failed invoices that will already +// be retried by the client due to sender or receiver fallbacks are not returned to the client. +export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute +export const WALLET_RETRY_BEFORE_MS = 3_600_000 // 1 hour +// we want to attempt a payment three times so we retry two times +export const WALLET_MAX_RETRIES = 2 +// when a pending retry for an invoice should be considered expired and can be attempted again +export const WALLET_RETRY_TIMEOUT_MS = 60_000 // 1 minute + export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l' diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql new file mode 100644 index 000000000..e60e2e9fe --- /dev/null +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "Invoice" ADD COLUMN "retryPendingSince" TIMESTAMP(3); +CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 2ec20fce7..a3f986b62 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -928,6 +928,8 @@ model Invoice { cancelled Boolean @default(false) cancelledAt DateTime? userCancel Boolean? + paymentAttempt Int @default(0) + retryPendingSince DateTime? msatsRequested BigInt msatsReceived BigInt? desc String? @@ -956,6 +958,7 @@ model Invoice { @@index([confirmedIndex], map: "Invoice.confirmedIndex_index") @@index([isHeld]) @@index([confirmedAt]) + @@index([cancelledAt]) @@index([actionType]) @@index([actionState]) } diff --git a/wallets/index.js b/wallets/index.js index bdf00083b..d72227b57 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,12 +1,15 @@ import { useMe } from '@/components/me' -import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' -import { SSR } from '@/lib/constants' -import { useApolloClient, useMutation, useQuery } from '@apollo/client' +import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' +import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants' +import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common' import useVault from '@/components/vault/use-vault' import walletDefs from '@/wallets/client' import { generateMutation } from './graphql' +import { useWalletPayment } from './payment' +import useInvoice from '@/components/use-invoice' +import { WalletConfigurationError } from './errors' const WalletsContext = createContext({ wallets: [] @@ -204,7 +207,9 @@ export function WalletsProvider ({ children }) { removeLocalWallets }} > - {children} + + {children} + ) } @@ -221,7 +226,79 @@ export function useWallet (name) { export function useSendWallets () { const { wallets } = useWallets() // return all enabled wallets that are available and can send - return wallets + return useMemo(() => wallets .filter(w => !w.def.isAvailable || w.def.isAvailable()) - .filter(w => w.config?.enabled && canSend(w)) + .filter(w => w.config?.enabled && canSend(w)), [wallets]) +} + +function RetryHandler ({ children }) { + const wallets = useSendWallets() + const waitForWalletPayment = useWalletPayment() + const invoiceHelper = useInvoice() + const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' }) + + const retry = useCallback(async (invoice) => { + const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true }) + + try { + await waitForWalletPayment(newInvoice) + } catch (err) { + if (err instanceof WalletConfigurationError) { + // consume attempt by canceling invoice + await invoiceHelper.cancel(newInvoice) + } + throw err + } + }, [invoiceHelper, waitForWalletPayment]) + + useEffect(() => { + // we always retry failed invoices, even if the user has no wallets on any client + // to make sure that failed payments will always show up in notifications eventually + + const retryPoll = async () => { + let failedInvoices + try { + const { data, error } = await getFailedInvoices() + if (error) throw error + failedInvoices = data.failedInvoices + } catch (err) { + console.error('failed to fetch invoices to retry:', err) + return + } + + for (const inv of failedInvoices) { + try { + await retry(inv) + } catch (err) { + // some retries are expected to fail since only one client at a time is allowed to retry + // these should show up as 'invoice not found' errors + console.error('retry failed:', err) + } + } + } + + let timeout, stopped + const queuePoll = () => { + timeout = setTimeout(async () => { + try { + await retryPoll() + } catch (err) { + // every error should already be handled in retryPoll + // but this catch is a safety net to not trigger an unhandled promise rejection + console.error('retry poll failed:', err) + } + if (!stopped) queuePoll() + }, NORMAL_POLL_INTERVAL) + } + + const stopPolling = () => { + stopped = true + clearTimeout(timeout) + } + + queuePoll() + return stopPolling + }, [wallets, getFailedInvoices, retry]) + + return children } diff --git a/wallets/payment.js b/wallets/payment.js index 043d57f89..7cb0bec64 100644 --- a/wallets/payment.js +++ b/wallets/payment.js @@ -17,7 +17,7 @@ export function useWalletPayment () { const loggerFactory = useWalletLoggerFactory() const invoiceHelper = useInvoice() - return useCallback(async (invoice, { waitFor, updateOnFallback }) => { + return useCallback(async (invoice, { waitFor, updateOnFallback } = {}) => { let aggregateError = new WalletAggregateError([]) let latestInvoice = invoice diff --git a/wallets/server.js b/wallets/server.js index bcf1078c1..58c00c11b 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -24,9 +24,13 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln] const MAX_PENDING_INVOICES_PER_WALLET = 25 -export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) { +export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { paymentAttempt, predecessorId, models }) { // get the wallets in order of priority - const wallets = await getInvoiceableWallets(userId, { predecessorId, models }) + const wallets = await getInvoiceableWallets(userId, { + paymentAttempt, + predecessorId, + models + }) msats = toPositiveNumber(msats) @@ -79,7 +83,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa export async function createWrappedInvoice (userId, { msats, feePercent, description, descriptionHash, expiry = 360 }, - { predecessorId, models, me, lnd }) { + { paymentAttempt, predecessorId, models, me, lnd }) { let logger, bolt11 try { const { invoice, wallet } = await createInvoice(userId, { @@ -88,7 +92,7 @@ export async function createWrappedInvoice (userId, description, descriptionHash, expiry - }, { predecessorId, models }) + }, { paymentAttempt, predecessorId, models }) logger = walletLogger({ wallet, models }) bolt11 = invoice @@ -108,7 +112,7 @@ export async function createWrappedInvoice (userId, } } -export async function getInvoiceableWallets (userId, { predecessorId, models }) { +export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it // so it has not been updated yet. @@ -141,6 +145,7 @@ export async function getInvoiceableWallets (userId, { predecessorId, models }) FROM "Invoice" JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId" WHERE "Invoice"."actionState" = 'RETRYING' + AND "Invoice"."paymentAttempt" = ${paymentAttempt} ) SELECT "InvoiceForward"."walletId"