Automated retries

api/paidAction/index.js

@@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) {
 }
 
 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context
   const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
 
   const db = tx ?? models
@@ -445,6 +445,7 @@ async function createDbInvoice (actionType, args, context) {
     actionArgs: args,
     expiresAt,
     actionId,
+    paymentAttempt,
     predecessorId
   }
 

api/resolvers/notifications.js

@@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate'
 import { replyToSubscription } from '@/lib/webPush'
 import { getSub } from './sub'
 import { GqlAuthenticationError, GqlInputError } from '@/lib/error'
+import { WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
 
 export default {
   Query: {
@@ -350,6 +351,13 @@ export default {
         WHERE "Invoice"."userId" = $1
         AND "Invoice"."updated_at" < $2
         AND "Invoice"."actionState" = 'FAILED'
+        AND (
+          -- this is the inverse of the filter for automated retries
+          "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES}
+          OR "Invoice"."actionType" = 'ITEM_CREATE'
+          OR "Invoice"."userCancel" = true
+          OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+        )
         AND (
           "Invoice"."actionType" = 'ITEM_CREATE' OR
           "Invoice"."actionType" = 'ZAP' OR

api/resolvers/paidAction.js

@@ -1,5 +1,5 @@
 import { retryPaidAction } from '../paidAction'
-import { USER_ID } from '@/lib/constants'
+import { USER_ID, WALLET_MAX_RETRIES, WALLET_RETRY_TIMEOUT_MS } from '@/lib/constants'
 
 function paidActionType (actionType) {
   switch (actionType) {
@@ -50,24 +50,32 @@ export default {
     }
   },
   Mutation: {
-    retryPaidAction: async (parent, { invoiceId }, { models, me, lnd }) => {
+    retryPaidAction: async (parent, { invoiceId, newAttempt }, { models, me, lnd }) => {
       if (!me) {
         throw new Error('You must be logged in')
       }
 
-      const invoice = await models.invoice.findUnique({ where: { id: invoiceId, userId: me.id } })
+      // make sure only one client at a time can retry by acquiring a lock that expires
+      const [invoice] = await models.$queryRaw`
+          UPDATE "Invoice"
+          SET "retryPendingSince" = now()
+          WHERE
+            id = ${invoiceId} AND
+            "userId" = ${me.id} AND
+            "actionState" = 'FAILED' AND
+            ("retryPendingSince" IS NULL OR "retryPendingSince" < now() - ${`${WALLET_RETRY_TIMEOUT_MS} milliseconds`}::interval)
+          RETURNING *`
       if (!invoice) {
-        throw new Error('Invoice not found')
+        throw new Error('Invoice not found or retry pending')
       }
 
-      if (invoice.actionState !== 'FAILED') {
-        if (invoice.actionState === 'PAID') {
-          throw new Error('Invoice is already paid')
-        }
-        throw new Error(`Invoice is not in failed state: ${invoice.actionState}`)
+      // do we want to retry a payment from the beginning with all sender and receiver wallets?
+      const paymentAttempt = newAttempt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt
+      if (paymentAttempt > WALLET_MAX_RETRIES) {
+        throw new Error('Payment has been retried too many times')
       }
 
-      const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd })
+      const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd })
 
       return {
         ...result,

api/resolvers/user.js

@@ -4,9 +4,9 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor'
 import { msatsToSats } from '@/lib/format'
 import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate'
 import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item'
-import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES } from '@/lib/constants'
+import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
 import { viewGroup } from './growth'
-import { timeUnitForRange, whenRange } from '@/lib/time'
+import { datePivot, timeUnitForRange, whenRange } from '@/lib/time'
 import assertApiKeyNotPermitted from './apiKey'
 import { hashEmail } from '@/lib/crypto'
 import { isMuted } from '@/lib/user'
@@ -543,7 +543,25 @@ export default {
           actionType: {
             in: INVOICE_ACTION_NOTIFICATION_TYPES
           },
-          actionState: 'FAILED'
+          actionState: 'FAILED',
+          OR: [
+            {
+              paymentAttempt: {
+                gte: WALLET_MAX_RETRIES
+              }
+            },
+            {
+              actionType: 'ITEM_CREATE'
+            },
+            {
+              userCancel: true
+            },
+            {
+              cancelledAt: {
+                lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS })
+              }
+            }
+          ]
         }
       })
 

api/resolvers/wallet.js

@@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib
 import {
   USER_ID, INVOICE_RETENTION_DAYS,
   PAID_ACTION_PAYMENT_METHODS,
-  WALLET_CREATE_INVOICE_TIMEOUT_MS
+  WALLET_CREATE_INVOICE_TIMEOUT_MS,
+  WALLET_RETRY_AFTER_MS,
+  WALLET_RETRY_BEFORE_MS,
+  WALLET_MAX_RETRIES
 } from '@/lib/constants'
 import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate'
 import assertGofacYourself from './ofac'
@@ -456,6 +459,23 @@ const resolvers = {
         cursor: nextCursor,
         entries: logs
       }
+    },
+    failedInvoices: async (parent, args, { me, models }) => {
+      if (!me) {
+        throw new GqlAuthenticationError()
+      }
+      return await models.$queryRaw`
+        SELECT * FROM "Invoice"
+        WHERE "userId" = ${me.id}
+        AND "actionState" = 'FAILED'
+        -- never retry if user has cancelled the invoice manually
+        AND "userCancel" = false
+        AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval
+        AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval
+        AND "paymentAttempt" < ${WALLET_MAX_RETRIES}
+        -- never retry failed posts because we always immediately show them in notifications
+        AND "actionType" <> 'ITEM_CREATE'
+        ORDER BY id DESC`
     }
   },
   Wallet: {

api/typeDefs/paidAction.js

@@ -7,7 +7,7 @@ extend type Query {
 }
 
 extend type Mutation {
-  retryPaidAction(invoiceId: Int!): PaidAction!
+  retryPaidAction(invoiceId: Int!, newAttempt: Boolean): PaidAction!
 }
 
 enum PaymentMethod {

api/typeDefs/wallet.js

@@ -72,6 +72,7 @@ const typeDefs = `
     wallet(id: ID!): Wallet
     walletByType(type: String!): Wallet
     walletLogs(type: String, from: String, to: String, cursor: String): WalletLog!
+    failedInvoices: [Invoice!]!
   }
 
   extend type Mutation {

components/use-invoice.js

@@ -1,5 +1,5 @@
 import { useApolloClient, useMutation } from '@apollo/client'
-import { useCallback } from 'react'
+import { useCallback, useMemo } from 'react'
 import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
@@ -42,9 +42,9 @@ export default function useInvoice () {
     return data.cancelInvoice
   }, [cancelInvoice])
 
-  const retry = useCallback(async ({ id, hash, hmac }, { update }) => {
+  const retry = useCallback(async ({ id, hash, hmac, newAttempt = false }, { update } = {}) => {
     console.log('retrying invoice:', hash)
-    const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update })
+    const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id), newAttempt }, update })
     if (error) throw error
 
     const newInvoice = data.retryPaidAction.invoice
@@ -53,5 +53,5 @@ export default function useInvoice () {
     return newInvoice
   }, [retryPaidAction])
 
-  return { cancel, retry, isInvoice }
+  return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice])
 }

fragments/paidAction.js

@@ -91,8 +91,8 @@ export const RETRY_PAID_ACTION = gql`
   ${PAID_ACTION}
   ${ITEM_PAID_ACTION_FIELDS}
   ${ITEM_ACT_PAID_ACTION_FIELDS}
-  mutation retryPaidAction($invoiceId: Int!) {
-    retryPaidAction(invoiceId: $invoiceId) {
+  mutation retryPaidAction($invoiceId: Int!, $newAttempt: Boolean) {
+    retryPaidAction(invoiceId: $invoiceId, newAttempt: $newAttempt) {
       __typename
       ...PaidActionFields
       ... on ItemPaidAction {

fragments/wallet.js

@@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql`
     }
   }
 `
+
+export const FAILED_INVOICES = gql`
+  ${INVOICE_FIELDS}
+  query FailedInvoices {
+    failedInvoices {
+      ...InvoiceFields
+    }
+  }
+`

lib/apollo.js

@@ -283,6 +283,12 @@ function getClient (uri) {
                   facts: [...(existing?.facts || []), ...incoming.facts]
                 }
               }
+            },
+            failedInvoices: {
+              keyArgs: [],
+              merge (existing, incoming) {
+                return incoming
+              }
             }
           }
         },

lib/constants.js

@@ -198,4 +198,14 @@ export const ZAP_UNDO_DELAY_MS = 5_000
 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000
 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000
 
+// interval between which failed invoices are returned to a client for automated retries.
+// retry-after must be high enough such that intermediate failed invoices that will already
+// be retried by the client due to sender or receiver fallbacks are not returned to the client.
+export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute
+export const WALLET_RETRY_BEFORE_MS = 3_600_000 // 1 hour
+// we want to attempt a payment three times so we retry two times
+export const WALLET_MAX_RETRIES = 2
+// when a pending retry for an invoice should be considered expired and can be attempted again
+export const WALLET_RETRY_TIMEOUT_MS = 60_000 // 1 minute
+
 export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l'

prisma/migrations/20250107084543_automated_retries/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "paymentAttempt" INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "Invoice" ADD COLUMN     "retryPendingSince" TIMESTAMP(3);
+CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt");

prisma/schema.prisma

@@ -928,6 +928,8 @@ model Invoice {
   cancelled          Boolean             @default(false)
   cancelledAt        DateTime?
   userCancel         Boolean?
+  paymentAttempt     Int                 @default(0)
+  retryPendingSince  DateTime?
   msatsRequested     BigInt
   msatsReceived      BigInt?
   desc               String?
@@ -956,6 +958,7 @@ model Invoice {
   @@index([confirmedIndex], map: "Invoice.confirmedIndex_index")
   @@index([isHeld])
   @@index([confirmedAt])
+  @@index([cancelledAt])
   @@index([actionType])
   @@index([actionState])
 }

wallets/index.js

@@ -1,12 +1,15 @@
 import { useMe } from '@/components/me'
-import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
-import { SSR } from '@/lib/constants'
-import { useApolloClient, useMutation, useQuery } from '@apollo/client'
+import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
+import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants'
+import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client'
 import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react'
 import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common'
 import useVault from '@/components/vault/use-vault'
 import walletDefs from '@/wallets/client'
 import { generateMutation } from './graphql'
+import { useWalletPayment } from './payment'
+import useInvoice from '@/components/use-invoice'
+import { WalletConfigurationError } from './errors'
 
 const WalletsContext = createContext({
   wallets: []
@@ -204,7 +207,9 @@ export function WalletsProvider ({ children }) {
         removeLocalWallets
       }}
     >
-      {children}
+      <RetryHandler>
+        {children}
+      </RetryHandler>
     </WalletsContext.Provider>
   )
 }
@@ -221,7 +226,79 @@ export function useWallet (name) {
 export function useSendWallets () {
   const { wallets } = useWallets()
   // return all enabled wallets that are available and can send
-  return wallets
+  return useMemo(() => wallets
     .filter(w => !w.def.isAvailable || w.def.isAvailable())
-    .filter(w => w.config?.enabled && canSend(w))
+    .filter(w => w.config?.enabled && canSend(w)), [wallets])
+}
+
+function RetryHandler ({ children }) {
+  const wallets = useSendWallets()
+  const waitForWalletPayment = useWalletPayment()
+  const invoiceHelper = useInvoice()
+  const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' })
+
+  const retry = useCallback(async (invoice) => {
+    const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true })
+
+    try {
+      await waitForWalletPayment(newInvoice)
+    } catch (err) {
+      if (err instanceof WalletConfigurationError) {
+        // consume attempt by canceling invoice
+        await invoiceHelper.cancel(newInvoice)
+      }
+      throw err
+    }
+  }, [invoiceHelper, waitForWalletPayment])
+
+  useEffect(() => {
+    // we always retry failed invoices, even if the user has no wallets on any client
+    // to make sure that failed payments will always show up in notifications eventually
+
+    const retryPoll = async () => {
+      let failedInvoices
+      try {
+        const { data, error } = await getFailedInvoices()
+        if (error) throw error
+        failedInvoices = data.failedInvoices
+      } catch (err) {
+        console.error('failed to fetch invoices to retry:', err)
+        return
+      }
+
+      for (const inv of failedInvoices) {
+        try {
+          await retry(inv)
+        } catch (err) {
+          // some retries are expected to fail since only one client at a time is allowed to retry
+          // these should show up as 'invoice not found' errors
+          console.error('retry failed:', err)
+        }
+      }
+    }
+
+    let timeout, stopped
+    const queuePoll = () => {
+      timeout = setTimeout(async () => {
+        try {
+          await retryPoll()
+        } catch (err) {
+          // every error should already be handled in retryPoll
+          // but this catch is a safety net to not trigger an unhandled promise rejection
+          console.error('retry poll failed:', err)
+        }
+        if (!stopped) queuePoll()
+      }, NORMAL_POLL_INTERVAL)
+    }
+
+    const stopPolling = () => {
+      stopped = true
+      clearTimeout(timeout)
+    }
+
+    queuePoll()
+    return stopPolling
+  }, [wallets, getFailedInvoices, retry])
+
+  return children
 }