Automated retries

api/paidAction/index.js

@@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) {
 }
 
 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context
   const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
 
   const db = tx ?? models
@@ -445,6 +445,7 @@ async function createDbInvoice (actionType, args, context) {
     actionArgs: args,
     expiresAt,
     actionId,
+    paymentAttempt,
     predecessorId
   }
 

api/resolvers/notifications.js

@@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate'
 import { replyToSubscription } from '@/lib/webPush'
 import { getSub } from './sub'
 import { GqlAuthenticationError, GqlInputError } from '@/lib/error'
+import { WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
 
 export default {
   Query: {
@@ -345,11 +346,25 @@ export default {
       )
 
       queries.push(
-        `(SELECT "Invoice".id::text, "Invoice"."updated_at" AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type
+        `(SELECT "Invoice".id::text,
+          CASE
+            WHEN
+              "Invoice"."paymentAttempt" < ${WALLET_MAX_RETRIES}
+              AND "Invoice"."userCancel" = false
+              AND "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+            THEN "Invoice"."cancelledAt" + interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+            ELSE "Invoice"."updated_at"
+          END AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type
         FROM "Invoice"
         WHERE "Invoice"."userId" = $1
         AND "Invoice"."updated_at" < $2
         AND "Invoice"."actionState" = 'FAILED'
+        AND (
+          -- this is the inverse of the filter for automated retries
+          "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES}
+          OR "Invoice"."userCancel" = true
+          OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+        )
         AND (
           "Invoice"."actionType" = 'ITEM_CREATE' OR
           "Invoice"."actionType" = 'ZAP' OR

api/resolvers/paidAction.js

@@ -1,5 +1,5 @@
 import { retryPaidAction } from '../paidAction'
-import { USER_ID } from '@/lib/constants'
+import { USER_ID, WALLET_MAX_RETRIES, WALLET_RETRY_TIMEOUT_MS } from '@/lib/constants'
 
 function paidActionType (actionType) {
   switch (actionType) {
@@ -50,24 +50,32 @@ export default {
     }
   },
   Mutation: {
-    retryPaidAction: async (parent, { invoiceId }, { models, me, lnd }) => {
+    retryPaidAction: async (parent, { invoiceId, newAttempt }, { models, me, lnd }) => {
       if (!me) {
         throw new Error('You must be logged in')
       }
 
-      const invoice = await models.invoice.findUnique({ where: { id: invoiceId, userId: me.id } })
+      // make sure only one client at a time can retry by acquiring a lock that expires
+      const [invoice] = await models.$queryRaw`
+          UPDATE "Invoice"
+          SET "retryPendingSince" = now()
+          WHERE
+            id = ${invoiceId} AND
+            "userId" = ${me.id} AND
+            "actionState" = 'FAILED' AND
+            ("retryPendingSince" IS NULL OR "retryPendingSince" < now() - ${`${WALLET_RETRY_TIMEOUT_MS} milliseconds`}::interval)
+          RETURNING *`
       if (!invoice) {
-        throw new Error('Invoice not found')
+        throw new Error('Invoice not found or retry pending')
       }
 
-      if (invoice.actionState !== 'FAILED') {
-        if (invoice.actionState === 'PAID') {
-          throw new Error('Invoice is already paid')
-        }
-        throw new Error(`Invoice is not in failed state: ${invoice.actionState}`)
+      // do we want to retry a payment from the beginning with all sender and receiver wallets?
+      const paymentAttempt = newAttempt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt
+      if (paymentAttempt > WALLET_MAX_RETRIES) {
+        throw new Error('Payment has been retried too many times')
       }
 
-      const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd })
+      const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd })
 
       return {
         ...result,

api/resolvers/user.js

@@ -4,9 +4,9 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor'
 import { msatsToSats } from '@/lib/format'
 import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate'
 import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item'
-import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES } from '@/lib/constants'
+import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
 import { viewGroup } from './growth'
-import { timeUnitForRange, whenRange } from '@/lib/time'
+import { datePivot, timeUnitForRange, whenRange } from '@/lib/time'
 import assertApiKeyNotPermitted from './apiKey'
 import { hashEmail } from '@/lib/crypto'
 import { isMuted } from '@/lib/user'
@@ -543,7 +543,17 @@ export default {
           actionType: {
             in: INVOICE_ACTION_NOTIFICATION_TYPES
           },
-          actionState: 'FAILED'
+          actionState: 'FAILED',
+          OR: [
+            {
+              paymentAttempt: {
+                gte: WALLET_MAX_RETRIES
+              }
+            },
+            {
+              userCancel: true
+            }
+          ]
         }
       })
 
@@ -552,6 +562,31 @@ export default {
         return true
       }
 
+      const invoiceActionFailed2 = await models.invoice.findFirst({
+        where: {
+          userId: me.id,
+          updatedAt: {
+            gt: datePivot(lastChecked, { milliseconds: -WALLET_RETRY_BEFORE_MS })
+          },
+          actionType: {
+            in: INVOICE_ACTION_NOTIFICATION_TYPES
+          },
+          actionState: 'FAILED',
+          paymentAttempt: {
+            lt: WALLET_MAX_RETRIES
+          },
+          userCancel: false,
+          cancelledAt: {
+            lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS })
+          }
+        }
+      })
+
+      if (invoiceActionFailed2) {
+        foundNotes()
+        return true
+      }
+
       // update checkedNotesAt to prevent rechecking same time period
       models.user.update({
         where: { id: me.id },

api/resolvers/wallet.js

@@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib
 import {
   USER_ID, INVOICE_RETENTION_DAYS,
   PAID_ACTION_PAYMENT_METHODS,
-  WALLET_CREATE_INVOICE_TIMEOUT_MS
+  WALLET_CREATE_INVOICE_TIMEOUT_MS,
+  WALLET_RETRY_AFTER_MS,
+  WALLET_RETRY_BEFORE_MS,
+  WALLET_MAX_RETRIES
 } from '@/lib/constants'
 import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate'
 import assertGofacYourself from './ofac'
@@ -456,6 +459,21 @@ const resolvers = {
         cursor: nextCursor,
         entries: logs
       }
+    },
+    failedInvoices: async (parent, args, { me, models }) => {
+      if (!me) {
+        throw new GqlAuthenticationError()
+      }
+      return await models.$queryRaw`
+        SELECT * FROM "Invoice"
+        WHERE "userId" = ${me.id}
+        AND "actionState" = 'FAILED'
+        -- never retry if user has cancelled the invoice manually
+        AND "userCancel" = false
+        AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval
+        AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval
+        AND "paymentAttempt" < ${WALLET_MAX_RETRIES}
+        ORDER BY id DESC`
     }
   },
   Wallet: {

api/typeDefs/paidAction.js

@@ -7,7 +7,7 @@ extend type Query {
 }
 
 extend type Mutation {
-  retryPaidAction(invoiceId: Int!): PaidAction!
+  retryPaidAction(invoiceId: Int!, newAttempt: Boolean): PaidAction!
 }
 
 enum PaymentMethod {

api/typeDefs/wallet.js

@@ -72,6 +72,7 @@ const typeDefs = `
     wallet(id: ID!): Wallet
     walletByType(type: String!): Wallet
     walletLogs(type: String, from: String, to: String, cursor: String): WalletLog!
+    failedInvoices: [Invoice!]!
   }
 
   extend type Mutation {

components/use-invoice.js

@@ -1,5 +1,5 @@
 import { useApolloClient, useMutation } from '@apollo/client'
-import { useCallback } from 'react'
+import { useCallback, useMemo } from 'react'
 import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
@@ -42,9 +42,9 @@ export default function useInvoice () {
     return data.cancelInvoice
   }, [cancelInvoice])
 
-  const retry = useCallback(async ({ id, hash, hmac }, { update }) => {
+  const retry = useCallback(async ({ id, hash, hmac, newAttempt = false }, { update } = {}) => {
     console.log('retrying invoice:', hash)
-    const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update })
+    const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id), newAttempt }, update })
     if (error) throw error
 
     const newInvoice = data.retryPaidAction.invoice
@@ -53,5 +53,5 @@ export default function useInvoice () {
     return newInvoice
   }, [retryPaidAction])
 
-  return { cancel, retry, isInvoice }
+  return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice])
 }

fragments/paidAction.js

@@ -91,8 +91,8 @@ export const RETRY_PAID_ACTION = gql`
   ${PAID_ACTION}
   ${ITEM_PAID_ACTION_FIELDS}
   ${ITEM_ACT_PAID_ACTION_FIELDS}
-  mutation retryPaidAction($invoiceId: Int!) {
-    retryPaidAction(invoiceId: $invoiceId) {
+  mutation retryPaidAction($invoiceId: Int!, $newAttempt: Boolean) {
+    retryPaidAction(invoiceId: $invoiceId, newAttempt: $newAttempt) {
       __typename
       ...PaidActionFields
       ... on ItemPaidAction {

fragments/wallet.js

@@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql`
     }
   }
 `
+
+export const FAILED_INVOICES = gql`
+  ${INVOICE_FIELDS}
+  query FailedInvoices {
+    failedInvoices {
+      ...InvoiceFields
+    }
+  }
+`

lib/apollo.js

@@ -283,6 +283,12 @@ function getClient (uri) {
                   facts: [...(existing?.facts || []), ...incoming.facts]
                 }
               }
+            },
+            failedInvoices: {
+              keyArgs: [],
+              merge (existing, incoming) {
+                return incoming
+              }
             }
           }
         },

lib/constants.js

@@ -198,4 +198,14 @@ export const ZAP_UNDO_DELAY_MS = 5_000
 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000
 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000
 
+// interval between which failed invoices are returned to a client for automated retries.
+// retry-after must be high enough such that intermediate failed invoices that will already
+// be retried by the client due to sender or receiver fallbacks are not returned to the client.
+export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute
+export const WALLET_RETRY_BEFORE_MS = 3_600_000 // 1 hour
+// we want to attempt a payment three times so we retry two times
+export const WALLET_MAX_RETRIES = 2
+// when a pending retry for an invoice should be considered expired and can be attempted again
+export const WALLET_RETRY_TIMEOUT_MS = 60_000 // 1 minute
+
 export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l'

prisma/migrations/20250107084543_automated_retries/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "paymentAttempt" INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "Invoice" ADD COLUMN     "retryPendingSince" TIMESTAMP(3);
+CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt");

prisma/schema.prisma

@@ -928,6 +928,8 @@ model Invoice {
   cancelled          Boolean             @default(false)
   cancelledAt        DateTime?
   userCancel         Boolean?
+  paymentAttempt     Int                 @default(0)
+  retryPendingSince  DateTime?
   msatsRequested     BigInt
   msatsReceived      BigInt?
   desc               String?
@@ -956,6 +958,7 @@ model Invoice {
   @@index([confirmedIndex], map: "Invoice.confirmedIndex_index")
   @@index([isHeld])
   @@index([confirmedAt])
+  @@index([cancelledAt])
   @@index([actionType])
   @@index([actionState])
 }