refactor: SocialProvider, TwitterBot

.gitignore

@@ -13,3 +13,4 @@ tags
 /backups/
 /alembic/versions*
 /.coverage
+/media/

charts/donate4fun.stage.yaml

@@ -6,7 +6,10 @@ backend:
   config:
     base_url: https://stage.donate4.fun
     twitter:
-      enable_bot: false
+      conversations_bot:
+        enabled: false
+      mentions_bot:
+        enabled: true
     posthog:
       debug: true
     sentry:

charts/donate4fun.yaml

@@ -14,6 +14,7 @@ backend:
     repository: europe-central2-docker.pkg.dev/donate4fun-prod/docker/donate4fun-backend
   config:
     posthog:
+      enabled: true
       project_api_key: 'phc_2CphDSfOn61NrqYZnloxHWpwFFjd4mHxUtZwcwrogC0'
       host: 'https://eu.posthog.com'
     sentry:
@@ -34,8 +35,11 @@ backend:
     youtube:
       refresh_timeout: P1D
     twitter:
-      greeting: "Hey! I'm a Donate4Fun Twitter bot. I'll check if you have any tips or donations."
-      enable_bot: false
+      conversations_bot:
+        greeting: "Hey! I'm a Donate4Fun Twitter bot. I'll check if you have any tips or donations."
+        enabled: false
+      mentions_bot:
+        enabled: false
       dm_check_interval: PT20S
       self_id: "1572908920485576704"
       refresh_timeout: P1D
@@ -80,6 +84,8 @@ backend:
       loggers:
         donate4fun:
           level: DEBUG
+        donate4fun.twitter_bot:
+          level: TRACE
         sqlalchemy.engine.Engine:
           level: WARNING
         hypercorn:

charts/secrets.donate4fun.yaml

@@ -12,12 +12,20 @@ backend:
             client_id: ENC[AES256_GCM,data:HxiY4yLoRtuHQshVNpp/HgRbvr0=,iv:p6gjZK1XQwna16UeBbEjm18GgRnTgY1WaGEW2Hbz/V4=,tag:ulFrkKFQJlgZSFCgP++XUw==,type:str]
             client_secret: ENC[AES256_GCM,data:ydFfKuPN3/AnGBGd6wZptNlniNF2lE7SQ3zacVzSrMLlnc0eUDPg/Q==,iv:bH+H4flBHEZU61uIxe0Q58JVcpjedomFpflt5aY+mIw=,tag:rJqRlpiHopZoiWXg8XY2ow==,type:str]
         twitter:
-            bearer_token: ENC[AES256_GCM,data:mGXAzh0LsL4Yk3pgt8icKZ9D9rHffZPDxlB2Cz4qy61SdF1ZHcL9NYlWYexdn9WOo2GGyAr0UtN2tWimbhIJp/GK/WnatStbGEd7349RZskRyBn9v9ug0FQftSHQ3Y+u9fjs6k1FJ7Rb5CQHYJPeRw==,iv:me5EQnETByFIkKcohYjUty8h523IdviKWSku6ICj5Yw=,tag:V8o6eapFWFw+Bfh2w6rsxw==,type:str]
-            oauth:
-                client_id: ENC[AES256_GCM,data:5PonrJSWD6kxYizZM8pWRpApTasbMLKd4Ybw2LPS4K0+Tg==,iv:/J9ThtCpohM5hNSVzAM1SSS0n3ytSBh39Bh8GmFnBmc=,tag:KbI90JuYoIXrO2D6GekqLg==,type:str]
-                client_secret: ENC[AES256_GCM,data:X7iDPNp2qmFNAZKDStGp2Pzl4n5L+k3otyB0vRZ8JnkAMJDA3rxoM84WbwznqV8mqMY=,iv:sTXbzQrPzkyRgeGxOmQ3ClEDsW1MrJzM4sSREfyAmm4=,tag:zzyEgtt3soFfQGis22yyEA==,type:str]
-                consumer_key: ENC[AES256_GCM,data:nJBxgXHOYlSAN2g7Mm2lJ3dnW+RuATmdaw==,iv:Q45as5sEqtJUyA7ZaaCztfAPbUIe/wbIH9rhd6Wu4uo=,tag:9ueOeIzW6qZtSzAw241FHw==,type:str]
-                consumer_secret: ENC[AES256_GCM,data:lsnZdH3dv0kpVcaRsgu6qDHt7qy57vfdzYcem5fbhp91V4hW+hdAY9PHyo97uHPgW00=,iv:OyqhdBcrohpspA2JiKDzEtjohJn3CDK3M4zU6U0wtd0=,tag:hs3IZFPuWYw3Xhq3N+Ln6A==,type:str]
+            linking_oauth:
+                bearer_token: ENC[AES256_GCM,data:x3OCP3ign7505bnfnPkfSfrRYu2/q1trlRZjP/Vik1ZOXU/utVova8Mp/BphkTtKVBehMj6+4JUuxKLmOioIvkpTYD1CJpo4NW6w/KmBbtWDekHHkH0Sj9/mCgqLyC16gQy5VRzx9mgsgFlUFrI4Sw==,iv:F/BLrVjBPMOJz8lBzSRG674L9QzXo+QG0XxrVFqonmc=,tag:gI+Bx3Fq7PnnSfsbgSXCVQ==,type:str]
+                client_id: ENC[AES256_GCM,data:8CeK5aa+LlksLBaBQcElAj5hVrVqQELnHh7E3/hjQYzfrw==,iv:QYFHs28pjScz284/J52djIPGAOh8tfreYDxVBhTp4GQ=,tag:l5cJJoSJ+VRUe3LEonkbng==,type:str]
+                client_secret: ENC[AES256_GCM,data:UUxRIXZd3DzN9hvSzA41w58E4NYBGdWHkH2JAQmce/IebMFwbXhp56mtrpgETMPtSlQ=,iv:tJz57ur+3nOe52+TNPSB57NjBm2b1b8p6cNXU7GvRMw=,tag:uixfO4opqqcuYti1Uh/QJQ==,type:str]
+                consumer_key: ENC[AES256_GCM,data:a9ve0hGRaMJdp+pnty6Zr+Uo3jcn/M9dVQ==,iv:22FIeTFRqci9kw7Ef0GCJ+0bZ/s3XCex/679xhBOhX8=,tag:rA0zYZcgFzN2/6lx66/GDg==,type:str]
+                consumer_secret: ENC[AES256_GCM,data:2s8IpyDR9tafI7YhzB9SBOpGZ+BLuyq4AkvJuAFl/VnKrbbTMJWk8C/PR+5ZXiUbYbw=,iv:imWfmMgQnTmuPAcf8vEdRiBT/7Ass9QE7kJeoNOIC2g=,tag:hV6zGlz+64sMIS5iPeCf0A==,type:str]
+            mentions_bot:
+                oauth:
+                    #ENC[AES256_GCM,data:xjELLPG1sqcAyu8=,iv:3f1CRU/cRacKBCBGsBFQVdjZLJiz6H/TA3b3XIDYeRs=,tag:HpThBJcvM62IJL1Hh0JR7w==,type:comment]
+                    bearer_token: ENC[AES256_GCM,data:fVrI/PmbTjwRNsn6WxrMJl/jTbOrGJLS6fb0JAAho1tBM8rFxDMIwknnh6sTk7Raxemdam/pZgL6Ic9Zj43LeZaknv7Yyzo48UF8N2sSAx9e/F2iqGE3VMT6VZihnZaRlGr/noPBLFYYkIcG8T3hlHWp0RE=,iv:IGtRrLBTPsnlzi7RFe7arfhi4IEiSRLvvo2BSTRWSR8=,tag:b71FjwSqcQ6g6EwNs9BAsQ==,type:str]
+                    consumer_key: ENC[AES256_GCM,data:faK5mmm9R+ke1GOFNCUEtx/TD0y1pMsdpg==,iv:bEBu/8jwnv0HJs2ug01JXaM/k6/T5IMnH7lVXygv1V4=,tag:OKXbHQFNWalzvvnf1MQ21g==,type:str]
+                    consumer_secret: ENC[AES256_GCM,data:wPSkJFGT0d0ygEQLbZxXIA1qt2gLk6cDEoCRJjLNp7n9xboTvxixodSFHfDvjpcGUf4=,iv:wekZsQbMl6wzhdPZv6bsB4n74RbznWxb7bqHHYHPk4Q=,tag:cOkcbDoppxBfyRNvM8VbsA==,type:str]
+                    client_id: ENC[AES256_GCM,data:dW4gap0DwMKr0VPHkZb51PsQBU+k1dGplAz55c1KjzWgUQ==,iv:8ErZKVxrgYE3a9lwEUz3a4U9796cusyD9w5PfpkeP/g=,tag:zgJSjsad+p5v24+5CF2S3g==,type:str]
+                    client_secret: ENC[AES256_GCM,data:UEiaQuy0It5hxPEcoOfIG1+bJNAgRCLGZv0UNL1Y2B9hS6WQj2XruHbTw91TLw06guw=,iv:ZbrE4DS0mVveRwez5amOzhiib9HyRqOZR2kPFvmAAvU=,tag:EcP4keJZl7OYsFcW4KFoSA==,type:str]
         db:
             url: ENC[AES256_GCM,data:HYoegvzZAkbqbH45y70VHZ/uXU9l+XtSCxf4/8i9O26QkOu+D9uhTUtP5sPBA9OOoa4rXv+bKU8Rk4oijYibNX1zPAQWKk3mH0DR,iv:idrSNHClGy0bKvHvuoT2wziOOeTdFhzm+Gp+KhHLhyI=,tag:amlcx7UDiAAFs0Q98lASRA==,type:str]
         jwt_secret: ENC[AES256_GCM,data:M5RNlxRvdu1Otd6r9zs96U40C0tKjTyRH/+atkl03WY=,iv:d3bffJmniDw2HaiNVZVdF7GkZ4LkU2KucFUtOLWArN0=,tag:wHyPVhegiajbbwMJUEULMw==,type:str]
@@ -44,8 +52,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-02-02T17:53:25Z"
-    mac: ENC[AES256_GCM,data:N+3tyXTOvxpTTH0NvqPzLvcXJWeBVoO7bKC/JNRxw0ULxnm6KeTt4iooyBJhv+uKiJ8JhWUEGVc+jx60+8iegZydz6hV9uEZGIRIWoLD+sZVQcw57wEGHTnL+bTaWu8ONy2cBVWFmAN5tDOPyNHlIbLQ1sB9V0tej+GOeeXHfqI=,iv:4sp4ss7zUfs+OHBnAjkS4PeEXTNb2Vg2w4OuUN+WnYQ=,tag:MILixSy9VrdXmAGzXzS9ew==,type:str]
+    lastmodified: "2023-02-11T13:42:02Z"
+    mac: ENC[AES256_GCM,data:+XNlBaVZp8qqd9TLUH4uo4bLKb4Y3DSNpQ3kfrVW+VbhwoW8Uk76rbgdAPLWx2wIMz+IXliQ6pUDSLE5AY+wJxqiIASWIUJEsyNJkNcK8tAHrPasdFwri+bJjlcuXJWj4PONaef5KBbInbdcZUyoOeOoqkx2deTcLqJWBuNM4J4=,iv:OKPGf1HyiDJzvxGTsxqUmVX1SKlhWC50yojp3ntMWPY=,tag:liTYcZdapSuJnKnG2jfL6Q==,type:str]
     pgp:
         - created_at: "2022-06-19T10:27:57Z"
           enc: |

config-test.yaml

@@ -6,13 +6,18 @@ youtube:
     client_id: client_id
     client_secret: client_secret
 twitter:
-  bearer_token: bearer_token
-  greeting: ""
-  enable_bot: false
   self_id: "123"
   dm_check_interval: PT30S
   refresh_timeout: P1D
-  oauth:
+  conversations_bot:
+    enabled: false
+    oauth: null
+    greeting: ""
+  mentions_bot:
+    enabled: false
+    oauth: null
+  linking_oauth:
+    bearer_token: bearer_token
     client_id: client_id
     client_secret: client_secret
     consumer_key: consumer_key

donate4fun/api.py

@@ -18,15 +18,16 @@
 from jwt import InvalidTokenError
 
 from .models import (
-    Donation, Donator, Invoice,
+    Donation, Donator, Invoice, SocialAccount,
     WithdrawalToken, BaseModel, Notification, Credentials, SubscribeEmailRequest,
-    DonatorStats, PayInvoiceResult, Donatee, OAuthState, SocialProvider, Toast,
+    DonatorStats, PayInvoiceResult, Donatee, OAuthState, Toast, SocialProviderId,
 )
 from .types import ValidationError, PaymentRequest, OAuthError, LnurlpError, AccountAlreadyLinked
 from .core import to_base64
 from .db_models import WithdrawalDb
 from .db_libs import WithdrawalDbLib, DonationsDbLib, OtherDbLib
 from .settings import settings
+from .social import SocialProvider
 from .api_utils import (
     get_donator, load_donator, get_db_session, task_group, only_me, make_redirect, get_donations_db, sha256hash,
     oauth_success_messages, signin_success_message,
@@ -222,14 +223,16 @@ class PaymentCallbackResponse(BaseModel):
     pr: PaymentRequest
 
 
-@router.get('/lnurl/{receiver_id}/payment-callback', response_model=PaymentCallbackResponse)
+@router.get('/lnurl/{provider_id}/{receiver_id}/payment-callback', response_model=PaymentCallbackResponse)
 async def payment_callback(
-    request: Request, receiver_id: UUID, amount: int = Query(...), comment: str = Query(...), db_session=Depends(get_db_session),
+    request: Request, provider_id: SocialProviderId, receiver_id: UUID, amount: int = Query(...), comment: str = Query(...),
+    db_session=Depends(get_db_session),
 ):
     """
-    This callback is needed for lightning address support. Currently it's used for internal testing only.
+    This callback is needed for lightning address support.
     """
-    receiver: Donator = await db_session.query_donator(id=receiver_id)
+    provider = SocialProvider.create(provider_id)
+    receiver: SocialAccount | Donator = await provider.wrap_db(db_session).query_account(id=receiver_id)
     amount = amount // 1000  # FIXME: handle msats correctly
     invoice: Invoice = await lnd.create_invoice(
         memo=comment, value=amount, description_hash=to_base64(sha256hash(lightning_payment_metadata(receiver))),
@@ -397,7 +400,7 @@ async def withdraw(request: Request, db=Depends(get_db_session), me: Donator = D
 
 @router.get('/oauth-redirect/{provider}')
 async def oauth_redirect(
-    request: Request, provider: SocialProvider, state: str, error: str = None, error_description: str = None, code: str = None,
+    request: Request, provider: SocialProviderId, state: str, error: str = None, error_description: str = None, code: str = None,
     donator=Depends(get_donator),
 ):
     try:

donate4fun/api_donation.py

@@ -1,50 +1,34 @@
-import json
 import logging
 import hashlib
 from uuid import UUID
 from datetime import datetime
 
-import httpx
 from fastapi import Request, Depends, HTTPException, APIRouter
+from furl import furl
 from sqlalchemy import select
-from lnpayencode import LnAddr
 
 from .models import (
-    Donation, Donator, Invoice, DonateResponse, DonateRequest,
-    DonationPaidRequest, PayInvoiceResult, PaymentRequest, RequestHash
+    Donation, Donator, Invoice, DonateResponse, DonateRequest, DonationPaidRequest, RequestHash
 )
-from .types import ValidationError, LnurlpError
+from .types import ValidationError
+from .social import SocialProviderId, SocialProvider
 from .api_utils import (
-    get_donator, get_db_session, load_donator, auto_transfer_donations, track_donation, HttpClient, get_donations_db, only_me,
-    sha256hash, get_social_provider_db,
+    get_donator, get_db_session, load_donator, track_donation, get_donations_db, only_me,
 )
 from .db_libs import GithubDbLib, TwitterDbLib, YoutubeDbLib, DonationsDbLib
 from .db_donations import sent_donations_subquery, received_donations_subquery
 from .db_models import DonationDb
 from .db_social import SocialDbWrapper
-from .twitter import query_or_fetch_twitter_account
-from .donatees import apply_target
 from .lnd import lnd
 from .settings import settings
+from .donation import donate, make_memo
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def make_memo(donation: Donation) -> str:
-    if account := donation.receiver_social_account:
-        return f"Donate4.Fun donation to {account.unique_name}"
-    elif donation.receiver:
-        if donation.receiver.id == donation.donator.id:
-            return f"[Donate4.fun] fulfillment for {donation.receiver.name}"
-        else:
-            return f"[Donate4.fun] donation to {donation.receiver.name}"
-    else:
-        raise ValueError(f"Could not make a memo for donation {donation}")
-
-
 @router.post("/donate", response_model=DonateResponse)
-async def donate(
+async def donate_api(
     web_request: Request, request: DonateRequest, donator: Donator = Depends(get_donator), db_session=Depends(get_db_session),
  ) -> DonateResponse:
     logger.debug(
@@ -59,17 +43,20 @@ async def donate(
         donation.lightning_address = request.lightning_address
     if request.receiver_id:
         # Donation to a donator - possibly just an own balance fulfillment
-        receiver = await load_donator(db_session, request.receiver_id)
+        receiver: Donator = await load_donator(db_session, request.receiver_id)
         if not receiver.connected:
             raise ValidationError("Money receiver should have a connected wallet")
         donation.receiver = receiver
     elif request.social_account_id and request.social_provider:
-        db_lib: SocialDbWrapper = get_social_provider_db(request.social_provider)
-        social_account = await db_lib(db_session).query_account(id=request.social_account_id)
-        setattr(donation, db_lib.donation_field, social_account)
+        provider: SocialProvider = SocialProvider.create(request.social_provider)
+        social_db: SocialDbWrapper = provider.wrap_db(db_session)
+        social_account = await social_db.query_account(id=request.social_account_id)
+        setattr(donation, social_db.donation_field, social_account)
         donation.lightning_address = donation.lightning_address or social_account.lightning_address
     elif request.target:
-        await apply_target(donation, request.target, db_session)
+        target = furl(request.target)
+        social_provider = SocialProvider.from_url(target)
+        await social_provider.apply_target(donation, target, db_session)
     # FIXME: the following elif blocks are deprecated
     elif request.channel_id:
         donation.youtube_channel = await YoutubeDbLib(db_session).query_account(id=request.channel_id)
@@ -83,116 +70,15 @@ async def donate(
     else:
         raise ValidationError("donation should have a target, channel_id or receiver_id")
     if request.donator_twitter_handle:
-        donation.donator_twitter_account = await query_or_fetch_twitter_account(
-            db=TwitterDbLib(db_session), handle=request.donator_twitter_handle,
-        )
-    donator = await load_donator(db_session, donator.id)
-    # If donator has enough money (and not fulfilling his own balance) - try to pay donation instantly
-    use_balance = (
-        request.receiver_id != donator.id
-        and (donator.available_balance if donation.lightning_address else donator.balance) >= request.amount
-    )
-    if donation.lightning_address:
-        pay_req: PaymentRequest = await fetch_lightning_address(donation)
-        r_hash = RequestHash(pay_req.decode().paymenthash)
-        if use_balance:
-            # FIXME: this leads to 'duplicate key value violates unique constraint "donation_rhash_key"'
-            # if we are donating to a local lightning address
-            donation.r_hash = r_hash
-        else:
-            donation.transient_r_hash = r_hash
-    elif not use_balance:
-        invoice: Invoice = await lnd.create_invoice(memo=make_memo(donation), value=request.amount)
-        donation.r_hash = invoice.r_hash  # This hash is needed to find and complete donation after payment succeeds
-        pay_req = invoice.payment_request
-    donations_db = DonationsDbLib(db_session)
-    await donations_db.create_donation(donation)
-    if use_balance:
-        if donation.lightning_address:
-            pay_result: PayInvoiceResult = await lnd.pay_invoice(pay_req)
-            amount = pay_result.value_sat
-            paid_at = pay_result.creation_date
-            fee_msat = pay_result.fee_msat
-            claimed_at = pay_result.creation_date
-        else:
-            amount = request.amount
-            paid_at = datetime.utcnow()
-            fee_msat = None
-            claimed_at = None
-        await donations_db.donation_paid(
-            donation_id=donation.id, amount=amount, paid_at=paid_at, fee_msat=fee_msat, claimed_at=claimed_at,
+        twitter_provider = SocialProvider.create(SocialProviderId.twitter.value)
+        donation.donator_twitter_account = await twitter_provider.query_or_fetch_account(
+            twitter_provider.wrap_db(db_session), handle=request.donator_twitter_handle
         )
-        await auto_transfer_donations(db_session, donation)
-        # Reload donation with a fresh state
-        donation = await donations_db.query_donation(id=donation.id)
+    pay_req, donation = await donate(donation, db_session)
+    if pay_req is None:
         # FIXME: balance is saved in cookie to notify extension about balance change, but it should be done via VAPID
         web_request.session['balance'] = (await db_session.query_donator(id=donator.id)).balance
-        track_donation(donation)
-        return DonateResponse(donation=donation, payment_request=None)
-    else:
-        return DonateResponse(donation=donation, payment_request=pay_req)
-
-
-async def fetch_lightning_address(donation: Donation) -> PaymentRequest:
-    name, host = donation.lightning_address.split('@', 1)
-    async with HttpClient() as client:
-        try:
-            response = await client.get(f'https://{host}/.well-known/lnurlp/{name}', follow_redirects=True)
-        except httpx.HTTPStatusError as exc:
-            raise LnurlpError(f"{exc.request.url} responded with {exc.response.status_code}: {exc.response.content}") from exc
-        except httpx.HTTPError as exc:
-            raise LnurlpError(f"HTTP error with {exc.request.url}: {exc}") from exc
-        metadata = response.json()
-        # https://github.com/lnurl/luds/blob/luds/06.md
-        if metadata.get('status', 'OK') != 'OK':
-            raise LnurlpError(f"Status is not OK: {metadata}")
-        if not metadata['minSendable'] <= donation.amount * 1000 <= metadata['maxSendable']:
-            raise LnurlpError(f"Amount is out of bounds: {donation.amount} {metadata}")
-        fields = dict(json.loads(metadata['metadata']))
-        if donation.donator_twitter_account:
-            name = '@' + donation.donator_twitter_account.handle
-        else:
-            name = donation.donator.name
-        params = dict(amount=donation.amount * 1000)
-        if payerdata_request := metadata.get('payerData'):
-            payerdata = {}
-            if 'name' in payerdata_request:
-                payerdata['name'] = f'{name} via Donate4.Fun'
-            # Separators are important for hashes to match
-            params['payerdata'] = json.dumps(payerdata, separators=(',', ':'))
-        if donation.youtube_video:
-            target = f'https://youtube.com/watch?v={donation.youtube_video.video_id}'
-        elif donation.twitter_tweet:
-            target = f'https://twitter.com/{donation.twitter_account.handle}/status/{donation.twitter_tweet.tweet_id}'
-        elif donation.youtube_channel:
-            target = f'https://youtube.com/channel/{donation.youtube_channel.channel_id}'
-        elif donation.twitter_account:
-            target = f'https://twitter.com/{donation.twitter_account.handle}'
-        elif donation.lightning_address:
-            target = fields.get('text/identifier', donation.lightning_address)
-        comment = f'Tip from {name} via Donate4.Fun for {target}'
-        if 'commentAllowed' in metadata:
-            params['comment'] = comment[:metadata['commentAllowed']]
-        try:
-            response = await client.get(metadata['callback'], params=params)
-        except httpx.HTTPStatusError as exc:
-            raise LnurlpError(response.content) from exc
-        except httpx.HTTPError as exc:
-            raise LnurlpError(exc) from exc
-        data = response.json()
-        if data.get('status', 'OK') != 'OK':
-            raise LnurlpError(f"Status is not OK: {data}")
-        pay_req = PaymentRequest(data['pr'])
-        invoice: LnAddr = pay_req.decode()
-        expected_hash = dict(invoice.tags)['h']
-        # https://github.com/lnurl/luds/blob/luds/18.md#3-committing-payer-to-the-invoice
-        full_metadata: str = metadata['metadata'] + params.get('payerdata', '')
-        if sha256hash(full_metadata) != expected_hash:
-            raise LnurlpError(f"Metadata hash does not match invoice hash: sha256({full_metadata}) != {expected_hash}")
-        invoice_amount = invoice.amount * 10 ** 8
-        if invoice_amount != donation.amount:
-            raise LnurlpError(f"Amount in invoice does not match requested amount: {invoice_amount} != {donation.amount}")
-        return pay_req
+    return DonateResponse(donation=donation, payment_request=pay_req)
 
 
 @router.get("/donation/{donation_id}", response_model=DonateResponse)