chore: next patch for Twitter Bot + huge refactoring for vcr and tests

config-test.yaml

@@ -15,10 +15,16 @@ twitter:
     greeting: ""
   mentions_bot:
     enabled: false
-    oauth: null
+    oauth:
+      consumer_key: JqyhuOUhGJNO4GLeb3t3PcySP
+      consumer_secret: secret
+      # @bryskin2
+      client_id: MWlyTVlGbUdXRkY2Q09ncGxkbzc6MTpjaQ
+      client_secret: secret
   linking_oauth:
+    # @bryskin2
     bearer_token: bearer_token
-    client_id: client_id
+    client_id: MWlyTVlGbUdXRkY2Q09ncGxkbzc6MTpjaQ
     client_secret: client_secret
     consumer_key: consumer_key
     consumer_secret: consumer_secret
@@ -36,7 +42,7 @@ lnd:
   macaroon_by_path: polar/volumes/lnd/alice/data/chain/bitcoin/regtest/admin.macaroon
   lnurl_base_url: http://localhost:5173
 github:
-  client_id: xxx
+  client_id: Iv1.47885cc7f2eb65ae
   client_secret: yyy
 db:
   url: postgresql+asyncpg://donate4fun@localhost/donate4fun
@@ -101,7 +107,7 @@ log:
     asgi_testclient:
       level: DEBUG
     vcr:
-      level: WARNING
+      level: DEBUG
     sqlalchemy.engine.Engine:
       level: WARNING
     sqlalchemy.pool:

donate4fun/api.py

@@ -18,9 +18,9 @@
 from jwt import InvalidTokenError
 
 from .models import (
-    Donation, Donator, Invoice, SocialAccount,
+    Donation, Donator, SocialAccount,
     WithdrawalToken, BaseModel, Notification, Credentials, SubscribeEmailRequest,
-    DonatorStats, PayInvoiceResult, Donatee, OAuthState, Toast, SocialProviderId,
+    DonatorStats, Donatee, OAuthState, Toast, SocialProviderId,
 )
 from .types import ValidationError, PaymentRequest, OAuthError, LnurlpError, AccountAlreadyLinked
 from .core import to_base64
@@ -32,7 +32,9 @@
     get_donator, load_donator, get_db_session, task_group, only_me, make_redirect, get_donations_db, sha256hash,
     oauth_success_messages, signin_success_message,
 )
-from .lnd import PayInvoiceError, LnurlWithdrawResponse, lnd, lightning_payment_metadata, LndIsNotReady
+from .lnd import (
+    Invoice, PayInvoiceResult, PayInvoiceError, LnurlWithdrawResponse, lnd, lightning_payment_metadata, LndIsNotReady,
+)
 from .pubsub import pubsub
 from . import api_twitter, api_youtube, api_github, api_social, api_donation
 

donate4fun/api_donation.py

@@ -9,7 +9,7 @@
 from sqlalchemy import select
 
 from .models import (
-    Donation, Donator, Invoice, DonateResponse, DonateRequest, DonationPaidRequest, RequestHash
+    Donation, Donator, DonateResponse, DonateRequest, DonationPaidRequest, RequestHash
 )
 from .types import ValidationError
 from .social import SocialProviderId, SocialProvider
@@ -20,7 +20,7 @@
 from .db_donations import sent_donations_subquery, received_donations_subquery
 from .db_models import DonationDb
 from .db_social import SocialDbWrapper
-from .lnd import lnd
+from .lnd import lnd, Invoice
 from .settings import settings
 from .donation import donate, make_memo
 

donate4fun/api_twitter.py

@@ -8,8 +8,9 @@
 from .models import TwitterAccount, Donator, TwitterAccountOwned, OAuthState, OAuthResponse, Toast
 from .types import ValidationError, OAuthError, AccountAlreadyLinked, Satoshi
 from .api_utils import get_donator, make_absolute_uri, make_redirect, signin_success_message, oauth_success_messages
-from .twitter import make_oauth1_client, make_oauth2_client, TwitterApiClient
+from .twitter import TwitterApiClient
 from .twitter_bot import make_prove_message
+from .twitter_models import OAuth1Token
 from .core import app
 from .db_twitter import TwitterDbLib
 from .db import db
@@ -26,13 +27,13 @@ async def twitter_ownership_message(me=Depends(get_donator)):
 
 @router.get('/oauth1', response_model=OAuthResponse)
 async def login_via_twitter_oauth1(request: Request, donator=Depends(get_donator)):
-    oauth1_ctx = make_oauth1_client(
+    oauth1_ctx = TwitterApiClient.create_oauth1(
         oauth=settings.twitter.linking_oauth,
         redirect_uri=make_absolute_uri(request.url_for('oauth1_callback')),
     )
     async with oauth1_ctx as client:
-        await client.fetch_request_token('https://api.twitter.com/oauth/request_token')
-        auth_url: str = client.create_authorization_url('https://api.twitter.com/oauth/authorize')
+        await client.client.fetch_request_token('https://api.twitter.com/oauth/request_token')
+        auth_url: str = client.client.create_authorization_url('https://api.twitter.com/oauth/authorize')
         return OAuthResponse(url=auth_url)
 
 
@@ -46,22 +47,22 @@ async def oauth1_callback(
         raise ValidationError("oauth_token and oauth_verifier parameters must be present")
     try:
         token = dict(oauth_token=oauth_token, oauth_verifier=oauth_verifier)
-        async with make_oauth1_client(oauth=settings.twitter.linking_oauth, token=token) as client:
-            token: dict = await client.fetch_access_token('https://api.twitter.com/oauth/access_token', verifier=oauth_verifier)
-            client.token = token
-            try:
+        logger.debug("authorizing using oauth1. token: %s", token)
+        token: OAuth1Token = await TwitterApiClient.fetch_access_token(oauth_token=oauth_token, oauth_verifier=oauth_verifier)
+        try:
+            async with TwitterApiClient.create_oauth1(oauth=settings.twitter.linking_oauth, token=token) as client:
                 transferred_amount, linked_account = await login_or_link_twitter_account(client, donator)
-            except AccountAlreadyLinked as exc:
-                if not donator.connected:
-                    linked_account = exc.args[0]
-                    request.session['donator'] = str(linked_account.owner_id)
-                    request.session['connected'] = True
-                    return make_redirect('donator/me', [signin_success_message(linked_account)])
-                else:
-                    raise OAuthError("Could not link an already linked account") from exc
-            else:
+        except AccountAlreadyLinked as exc:
+            if not donator.connected:
+                linked_account = exc.args[0]
+                request.session['donator'] = str(linked_account.owner_id)
                 request.session['connected'] = True
-                return make_redirect('donator/me', oauth_success_messages(linked_account, transferred_amount))
+                return make_redirect('donator/me', [signin_success_message(linked_account)])
+            else:
+                raise OAuthError("Could not link an already linked account") from exc
+        else:
+            request.session['connected'] = True
+            return make_redirect('donator/me', oauth_success_messages(linked_account, transferred_amount))
     except OAuthError as exc:
         logger.exception("OAuth error")
         return make_redirect('settings', [Toast("error", exc.args[0], exc.__cause__)])
@@ -81,26 +82,22 @@ async def login_via_twitter(request: Request, return_to: str, expected_account:
     if len(encrypted_state) > 500:
         raise ValidationError(f"State is too long for Twitter: {len(encrypted_state)} chars")
     async with make_link_oauth2_client() as client:
-        url, state = client.create_authorization_url(
-            url='https://twitter.com/i/oauth2/authorize', code_verifier=code_verifier,
-            state=encrypted_state,
-        )
+        url = client.create_authorization_url(code_verifier=code_verifier, state=encrypted_state)
         return OAuthResponse(url=url)
 
 
 async def finish_twitter_oauth(code: str, donator: Donator, code_verifier: str) -> tuple[Satoshi, TwitterAccountOwned]:
     async with make_link_oauth2_client() as client:
         try:
-            token: dict = await client.fetch_token(code=code, code_verifier=code_verifier)
+            await client.fetch_token(code=code, code_verifier=code_verifier)
         except Exception as exc:
             raise OAuthError("Failed to fetch OAuth token") from exc
-        client.token = token
         return await login_or_link_twitter_account(client, donator)
 
 
-async def login_or_link_twitter_account(client, donator: Donator) -> tuple[Satoshi, TwitterAccountOwned]:
+async def login_or_link_twitter_account(client: TwitterApiClient, donator: Donator) -> tuple[Satoshi, TwitterAccountOwned]:
     try:
-        account: TwitterAccount = await TwitterApiClient(client).get_me()
+        account: TwitterAccount = await client.get_me()
     except Exception as exc:
         raise OAuthError("Failed to fetch user's account") from exc
 
@@ -118,13 +115,12 @@ async def login_or_link_twitter_account(client, donator: Donator) -> tuple[Satos
         raise AccountAlreadyLinked(owned_account)
 
 
-def make_link_oauth2_client(token=None):
+def make_link_oauth2_client():
     """
     This client is used to link Twitter account to donator (OAuth2 flow)
     """
-    return make_oauth2_client(
+    return TwitterApiClient.create_oauth2(
         settings.twitter.linking_oauth,
         scope='tweet.read users.read',
-        token=token,
         redirect_uri=make_absolute_uri(app.url_path_for('oauth_redirect', provider='twitter')),
     )

donate4fun/db_models.py

@@ -155,6 +155,7 @@ class DonationDb(Base):
     cancelled_at = Column(TIMESTAMP)
     claimed_at = Column(TIMESTAMP)
 
+    # FIXME: split this table to multiple tables one for each social provider
     receiver_id = Column(Uuid(as_uuid=True), ForeignKey(DonatorDb.id))
     receiver = relationship(DonatorDb, lazy='joined', foreign_keys=[receiver_id])
 
@@ -168,7 +169,9 @@ class DonationDb(Base):
     twitter_account = relationship(TwitterAuthorDb, lazy='joined', foreign_keys=[twitter_account_id])
 
     twitter_tweet_id = Column(Uuid(as_uuid=True), ForeignKey(TwitterTweetDb.id))
-    twitter_tweet = relationship(TwitterTweetDb, lazy='joined')
+    twitter_tweet = relationship(TwitterTweetDb, lazy='joined', foreign_keys=[twitter_tweet_id])
+
+    twitter_invoice_tweet_id = Column(BigInteger)
 
     github_user_id = Column(Uuid(as_uuid=True), ForeignKey(GithubUserDb.id))
     github_user = relationship(GithubUserDb, lazy='joined')

donate4fun/db_other.py

@@ -1,17 +1,18 @@
 from datetime import datetime
+from typing import Any
 from uuid import UUID
 
 from sqlalchemy import select, desc, union, literal_column
 from sqlalchemy.dialects.postgresql import insert
 
 from .models import Donatee
 from .db import DbSessionWrapper
-from .db_models import EmailNotificationDb
-from .db_libs import YoutubeDbLib, TwitterDbLib, GithubDbLib
+from .db_models import EmailNotificationDb, OAuthTokenDb
 
 
 class OtherDbLib(DbSessionWrapper):
     def donatees_subquery(self):
+        from .db_libs import YoutubeDbLib, TwitterDbLib, GithubDbLib
         return union(*[
             select(
                 db_lib.db_model.id,
@@ -53,3 +54,23 @@ async def save_email(self, email: str) -> UUID | None:
             .returning(EmailNotificationDb.id)
         )
         return resp.scalar()
+
+
+class OAuthDbLib(DbSessionWrapper):
+    async def query_oauth_token(self, name: str) -> dict[str, Any]:
+        result = await self.execute(
+            select(OAuthTokenDb.token)
+            .where(OAuthTokenDb.name == name)
+        )
+        return result.scalars().one()
+
+    async def save_oauth_token(self, name: str, token: dict[str, Any]):
+        await self.execute(
+            insert(OAuthTokenDb)
+            .values(name=name, token=token)
+            .on_conflict_do_update(
+                index_elements=[OAuthTokenDb.name],
+                set_=dict(token=token),
+                where=OAuthTokenDb.name == name,
+            )
+        )

donate4fun/db_twitter.py

@@ -1,12 +1,12 @@
-from typing import Any
 from uuid import UUID
 
-from sqlalchemy import select
+from sqlalchemy import select, update
 from sqlalchemy.dialects.postgresql import insert
 
-from .models import TwitterAccount, TwitterTweet, TwitterAccountOwned
-from .db_models import TwitterAuthorDb, TwitterTweetDb, OAuthTokenDb, TwitterAuthorLink
+from .models import TwitterAccount, TwitterTweet, TwitterAccountOwned, Donation
+from .db_models import TwitterAuthorDb, TwitterTweetDb, TwitterAuthorLink, DonationDb
 from .db_social import SocialDbWrapper
+from .twitter_models import Tweet
 
 
 class TwitterDbLib(SocialDbWrapper):
@@ -35,20 +35,9 @@ async def get_or_create_tweet(self, tweet: TwitterTweet) -> TwitterAuthorDb:
             id_ = resp.scalar()
         tweet.id = id_
 
-    async def query_oauth_token(self, name: str) -> dict[str, Any]:
-        result = await self.execute(
-            select(OAuthTokenDb.token)
-            .where(OAuthTokenDb.name == name)
-        )
-        return result.scalars().one()
-
-    async def save_oauth_token(self, name: str, token: dict[str, Any]):
+    async def add_invoice_tweet_to_donation(self, donation: Donation, tweet: Tweet):
         await self.execute(
-            insert(OAuthTokenDb)
-            .values(name=name, token=token)
-            .on_conflict_do_update(
-                index_elements=[OAuthTokenDb.name],
-                set_=dict(token=token),
-                where=OAuthTokenDb.name == name,
-            )
+            update(DonationDb)
+            .values(twitter_invoice_tweet_id=tweet.id)
+            .where(DonationDb.id == donation.id)
         )

donate4fun/donation.py

@@ -4,16 +4,23 @@
 import httpx
 from lnpayencode import LnAddr
 
-from .models import Donation, PaymentRequest, Invoice, PayInvoiceResult
+from .models import Donation, PaymentRequest
 from .db import DbSession
 from .db_donations import DonationsDbLib
 from .types import LnurlpError, RequestHash, Satoshi
 from .api_utils import load_donator, auto_transfer_donations, track_donation, HttpClient, sha256hash
-
-from .lnd import lnd
+from .lnd import lnd, Invoice, PayInvoiceResult
 
 
 async def donate(donation: Donation, db_session: DbSession, expiry: int = None) -> (PaymentRequest, Donation):
+    """
+    Takes pre-filled Donation object and do one of the following:
+         - if receiver is a Donate4.Fun account and sender has enough balance then just transfers money without lightning
+         - if receievr has a lightning address and sender has enough money then pays from balance to a lightning address
+         - if receiver has no lightning address (just a social account) then transfers money from sender balance
+           or create a payment request to send money from a lightning wallet
+    In the end money from social accounts are automatically transferred to linked Donate4.Fun account if they exist.
+    """
     donator = await load_donator(db_session, donation.donator.id)
     # If donator has enough money (and not fulfilling his own balance) - try to pay donation instantly
     use_balance = (

donate4fun/jobs.py

@@ -7,7 +7,7 @@
 from .db_models import TwitterAuthorDb, YoutubeChannelDb
 from .db_libs import TwitterDbLib, YoutubeDbLib
 from .settings import settings
-from .twitter import TwitterApiClient, make_apponly_client
+from .twitter import TwitterApiClient
 from .youtube import fetch_youtube_channel
 from .api_utils import register_app_command
 
@@ -22,11 +22,10 @@ async def refetch_twitter_authors():
             | TwitterAuthorDb.last_fetched_at.is_(None)
         )
     logger.info("refetching %d authors", len(accounts))
-    async with make_apponly_client(token=settings.twitter.linking_oauth.bearer_token) as client:
-        api = TwitterApiClient(client)
+    async with TwitterApiClient.create_apponly(token=settings.twitter.linking_oauth.bearer_token) as client:
         for account in accounts:
             try:
-                account: TwitterAccount = await api.get_user_by(user_id=account.user_id)
+                account: TwitterAccount = await client.get_user_by(user_id=account.user_id)
             except Exception:
                 logger.exception("Failed to fetch twitter account %s", account)
             else: