diff --git a/.gitignore b/.gitignore index 9cfa41cb..d21441e8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tags /backups/ /alembic/versions* /.coverage +/media/ diff --git a/charts/donate4fun.stage.yaml b/charts/donate4fun.stage.yaml index 1d432331..01ec19eb 100644 --- a/charts/donate4fun.stage.yaml +++ b/charts/donate4fun.stage.yaml @@ -6,7 +6,10 @@ backend: config: base_url: https://stage.donate4.fun twitter: - enable_bot: false + conversations_bot: + enabled: false + mentions_bot: + enabled: true posthog: debug: true sentry: diff --git a/charts/donate4fun.yaml b/charts/donate4fun.yaml index 10c40794..205b2137 100644 --- a/charts/donate4fun.yaml +++ b/charts/donate4fun.yaml @@ -14,6 +14,7 @@ backend: repository: europe-central2-docker.pkg.dev/donate4fun-prod/docker/donate4fun-backend config: posthog: + enabled: true project_api_key: 'phc_2CphDSfOn61NrqYZnloxHWpwFFjd4mHxUtZwcwrogC0' host: 'https://eu.posthog.com' sentry: @@ -34,8 +35,11 @@ backend: youtube: refresh_timeout: P1D twitter: - greeting: "Hey! I'm a Donate4Fun Twitter bot. I'll check if you have any tips or donations." - enable_bot: false + conversations_bot: + greeting: "Hey! I'm a Donate4Fun Twitter bot. I'll check if you have any tips or donations." + enabled: false + mentions_bot: + enabled: false dm_check_interval: PT20S self_id: "1572908920485576704" refresh_timeout: P1D @@ -80,6 +84,8 @@ backend: loggers: donate4fun: level: DEBUG + donate4fun.twitter_bot: + level: TRACE sqlalchemy.engine.Engine: level: WARNING hypercorn: diff --git a/charts/secrets.donate4fun.yaml b/charts/secrets.donate4fun.yaml index 1de24cfd..58f25171 100644 --- a/charts/secrets.donate4fun.yaml +++ b/charts/secrets.donate4fun.yaml @@ -12,12 +12,20 @@ backend: client_id: ENC[AES256_GCM,data:HxiY4yLoRtuHQshVNpp/HgRbvr0=,iv:p6gjZK1XQwna16UeBbEjm18GgRnTgY1WaGEW2Hbz/V4=,tag:ulFrkKFQJlgZSFCgP++XUw==,type:str] client_secret: ENC[AES256_GCM,data:ydFfKuPN3/AnGBGd6wZptNlniNF2lE7SQ3zacVzSrMLlnc0eUDPg/Q==,iv:bH+H4flBHEZU61uIxe0Q58JVcpjedomFpflt5aY+mIw=,tag:rJqRlpiHopZoiWXg8XY2ow==,type:str] twitter: - bearer_token: ENC[AES256_GCM,data:mGXAzh0LsL4Yk3pgt8icKZ9D9rHffZPDxlB2Cz4qy61SdF1ZHcL9NYlWYexdn9WOo2GGyAr0UtN2tWimbhIJp/GK/WnatStbGEd7349RZskRyBn9v9ug0FQftSHQ3Y+u9fjs6k1FJ7Rb5CQHYJPeRw==,iv:me5EQnETByFIkKcohYjUty8h523IdviKWSku6ICj5Yw=,tag:V8o6eapFWFw+Bfh2w6rsxw==,type:str] - oauth: - client_id: ENC[AES256_GCM,data:5PonrJSWD6kxYizZM8pWRpApTasbMLKd4Ybw2LPS4K0+Tg==,iv:/J9ThtCpohM5hNSVzAM1SSS0n3ytSBh39Bh8GmFnBmc=,tag:KbI90JuYoIXrO2D6GekqLg==,type:str] - client_secret: ENC[AES256_GCM,data:X7iDPNp2qmFNAZKDStGp2Pzl4n5L+k3otyB0vRZ8JnkAMJDA3rxoM84WbwznqV8mqMY=,iv:sTXbzQrPzkyRgeGxOmQ3ClEDsW1MrJzM4sSREfyAmm4=,tag:zzyEgtt3soFfQGis22yyEA==,type:str] - consumer_key: ENC[AES256_GCM,data:nJBxgXHOYlSAN2g7Mm2lJ3dnW+RuATmdaw==,iv:Q45as5sEqtJUyA7ZaaCztfAPbUIe/wbIH9rhd6Wu4uo=,tag:9ueOeIzW6qZtSzAw241FHw==,type:str] - consumer_secret: ENC[AES256_GCM,data:lsnZdH3dv0kpVcaRsgu6qDHt7qy57vfdzYcem5fbhp91V4hW+hdAY9PHyo97uHPgW00=,iv:OyqhdBcrohpspA2JiKDzEtjohJn3CDK3M4zU6U0wtd0=,tag:hs3IZFPuWYw3Xhq3N+Ln6A==,type:str] + linking_oauth: + bearer_token: ENC[AES256_GCM,data:x3OCP3ign7505bnfnPkfSfrRYu2/q1trlRZjP/Vik1ZOXU/utVova8Mp/BphkTtKVBehMj6+4JUuxKLmOioIvkpTYD1CJpo4NW6w/KmBbtWDekHHkH0Sj9/mCgqLyC16gQy5VRzx9mgsgFlUFrI4Sw==,iv:F/BLrVjBPMOJz8lBzSRG674L9QzXo+QG0XxrVFqonmc=,tag:gI+Bx3Fq7PnnSfsbgSXCVQ==,type:str] + client_id: ENC[AES256_GCM,data:8CeK5aa+LlksLBaBQcElAj5hVrVqQELnHh7E3/hjQYzfrw==,iv:QYFHs28pjScz284/J52djIPGAOh8tfreYDxVBhTp4GQ=,tag:l5cJJoSJ+VRUe3LEonkbng==,type:str] + client_secret: ENC[AES256_GCM,data:UUxRIXZd3DzN9hvSzA41w58E4NYBGdWHkH2JAQmce/IebMFwbXhp56mtrpgETMPtSlQ=,iv:tJz57ur+3nOe52+TNPSB57NjBm2b1b8p6cNXU7GvRMw=,tag:uixfO4opqqcuYti1Uh/QJQ==,type:str] + consumer_key: ENC[AES256_GCM,data:a9ve0hGRaMJdp+pnty6Zr+Uo3jcn/M9dVQ==,iv:22FIeTFRqci9kw7Ef0GCJ+0bZ/s3XCex/679xhBOhX8=,tag:rA0zYZcgFzN2/6lx66/GDg==,type:str] + consumer_secret: ENC[AES256_GCM,data:2s8IpyDR9tafI7YhzB9SBOpGZ+BLuyq4AkvJuAFl/VnKrbbTMJWk8C/PR+5ZXiUbYbw=,iv:imWfmMgQnTmuPAcf8vEdRiBT/7Ass9QE7kJeoNOIC2g=,tag:hV6zGlz+64sMIS5iPeCf0A==,type:str] + mentions_bot: + oauth: + #ENC[AES256_GCM,data:xjELLPG1sqcAyu8=,iv:3f1CRU/cRacKBCBGsBFQVdjZLJiz6H/TA3b3XIDYeRs=,tag:HpThBJcvM62IJL1Hh0JR7w==,type:comment] + bearer_token: ENC[AES256_GCM,data:fVrI/PmbTjwRNsn6WxrMJl/jTbOrGJLS6fb0JAAho1tBM8rFxDMIwknnh6sTk7Raxemdam/pZgL6Ic9Zj43LeZaknv7Yyzo48UF8N2sSAx9e/F2iqGE3VMT6VZihnZaRlGr/noPBLFYYkIcG8T3hlHWp0RE=,iv:IGtRrLBTPsnlzi7RFe7arfhi4IEiSRLvvo2BSTRWSR8=,tag:b71FjwSqcQ6g6EwNs9BAsQ==,type:str] + consumer_key: ENC[AES256_GCM,data:faK5mmm9R+ke1GOFNCUEtx/TD0y1pMsdpg==,iv:bEBu/8jwnv0HJs2ug01JXaM/k6/T5IMnH7lVXygv1V4=,tag:OKXbHQFNWalzvvnf1MQ21g==,type:str] + consumer_secret: ENC[AES256_GCM,data:wPSkJFGT0d0ygEQLbZxXIA1qt2gLk6cDEoCRJjLNp7n9xboTvxixodSFHfDvjpcGUf4=,iv:wekZsQbMl6wzhdPZv6bsB4n74RbznWxb7bqHHYHPk4Q=,tag:cOkcbDoppxBfyRNvM8VbsA==,type:str] + client_id: ENC[AES256_GCM,data:dW4gap0DwMKr0VPHkZb51PsQBU+k1dGplAz55c1KjzWgUQ==,iv:8ErZKVxrgYE3a9lwEUz3a4U9796cusyD9w5PfpkeP/g=,tag:zgJSjsad+p5v24+5CF2S3g==,type:str] + client_secret: ENC[AES256_GCM,data:UEiaQuy0It5hxPEcoOfIG1+bJNAgRCLGZv0UNL1Y2B9hS6WQj2XruHbTw91TLw06guw=,iv:ZbrE4DS0mVveRwez5amOzhiib9HyRqOZR2kPFvmAAvU=,tag:EcP4keJZl7OYsFcW4KFoSA==,type:str] db: url: ENC[AES256_GCM,data:HYoegvzZAkbqbH45y70VHZ/uXU9l+XtSCxf4/8i9O26QkOu+D9uhTUtP5sPBA9OOoa4rXv+bKU8Rk4oijYibNX1zPAQWKk3mH0DR,iv:idrSNHClGy0bKvHvuoT2wziOOeTdFhzm+Gp+KhHLhyI=,tag:amlcx7UDiAAFs0Q98lASRA==,type:str] jwt_secret: ENC[AES256_GCM,data:M5RNlxRvdu1Otd6r9zs96U40C0tKjTyRH/+atkl03WY=,iv:d3bffJmniDw2HaiNVZVdF7GkZ4LkU2KucFUtOLWArN0=,tag:wHyPVhegiajbbwMJUEULMw==,type:str] @@ -44,8 +52,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2023-02-02T17:53:25Z" - mac: ENC[AES256_GCM,data:N+3tyXTOvxpTTH0NvqPzLvcXJWeBVoO7bKC/JNRxw0ULxnm6KeTt4iooyBJhv+uKiJ8JhWUEGVc+jx60+8iegZydz6hV9uEZGIRIWoLD+sZVQcw57wEGHTnL+bTaWu8ONy2cBVWFmAN5tDOPyNHlIbLQ1sB9V0tej+GOeeXHfqI=,iv:4sp4ss7zUfs+OHBnAjkS4PeEXTNb2Vg2w4OuUN+WnYQ=,tag:MILixSy9VrdXmAGzXzS9ew==,type:str] + lastmodified: "2023-02-11T13:42:02Z" + mac: ENC[AES256_GCM,data:+XNlBaVZp8qqd9TLUH4uo4bLKb4Y3DSNpQ3kfrVW+VbhwoW8Uk76rbgdAPLWx2wIMz+IXliQ6pUDSLE5AY+wJxqiIASWIUJEsyNJkNcK8tAHrPasdFwri+bJjlcuXJWj4PONaef5KBbInbdcZUyoOeOoqkx2deTcLqJWBuNM4J4=,iv:OKPGf1HyiDJzvxGTsxqUmVX1SKlhWC50yojp3ntMWPY=,tag:liTYcZdapSuJnKnG2jfL6Q==,type:str] pgp: - created_at: "2022-06-19T10:27:57Z" enc: | diff --git a/config-test.yaml b/config-test.yaml index 0d4b5184..96a564a7 100644 --- a/config-test.yaml +++ b/config-test.yaml @@ -6,13 +6,18 @@ youtube: client_id: client_id client_secret: client_secret twitter: - bearer_token: bearer_token - greeting: "" - enable_bot: false self_id: "123" dm_check_interval: PT30S refresh_timeout: P1D - oauth: + conversations_bot: + enabled: false + oauth: null + greeting: "" + mentions_bot: + enabled: false + oauth: null + linking_oauth: + bearer_token: bearer_token client_id: client_id client_secret: client_secret consumer_key: consumer_key diff --git a/donate4fun/api.py b/donate4fun/api.py index 5ea217c1..91c81a35 100644 --- a/donate4fun/api.py +++ b/donate4fun/api.py @@ -18,15 +18,16 @@ from jwt import InvalidTokenError from .models import ( - Donation, Donator, Invoice, + Donation, Donator, Invoice, SocialAccount, WithdrawalToken, BaseModel, Notification, Credentials, SubscribeEmailRequest, - DonatorStats, PayInvoiceResult, Donatee, OAuthState, SocialProvider, Toast, + DonatorStats, PayInvoiceResult, Donatee, OAuthState, Toast, SocialProviderId, ) from .types import ValidationError, PaymentRequest, OAuthError, LnurlpError, AccountAlreadyLinked from .core import to_base64 from .db_models import WithdrawalDb from .db_libs import WithdrawalDbLib, DonationsDbLib, OtherDbLib from .settings import settings +from .social import SocialProvider from .api_utils import ( get_donator, load_donator, get_db_session, task_group, only_me, make_redirect, get_donations_db, sha256hash, oauth_success_messages, signin_success_message, @@ -222,14 +223,16 @@ class PaymentCallbackResponse(BaseModel): pr: PaymentRequest -@router.get('/lnurl/{receiver_id}/payment-callback', response_model=PaymentCallbackResponse) +@router.get('/lnurl/{provider_id}/{receiver_id}/payment-callback', response_model=PaymentCallbackResponse) async def payment_callback( - request: Request, receiver_id: UUID, amount: int = Query(...), comment: str = Query(...), db_session=Depends(get_db_session), + request: Request, provider_id: SocialProviderId, receiver_id: UUID, amount: int = Query(...), comment: str = Query(...), + db_session=Depends(get_db_session), ): """ - This callback is needed for lightning address support. Currently it's used for internal testing only. + This callback is needed for lightning address support. """ - receiver: Donator = await db_session.query_donator(id=receiver_id) + provider = SocialProvider.create(provider_id) + receiver: SocialAccount | Donator = await provider.wrap_db(db_session).query_account(id=receiver_id) amount = amount // 1000 # FIXME: handle msats correctly invoice: Invoice = await lnd.create_invoice( memo=comment, value=amount, description_hash=to_base64(sha256hash(lightning_payment_metadata(receiver))), @@ -397,7 +400,7 @@ async def withdraw(request: Request, db=Depends(get_db_session), me: Donator = D @router.get('/oauth-redirect/{provider}') async def oauth_redirect( - request: Request, provider: SocialProvider, state: str, error: str = None, error_description: str = None, code: str = None, + request: Request, provider: SocialProviderId, state: str, error: str = None, error_description: str = None, code: str = None, donator=Depends(get_donator), ): try: diff --git a/donate4fun/api_donation.py b/donate4fun/api_donation.py index e3823793..374436bb 100644 --- a/donate4fun/api_donation.py +++ b/donate4fun/api_donation.py @@ -1,50 +1,34 @@ -import json import logging import hashlib from uuid import UUID from datetime import datetime -import httpx from fastapi import Request, Depends, HTTPException, APIRouter +from furl import furl from sqlalchemy import select -from lnpayencode import LnAddr from .models import ( - Donation, Donator, Invoice, DonateResponse, DonateRequest, - DonationPaidRequest, PayInvoiceResult, PaymentRequest, RequestHash + Donation, Donator, Invoice, DonateResponse, DonateRequest, DonationPaidRequest, RequestHash ) -from .types import ValidationError, LnurlpError +from .types import ValidationError +from .social import SocialProviderId, SocialProvider from .api_utils import ( - get_donator, get_db_session, load_donator, auto_transfer_donations, track_donation, HttpClient, get_donations_db, only_me, - sha256hash, get_social_provider_db, + get_donator, get_db_session, load_donator, track_donation, get_donations_db, only_me, ) from .db_libs import GithubDbLib, TwitterDbLib, YoutubeDbLib, DonationsDbLib from .db_donations import sent_donations_subquery, received_donations_subquery from .db_models import DonationDb from .db_social import SocialDbWrapper -from .twitter import query_or_fetch_twitter_account -from .donatees import apply_target from .lnd import lnd from .settings import settings +from .donation import donate, make_memo logger = logging.getLogger(__name__) router = APIRouter() -def make_memo(donation: Donation) -> str: - if account := donation.receiver_social_account: - return f"Donate4.Fun donation to {account.unique_name}" - elif donation.receiver: - if donation.receiver.id == donation.donator.id: - return f"[Donate4.fun] fulfillment for {donation.receiver.name}" - else: - return f"[Donate4.fun] donation to {donation.receiver.name}" - else: - raise ValueError(f"Could not make a memo for donation {donation}") - - @router.post("/donate", response_model=DonateResponse) -async def donate( +async def donate_api( web_request: Request, request: DonateRequest, donator: Donator = Depends(get_donator), db_session=Depends(get_db_session), ) -> DonateResponse: logger.debug( @@ -59,17 +43,20 @@ async def donate( donation.lightning_address = request.lightning_address if request.receiver_id: # Donation to a donator - possibly just an own balance fulfillment - receiver = await load_donator(db_session, request.receiver_id) + receiver: Donator = await load_donator(db_session, request.receiver_id) if not receiver.connected: raise ValidationError("Money receiver should have a connected wallet") donation.receiver = receiver elif request.social_account_id and request.social_provider: - db_lib: SocialDbWrapper = get_social_provider_db(request.social_provider) - social_account = await db_lib(db_session).query_account(id=request.social_account_id) - setattr(donation, db_lib.donation_field, social_account) + provider: SocialProvider = SocialProvider.create(request.social_provider) + social_db: SocialDbWrapper = provider.wrap_db(db_session) + social_account = await social_db.query_account(id=request.social_account_id) + setattr(donation, social_db.donation_field, social_account) donation.lightning_address = donation.lightning_address or social_account.lightning_address elif request.target: - await apply_target(donation, request.target, db_session) + target = furl(request.target) + social_provider = SocialProvider.from_url(target) + await social_provider.apply_target(donation, target, db_session) # FIXME: the following elif blocks are deprecated elif request.channel_id: donation.youtube_channel = await YoutubeDbLib(db_session).query_account(id=request.channel_id) @@ -83,116 +70,15 @@ async def donate( else: raise ValidationError("donation should have a target, channel_id or receiver_id") if request.donator_twitter_handle: - donation.donator_twitter_account = await query_or_fetch_twitter_account( - db=TwitterDbLib(db_session), handle=request.donator_twitter_handle, - ) - donator = await load_donator(db_session, donator.id) - # If donator has enough money (and not fulfilling his own balance) - try to pay donation instantly - use_balance = ( - request.receiver_id != donator.id - and (donator.available_balance if donation.lightning_address else donator.balance) >= request.amount - ) - if donation.lightning_address: - pay_req: PaymentRequest = await fetch_lightning_address(donation) - r_hash = RequestHash(pay_req.decode().paymenthash) - if use_balance: - # FIXME: this leads to 'duplicate key value violates unique constraint "donation_rhash_key"' - # if we are donating to a local lightning address - donation.r_hash = r_hash - else: - donation.transient_r_hash = r_hash - elif not use_balance: - invoice: Invoice = await lnd.create_invoice(memo=make_memo(donation), value=request.amount) - donation.r_hash = invoice.r_hash # This hash is needed to find and complete donation after payment succeeds - pay_req = invoice.payment_request - donations_db = DonationsDbLib(db_session) - await donations_db.create_donation(donation) - if use_balance: - if donation.lightning_address: - pay_result: PayInvoiceResult = await lnd.pay_invoice(pay_req) - amount = pay_result.value_sat - paid_at = pay_result.creation_date - fee_msat = pay_result.fee_msat - claimed_at = pay_result.creation_date - else: - amount = request.amount - paid_at = datetime.utcnow() - fee_msat = None - claimed_at = None - await donations_db.donation_paid( - donation_id=donation.id, amount=amount, paid_at=paid_at, fee_msat=fee_msat, claimed_at=claimed_at, + twitter_provider = SocialProvider.create(SocialProviderId.twitter.value) + donation.donator_twitter_account = await twitter_provider.query_or_fetch_account( + twitter_provider.wrap_db(db_session), handle=request.donator_twitter_handle ) - await auto_transfer_donations(db_session, donation) - # Reload donation with a fresh state - donation = await donations_db.query_donation(id=donation.id) + pay_req, donation = await donate(donation, db_session) + if pay_req is None: # FIXME: balance is saved in cookie to notify extension about balance change, but it should be done via VAPID web_request.session['balance'] = (await db_session.query_donator(id=donator.id)).balance - track_donation(donation) - return DonateResponse(donation=donation, payment_request=None) - else: - return DonateResponse(donation=donation, payment_request=pay_req) - - -async def fetch_lightning_address(donation: Donation) -> PaymentRequest: - name, host = donation.lightning_address.split('@', 1) - async with HttpClient() as client: - try: - response = await client.get(f'https://{host}/.well-known/lnurlp/{name}', follow_redirects=True) - except httpx.HTTPStatusError as exc: - raise LnurlpError(f"{exc.request.url} responded with {exc.response.status_code}: {exc.response.content}") from exc - except httpx.HTTPError as exc: - raise LnurlpError(f"HTTP error with {exc.request.url}: {exc}") from exc - metadata = response.json() - # https://github.com/lnurl/luds/blob/luds/06.md - if metadata.get('status', 'OK') != 'OK': - raise LnurlpError(f"Status is not OK: {metadata}") - if not metadata['minSendable'] <= donation.amount * 1000 <= metadata['maxSendable']: - raise LnurlpError(f"Amount is out of bounds: {donation.amount} {metadata}") - fields = dict(json.loads(metadata['metadata'])) - if donation.donator_twitter_account: - name = '@' + donation.donator_twitter_account.handle - else: - name = donation.donator.name - params = dict(amount=donation.amount * 1000) - if payerdata_request := metadata.get('payerData'): - payerdata = {} - if 'name' in payerdata_request: - payerdata['name'] = f'{name} via Donate4.Fun' - # Separators are important for hashes to match - params['payerdata'] = json.dumps(payerdata, separators=(',', ':')) - if donation.youtube_video: - target = f'https://youtube.com/watch?v={donation.youtube_video.video_id}' - elif donation.twitter_tweet: - target = f'https://twitter.com/{donation.twitter_account.handle}/status/{donation.twitter_tweet.tweet_id}' - elif donation.youtube_channel: - target = f'https://youtube.com/channel/{donation.youtube_channel.channel_id}' - elif donation.twitter_account: - target = f'https://twitter.com/{donation.twitter_account.handle}' - elif donation.lightning_address: - target = fields.get('text/identifier', donation.lightning_address) - comment = f'Tip from {name} via Donate4.Fun for {target}' - if 'commentAllowed' in metadata: - params['comment'] = comment[:metadata['commentAllowed']] - try: - response = await client.get(metadata['callback'], params=params) - except httpx.HTTPStatusError as exc: - raise LnurlpError(response.content) from exc - except httpx.HTTPError as exc: - raise LnurlpError(exc) from exc - data = response.json() - if data.get('status', 'OK') != 'OK': - raise LnurlpError(f"Status is not OK: {data}") - pay_req = PaymentRequest(data['pr']) - invoice: LnAddr = pay_req.decode() - expected_hash = dict(invoice.tags)['h'] - # https://github.com/lnurl/luds/blob/luds/18.md#3-committing-payer-to-the-invoice - full_metadata: str = metadata['metadata'] + params.get('payerdata', '') - if sha256hash(full_metadata) != expected_hash: - raise LnurlpError(f"Metadata hash does not match invoice hash: sha256({full_metadata}) != {expected_hash}") - invoice_amount = invoice.amount * 10 ** 8 - if invoice_amount != donation.amount: - raise LnurlpError(f"Amount in invoice does not match requested amount: {invoice_amount} != {donation.amount}") - return pay_req + return DonateResponse(donation=donation, payment_request=pay_req) @router.get("/donation/{donation_id}", response_model=DonateResponse) diff --git a/donate4fun/api_social.py b/donate4fun/api_social.py index 405dffaf..7a6fcebd 100644 --- a/donate4fun/api_social.py +++ b/donate4fun/api_social.py @@ -3,7 +3,7 @@ import posthog from fastapi import APIRouter, Depends, HTTPException -from .models import TransferResponse, Donator, SocialAccountOwned, Donation, SocialProvider +from .models import TransferResponse, Donator, SocialAccountOwned, Donation, SocialProviderId from .types import ValidationError from .api_utils import get_db_session, load_donator, get_donator, get_donations_db, get_social_provider_db from .db_models import DonationDb @@ -13,7 +13,7 @@ @router.post('/{social_provider}/{account_id}/transfer', response_model=TransferResponse) async def transfer_social_account_donations( - social_provider: SocialProvider, account_id: UUID, db=Depends(get_db_session), donator: Donator = Depends(get_donator), + social_provider: SocialProviderId, account_id: UUID, db=Depends(get_db_session), donator: Donator = Depends(get_donator), ): donator = await load_donator(db, donator.id) social_db = get_social_provider_db(social_provider)(db) diff --git a/donate4fun/api_twitter.py b/donate4fun/api_twitter.py index 271f2b57..ca16fced 100644 --- a/donate4fun/api_twitter.py +++ b/donate4fun/api_twitter.py @@ -8,9 +8,12 @@ from .models import TwitterAccount, Donator, TwitterAccountOwned, OAuthState, OAuthResponse, Toast from .types import ValidationError, OAuthError, AccountAlreadyLinked, Satoshi from .api_utils import get_donator, make_absolute_uri, make_redirect, signin_success_message, oauth_success_messages -from .twitter import make_prove_message, make_link_oauth2_client, make_oauth1_client, fetch_twitter_me +from .twitter import make_oauth1_client, make_oauth2_client, TwitterApiClient +from .twitter_bot import make_prove_message +from .core import app from .db_twitter import TwitterDbLib from .db import db +from .settings import settings router = APIRouter(prefix='/twitter') logger = logging.getLogger(__name__) @@ -23,7 +26,11 @@ async def twitter_ownership_message(me=Depends(get_donator)): @router.get('/oauth1', response_model=OAuthResponse) async def login_via_twitter_oauth1(request: Request, donator=Depends(get_donator)): - async with make_oauth1_client(redirect_uri=make_absolute_uri(request.url_for('oauth1_callback'))) as client: + oauth1_ctx = make_oauth1_client( + oauth=settings.twitter.linking_oauth, + redirect_uri=make_absolute_uri(request.url_for('oauth1_callback')), + ) + async with oauth1_ctx as client: await client.fetch_request_token('https://api.twitter.com/oauth/request_token') auth_url: str = client.create_authorization_url('https://api.twitter.com/oauth/authorize') return OAuthResponse(url=auth_url) @@ -38,7 +45,8 @@ async def oauth1_callback( if oauth_token is None or oauth_verifier is None: raise ValidationError("oauth_token and oauth_verifier parameters must be present") try: - async with make_oauth1_client(token=oauth_token) as client: + token = dict(oauth_token=oauth_token, oauth_verifier=oauth_verifier) + async with make_oauth1_client(oauth=settings.twitter.linking_oauth, token=token) as client: token: dict = await client.fetch_access_token('https://api.twitter.com/oauth/access_token', verifier=oauth_verifier) client.token = token try: @@ -92,7 +100,7 @@ async def finish_twitter_oauth(code: str, donator: Donator, code_verifier: str) async def login_or_link_twitter_account(client, donator: Donator) -> tuple[Satoshi, TwitterAccountOwned]: try: - account: TwitterAccount = await fetch_twitter_me(client) + account: TwitterAccount = await TwitterApiClient(client).get_me() except Exception as exc: raise OAuthError("Failed to fetch user's account") from exc @@ -108,3 +116,15 @@ async def login_or_link_twitter_account(client, donator: Donator) -> tuple[Satos raise OAuthError("Failed to link account") from exc else: raise AccountAlreadyLinked(owned_account) + + +def make_link_oauth2_client(token=None): + """ + This client is used to link Twitter account to donator (OAuth2 flow) + """ + return make_oauth2_client( + settings.twitter.linking_oauth, + scope='tweet.read users.read', + token=token, + redirect_uri=make_absolute_uri(app.url_path_for('oauth_redirect', provider='twitter')), + ) diff --git a/donate4fun/api_utils.py b/donate4fun/api_utils.py index b00b6980..d94f13ad 100644 --- a/donate4fun/api_utils.py +++ b/donate4fun/api_utils.py @@ -12,7 +12,7 @@ from jwcrypto.jwt import JWT from jwcrypto.jwk import JWK -from .models import Donator, Credentials, Donation, SocialProvider, SocialAccountOwned, SocialAccount, Toast +from .models import Donator, Credentials, Donation, SocialProviderId, SocialAccountOwned, SocialAccount, Toast from .db import DbSession, db from .db_libs import TwitterDbLib, YoutubeDbLib, GithubDbLib, DonationsDbLib from .core import ContextualObject @@ -133,7 +133,7 @@ def oauth_success_messages(linked_account: SocialAccount, transferred_amount: Sa f"{linked_account.provider.capitalize()} account {linked_account.unique_name} was successefully linked", ) if transferred_amount > 0: - yield Toast('success', "Funds claimed", f"{transferred_amount} sats are successefully claimed") + yield Toast('success', "Funds claimed", f"{transferred_amount} sats were successefully claimed") def signin_success_message(account: SocialAccount) -> Toast: @@ -156,9 +156,9 @@ def sha256hash(data: str) -> bytes: return hashlib.sha256(data.encode()).digest() -def get_social_provider_db(social_provider: SocialProvider): +def get_social_provider_db(social_provider: SocialProviderId): return { - SocialProvider.youtube: YoutubeDbLib, - SocialProvider.twitter: TwitterDbLib, - SocialProvider.github: GithubDbLib, + SocialProviderId.youtube: YoutubeDbLib, + SocialProviderId.twitter: TwitterDbLib, + SocialProviderId.github: GithubDbLib, }[social_provider] diff --git a/donate4fun/api_youtube.py b/donate4fun/api_youtube.py index b96c36c7..7b45c90f 100644 --- a/donate4fun/api_youtube.py +++ b/donate4fun/api_youtube.py @@ -10,10 +10,8 @@ BaseModel, YoutubeVideo, YoutubeChannel, Donator, YoutubeChannelOwned, OAuthState, OAuthResponse, ) -from .youtube import ( - find_comment, query_or_fetch_youtube_channel, ChannelInfo, fetch_user_channel, -) -from .db_youtube import YoutubeDbLib +from .youtube import find_comment, ChannelInfo, fetch_user_channel +from .youtube_provider import YoutubeProvider from .settings import settings from .types import OAuthError, AccountAlreadyLinked, Satoshi from .db import db @@ -34,7 +32,7 @@ class YoutubeVideoResponse(BaseModel): @router.get('/video/{video_id}', response_model=YoutubeVideoResponse) async def youtube_video_info(video_id: str, db=Depends(get_db_session)): try: - video: YoutubeVideo = await YoutubeDbLib(db).query_youtube_video(video_id=video_id) + video: YoutubeVideo = await YoutubeProvider().wrap_db(db).query_youtube_video(video_id=video_id) return YoutubeVideoResponse(id=video.id, total_donated=video.total_donated) except NoResultFound: return YoutubeVideoResponse(id=None, total_donated=0) @@ -57,7 +55,7 @@ async def ownership_check(donator=Depends(get_donator), db=Depends(get_db_sessio ) channels = [] for channel_id in channel_ids: - youtube_channel: YoutubeChannel = await query_or_fetch_youtube_channel(channel_id=channel_id, db=db) + youtube_channel: YoutubeChannel = await YoutubeProvider().query_or_fetch_account(channel_id=channel_id, db=db) is_new = await db.link_youtube_channel(youtube_channel, donator, via_oauth=False) if is_new: channels.append(youtube_channel) @@ -96,8 +94,9 @@ async def finish_youtube_oauth(code: str, donator: Donator) -> tuple[Satoshi, Yo raise OAuthError("Failed to fetch user's channel") from exc try: async with db.session() as db_session: - youtube_db = YoutubeDbLib(db_session) - channel: YoutubeChannel = await query_or_fetch_youtube_channel(channel_id=channel_info.id, db=youtube_db) + provider = YoutubeProvider() + youtube_db = provider.wrap_db(db_session) + channel: YoutubeChannel = await provider.query_or_fetch_account(channel_id=channel_info.id, db=youtube_db) owned_channel: YoutubeChannelOwned = await youtube_db.query_account(id=channel.id) if not owned_channel.via_oauth: await youtube_db.link_account(channel, donator, via_oauth=True) diff --git a/donate4fun/app.py b/donate4fun/app.py index 2a5db764..25a3460c 100644 --- a/donate4fun/app.py +++ b/donate4fun/app.py @@ -23,7 +23,7 @@ from .db import Database, db from .lnd import monitor_invoices, LndClient, lnd from .pubsub import PubSubBroker, pubsub -from .twitter import run_twitter_bot_restarting +from . import twitter_bot from .core import app, register_command, commands from .screenshot import create_screenshoter_app from .api_utils import task_group @@ -33,13 +33,35 @@ @asynccontextmanager -async def create_app(settings: Settings): +async def create_common(): + if settings.rollbar: + rollbar.init(**settings.rollbar.dict()) + if settings.sentry: + sentry_sdk.init( + dsn=settings.sentry.dsn, + traces_sample_rate=settings.sentry.traces_sample_rate, + environment=settings.sentry.environment, + ) + if settings.google_cloud_logging: + client = google.cloud.logging.Client() + client.setup_logging() + if settings.bugsnag: + bugsnag.configure(**settings.bugsnag.dict(), project_root=os.path.dirname(__file__)) + if settings.posthog and settings.posthog.enabled: + posthog.project_api_key = settings.posthog.project_api_key + posthog.host = settings.posthog.host + posthog.debug = settings.posthog.debug + else: + posthog.disabled = True + yield + + +@asynccontextmanager +async def create_app(): app = FastAPI( debug=settings.fastapi.debug, root_path=settings.fastapi.root_path, ) - if settings.rollbar: - rollbar.init(**settings.rollbar.dict()) if settings.fastapi.debug: api.app.add_middleware( DebugToolbarMiddleware, @@ -47,12 +69,8 @@ async def create_app(settings: Settings): profiler_options=dict(interval=.0002, async_mode='enabled'), ) if settings.sentry: - sentry_sdk.init( - dsn=settings.sentry.dsn, - traces_sample_rate=settings.sentry.traces_sample_rate, - environment=settings.sentry.environment, - ) app.add_middleware(SentryAsgiMiddleware) + api.app.add_middleware( CORSMiddleware, allow_origins=[ @@ -114,9 +132,10 @@ async def main(args): module, command = command.split('.') __import__(f'donate4fun.{module}') with load_settings(), db.assign(Database(settings.db)): - result = await commands[command](*args[2:]) - if result is not None: - print(result) + async with create_common(): + result = await commands[command](*args[2:]) + if result is not None: + print(result) @register_command @@ -141,22 +160,13 @@ async def create_table(tablename: str): async def serve(): pubsub_ = PubSubBroker() lnd_ = LndClient(settings.lnd) - async with create_app(settings) as app_, anyio.create_task_group() as tg: - if settings.google_cloud_logging: - client = google.cloud.logging.Client() - client.setup_logging() - if settings.bugsnag: - bugsnag.configure(**settings.bugsnag.dict(), project_root=os.path.dirname(__file__)) - if settings.posthog: - posthog.project_api_key = settings.posthog.project_api_key - posthog.host = settings.posthog.host - posthog.debug = settings.posthog.debug - else: - posthog.disabled = True + async with create_app() as app_, anyio.create_task_group() as tg: with app.assign(app_), lnd.assign(lnd_), pubsub.assign(pubsub_), task_group.assign(tg): async with pubsub.run(db), monitor_invoices(lnd_, db), AsyncExitStack() as stack: - if settings.twitter.enable_bot: - await stack.enter_async_context(run_twitter_bot_restarting(db)) + if settings.twitter.conversations_bot.enabled: + await stack.enter_async_context(twitter_bot.run_converstaions_bot()) + if settings.twitter.mentions_bot.enabled: + await stack.enter_async_context(twitter_bot.run_mentions_bot()) hyper_config = Config.from_mapping(settings.hypercorn) hyper_config.accesslog = logging.getLogger('hypercorn.accesslog') iface = hyper_config.bind[0].split(':')[0] diff --git a/donate4fun/core.py b/donate4fun/core.py index b6037975..683c26fe 100644 --- a/donate4fun/core.py +++ b/donate4fun/core.py @@ -103,8 +103,8 @@ async def starter(coro): commands = {} -def register_command(func): - commands[func.__name__] = func +def register_command(func, name: str = None): + commands[name or func.__name__] = func return func diff --git a/donate4fun/db_social.py b/donate4fun/db_social.py index 74ca2faf..b2fd229a 100644 --- a/donate4fun/db_social.py +++ b/donate4fun/db_social.py @@ -8,7 +8,7 @@ from .db_models import DonatorDb, DonationDb, TransferDb, DonateeDb, Base as BaseDbModel, BaseLink from .db_utils import insert_on_conflict_update from .db import DbSessionWrapper -from .models import BaseModel, Donator, SocialAccount +from .models import BaseModel, Donator, SocialAccount, SocialAccountOwned from .types import InvalidDbState, Satoshi, NotEnoughBalance @@ -84,7 +84,7 @@ def link_db_model_foreign_key(cls): # FIXME: this could be made simpler by unifying link table column names return getattr(cls.link_db_model, list(cls.link_db_model.__table__.foreign_keys)[0].parent.name) - async def query_account(self, *, owner_id: UUID | None = None, **filter_by): + async def query_account(self, *, owner_id: UUID | None = None, **filter_by) -> SocialAccountOwned: """ This is a generic function to get specified by `link_table` social accoutnt for donator (`owner_id`) It assumes that first foreign key is linked to a social account table @@ -108,7 +108,7 @@ async def query_account(self, *, owner_id: UUID | None = None, **filter_by): ) return self.owned_model.from_orm(resp.one()) - async def link_account(self, account: BaseModel, donator: Donator, via_oauth: bool) -> bool: + async def link_account(self, account: SocialAccount, donator: Donator, via_oauth: bool) -> bool: """ Links a social account to the donator account. Returns True if new link is created, False otherwise diff --git a/donate4fun/donate4fun_provider.py b/donate4fun/donate4fun_provider.py new file mode 100644 index 00000000..86708a45 --- /dev/null +++ b/donate4fun/donate4fun_provider.py @@ -0,0 +1,27 @@ +from furl import furl + +from .social import SocialProvider +from .db import DbSession +from .models import Donator, Donation + + +class DonatorDbLib: + def __init__(self, session: DbSession): + self.session = session + + async def query_account(self, *, id) -> Donator: + return await self.session.query_donator(id=id) + + +class Donate4FunProvider(SocialProvider): + def wrap_db(self, db_session: DbSession): + return DonatorDbLib(db_session) + + async def apply_target(self, donation: Donation, target: furl, db_session: DbSession): + raise NotImplementedError + + async def query_or_fetch_account(self, *, db_session: DbSession, **params): + raise NotImplementedError + + async def get_account_path(self, account: Donator): + raise NotImplementedError diff --git a/donate4fun/donatees.py b/donate4fun/donatees.py deleted file mode 100644 index fa09ba58..00000000 --- a/donate4fun/donatees.py +++ /dev/null @@ -1,31 +0,0 @@ -import re -from urllib.parse import urlparse - -from email_validator import validate_email - -from .youtube import validate_youtube_url -from .twitter import validate_twitter_url -from .models import Donation -from .db import DbSession -from .types import UnsupportedTarget, Url - - -async def validate_target(target: str): - if re.match(r'https?://.+', target): - return await validate_target_url(target) - return validate_email(target).email - - -async def validate_target_url(target: Url): - parsed = urlparse(target) - if parsed.hostname in ['youtube.com', 'www.youtube.com', 'youtu.be']: - return validate_youtube_url(parsed) - elif parsed.hostname in ['twitter.com', 'www.twitter.com']: - return validate_twitter_url(parsed) - else: - raise UnsupportedTarget("URL is invalid") - - -async def apply_target(donation: Donation, target: str, db: DbSession): - donatee = await validate_target(target) - await donatee.fetch(donation, db) diff --git a/donate4fun/donation.py b/donate4fun/donation.py new file mode 100644 index 00000000..4f1454a3 --- /dev/null +++ b/donate4fun/donation.py @@ -0,0 +1,137 @@ +import json +from datetime import datetime + +import httpx +from lnpayencode import LnAddr + +from .models import Donation, PaymentRequest, Invoice, PayInvoiceResult +from .db import DbSession +from .db_donations import DonationsDbLib +from .types import LnurlpError, RequestHash +from .api_utils import load_donator, auto_transfer_donations, track_donation, HttpClient, sha256hash + +from .lnd import lnd + + +async def donate(donation: Donation, db_session: DbSession, expiry: int = None) -> (PaymentRequest, Donation): + donator = await load_donator(db_session, donation.donator.id) + # If donator has enough money (and not fulfilling his own balance) - try to pay donation instantly + use_balance = ( + (donation.receiver and donation.receiver.id) != donator.id + and (donator.available_balance if donation.lightning_address else donator.balance) >= donation.amount + ) + if donation.lightning_address: + # Payment to a lnurlp + pay_req: PaymentRequest = await fetch_lightning_address(donation) + r_hash = RequestHash(pay_req.decode().paymenthash) + if use_balance: + # FIXME: this leads to 'duplicate key value violates unique constraint "donation_rhash_key"' + # if we are donating to a local lightning address + donation.r_hash = r_hash + else: + donation.transient_r_hash = r_hash + elif not use_balance: + # Payment by invoice + invoice: Invoice = await lnd.create_invoice(memo=make_memo(donation), value=donation.amount, expiry=expiry) + donation.r_hash = invoice.r_hash # This hash is needed to find and complete donation after payment succeeds + pay_req = invoice.payment_request + else: + # Fully internal payment + pay_req = None + donations_db = DonationsDbLib(db_session) + await donations_db.create_donation(donation) + if use_balance: + if donation.lightning_address: + pay_result: PayInvoiceResult = await lnd.pay_invoice(pay_req) + amount = pay_result.value_sat + paid_at = pay_result.creation_date + fee_msat = pay_result.fee_msat + claimed_at = pay_result.creation_date + pay_req = None # We should not return pay_req to client because it's already paid + else: + amount = donation.amount + paid_at = datetime.utcnow() + fee_msat = None + claimed_at = None + await donations_db.donation_paid( + donation_id=donation.id, amount=amount, paid_at=paid_at, fee_msat=fee_msat, claimed_at=claimed_at, + ) + await auto_transfer_donations(db_session, donation) + # Reload donation with a fresh state + donation = await donations_db.query_donation(id=donation.id) + track_donation(donation) + return pay_req, donation + + +async def fetch_lightning_address(donation: Donation) -> PaymentRequest: + name, host = donation.lightning_address.split('@', 1) + async with HttpClient() as client: + try: + response = await client.get(f'https://{host}/.well-known/lnurlp/{name}', follow_redirects=True) + except httpx.HTTPStatusError as exc: + raise LnurlpError(f"{exc.request.url} responded with {exc.response.status_code}: {exc.response.content}") from exc + except httpx.HTTPError as exc: + raise LnurlpError(f"HTTP error with {exc.request.url}: {exc}") from exc + metadata = response.json() + # https://github.com/lnurl/luds/blob/luds/06.md + if metadata.get('status', 'OK') != 'OK': + raise LnurlpError(f"Status is not OK: {metadata}") + if not metadata['minSendable'] <= donation.amount * 1000 <= metadata['maxSendable']: + raise LnurlpError(f"Amount is out of bounds: {donation.amount} {metadata}") + fields = dict(json.loads(metadata['metadata'])) + if donation.donator_twitter_account: + name = '@' + donation.donator_twitter_account.handle + else: + name = donation.donator.name + params = dict(amount=donation.amount * 1000) + if payerdata_request := metadata.get('payerData'): + payerdata = {} + if 'name' in payerdata_request: + payerdata['name'] = f'{name} via Donate4.Fun' + # Separators are important for hashes to match + params['payerdata'] = json.dumps(payerdata, separators=(',', ':')) + if donation.youtube_video: + target = f'https://youtube.com/watch?v={donation.youtube_video.video_id}' + elif donation.twitter_tweet: + target = f'https://twitter.com/{donation.twitter_account.handle}/status/{donation.twitter_tweet.tweet_id}' + elif donation.youtube_channel: + target = f'https://youtube.com/channel/{donation.youtube_channel.channel_id}' + elif donation.twitter_account: + target = f'https://twitter.com/{donation.twitter_account.handle}' + elif donation.lightning_address: + target = fields.get('text/identifier', donation.lightning_address) + comment = f'Tip from {name} via Donate4.Fun for {target}' + if 'commentAllowed' in metadata: + params['comment'] = comment[:metadata['commentAllowed']] + try: + response = await client.get(metadata['callback'], params=params) + except httpx.HTTPStatusError as exc: + raise LnurlpError(f"Callback responded with {exc}: {response.content}") from exc + except httpx.HTTPError as exc: + raise LnurlpError(f"Callback responded with {exc}") from exc + data = response.json() + if data.get('status', 'OK') != 'OK': + raise LnurlpError(f"Status is not OK: {data}") + pay_req = PaymentRequest(data['pr']) + invoice: LnAddr = pay_req.decode() + expected_hash = dict(invoice.tags)['h'] + # https://github.com/lnurl/luds/blob/luds/18.md#3-committing-payer-to-the-invoice + full_metadata: str = metadata['metadata'] + params.get('payerdata', '') + if sha256hash(full_metadata) != expected_hash: + raise LnurlpError(f"Metadata hash does not match invoice hash: sha256({full_metadata}) != {expected_hash}") + invoice_amount = invoice.amount * 10 ** 8 + if invoice_amount != donation.amount: + raise LnurlpError(f"Amount in invoice does not match requested amount: {invoice_amount} != {donation.amount}") + return pay_req + + +def make_memo(donation: Donation) -> str: + if account := donation.receiver_social_account: + return f"Donate4.Fun donation to {account.unique_name}" + elif donation.receiver: + if donation.receiver.id == donation.donator.id: + return f"[Donate4.fun] fulfillment for {donation.receiver.name}" + else: + return f"[Donate4.fun] donation to {donation.receiver.name}" + else: + raise ValueError(f"Could not make a memo for donation {donation}") diff --git a/donate4fun/jobs.py b/donate4fun/jobs.py index 98c47e21..40112a0c 100644 --- a/donate4fun/jobs.py +++ b/donate4fun/jobs.py @@ -7,7 +7,7 @@ from .db_models import TwitterAuthorDb, YoutubeChannelDb from .db_libs import TwitterDbLib, YoutubeDbLib from .settings import settings -from .twitter import fetch_twitter_author +from .twitter import TwitterApiClient, make_apponly_client from .youtube import fetch_youtube_channel from .core import register_command @@ -22,14 +22,16 @@ async def refetch_twitter_authors(): | TwitterAuthorDb.last_fetched_at.is_(None) ) logger.info("refetching %d authors", len(accounts)) - for account in accounts: - try: - account: TwitterAccount = await fetch_twitter_author(user_id=account.user_id) - except Exception: - logger.exception("Failed to fetch twitter account %s", account) - else: - async with db.session() as db_session: - await TwitterDbLib(db_session).save_account(account) + async with make_apponly_client(token=settings.twitter.linking_oauth.bearer_token) as client: + api = TwitterApiClient(client) + for account in accounts: + try: + account: TwitterAccount = await api.get_user_by(user_id=account.user_id) + except Exception: + logger.exception("Failed to fetch twitter account %s", account) + else: + async with db.session() as db_session: + await TwitterDbLib(db_session).save_account(account) @register_command diff --git a/donate4fun/lnd.py b/donate4fun/lnd.py index 76e1a6fd..99a37e87 100644 --- a/donate4fun/lnd.py +++ b/donate4fun/lnd.py @@ -17,7 +17,7 @@ from .settings import LndSettings, settings from .types import RequestHash, PaymentRequest -from .models import Invoice, Donator, PayInvoiceResult, Donation +from .models import Invoice, Donator, PayInvoiceResult, Donation, SocialAccount from .core import as_task, register_command, ContextualObject, from_base64, to_base64 from .api_utils import track_donation, auto_transfer_donations from .db_donations import DonationsDbLib @@ -91,7 +91,7 @@ async def request(self, api: str, method: str, **kwargs): url=url, **kwargs, ) as resp: - logger.debug("response: %s %s %d", method, url, resp.status_code) + logger.trace("response: %s %s %d", method, url, resp.status_code) if not resp.is_success: await resp.aread() resp.raise_for_status() @@ -121,14 +121,14 @@ async def request_impl(queue): while result := await queue.get(): yield result - async def create_invoice(self, **kwargs) -> Invoice: + async def create_invoice(self, expiry: int = None, **kwargs) -> Invoice: await self.ensure_ready() resp = await self.query( 'POST', '/v1/invoices', data=dict( **kwargs, - expiry=self.settings.invoice_expiry, + expiry=expiry or self.settings.invoice_expiry, private=self.settings.private, ), ) @@ -254,10 +254,10 @@ def svg_to_png(svg_data: bytes) -> bytes: return svg2png(svg_data) -def lightning_payment_metadata(receiver: Donator) -> str: +def lightning_payment_metadata(receiver: Donator | SocialAccount) -> str: fields = [ ("text/identifier", receiver.lightning_address), - ("text/plain", f"Tip for {receiver.name} [{receiver.id}]"), + ("text/plain", f"Tip for {receiver.unique_name} [{receiver.id}]"), ] prefix = 'data:image/svg+xml;base64,' if receiver.avatar_url.startswith(prefix): diff --git a/donate4fun/models.py b/donate4fun/models.py index 9b18fece..2105d969 100644 --- a/donate4fun/models.py +++ b/donate4fun/models.py @@ -78,10 +78,11 @@ class WithdrawalToken(BaseModel): withdrawal_id: UUID -class SocialProvider(str, Enum): +class SocialProviderId(str, Enum): youtube = 'youtube' twitter = 'twitter' github = 'github' + donate4fun = 'donate4fun' class DonateRequest(BaseModel): @@ -91,7 +92,7 @@ class DonateRequest(BaseModel): twitter_account_id: UUID | None # Deprecated github_user_id: UUID | None # Deprecated social_account_id: UUID | None - social_provider: SocialProvider | None + social_provider: SocialProviderId | None target: HttpUrl | None lightning_address: LightningAddress | None donator_twitter_handle: str | None @@ -141,7 +142,7 @@ class IdModel(BaseModel): class SocialAccount(IdModel): - provider: SocialProvider + provider: SocialProviderId last_fetched_at: datetime | None balance: int = 0 total_donated: int = 0 @@ -157,7 +158,7 @@ def display_name(self): class YoutubeChannel(SocialAccount): - provider: SocialProvider = SocialProvider.youtube.value + provider: SocialProviderId = SocialProviderId.youtube.value title: str channel_id: str thumbnail_url: Url | None @@ -202,7 +203,7 @@ class Config: class TwitterAccount(SocialAccount): - provider: SocialProvider = SocialProvider.twitter.value + provider: SocialProviderId = SocialProviderId.twitter.value user_id: int handle: str name: str | None @@ -232,14 +233,18 @@ class Config: class GithubUser(SocialAccount): - provider: SocialProvider = SocialProvider.github.value + provider: SocialProviderId = SocialProviderId.github.value user_id: int login: str name: str avatar_url: AnyUrl @property - def unique_name(self): + def unique_name(self) -> str: + return self.login + + @property + def display_name(self) -> str: return self.name class Config: @@ -258,6 +263,10 @@ class Donator(IdModel): lightning_address: str | None connected: bool | None + @property + def unique_name(self) -> str: + return self.name + @validator('name', always=True) def generate_name(cls, v, values): if v is not None: diff --git a/donate4fun/settings.py b/donate4fun/settings.py index a9f79717..19cfe7b3 100644 --- a/donate4fun/settings.py +++ b/donate4fun/settings.py @@ -28,20 +28,29 @@ class YoutubeSettings(BaseModel): class TwitterOAuth(BaseModel): + bearer_token: str | None client_id: str client_secret: str consumer_key: str consumer_secret: str -class TwitterSettings(BaseModel): - bearer_token: str +class TwitterBotSettings(BaseModel): + enabled: bool + oauth: TwitterOAuth | None + + +class TwitterConversationsBotSettings(TwitterBotSettings): greeting: str - enable_bot: bool + + +class TwitterSettings(BaseModel): self_id: int dm_check_interval: timedelta refresh_timeout: timedelta - oauth: TwitterOAuth + linking_oauth: TwitterOAuth + conversations_bot: TwitterConversationsBotSettings + mentions_bot: TwitterBotSettings class GithubSettings(BaseModel): @@ -121,6 +130,7 @@ class LnurlpSettings(BaseModel): class PostHogSettings(BaseModel): + enabled: bool project_api_key: str host: str debug: bool = False diff --git a/donate4fun/social.py b/donate4fun/social.py new file mode 100644 index 00000000..777152c8 --- /dev/null +++ b/donate4fun/social.py @@ -0,0 +1,71 @@ +from abc import ABC, abstractmethod + +from furl import furl + +from .db import DbSession +from .db_social import SocialDbWrapper +from .models import SocialProviderId, Donation, SocialAccount +from .types import UnsupportedTarget + + +class SocialProvider(ABC): + @abstractmethod + async def query_or_fetch_account(self, db: DbSession, handle: str): + pass + + @abstractmethod + def wrap_db(self, db_session: DbSession) -> SocialDbWrapper: + pass + + @abstractmethod + async def apply_target(self, donation: Donation, target: str, db_session: DbSession): + """ + Resolves target url and fills donation fields + """ + pass + + @abstractmethod + def get_account_path(self, account: SocialAccount) -> str: + pass + + @staticmethod + def create(provider: SocialProviderId) -> 'SocialProvider': + match provider: + case SocialProviderId.twitter: + from .twitter_provider import TwitterProvider + return TwitterProvider() + case SocialProviderId.youtube: + from .youtube_provider import YoutubeProvider + return YoutubeProvider() + case SocialProviderId.github: + from .github_provider import GithubProvider + return GithubProvider() + case SocialProviderId.donate4fun: + from .donate4fun_provider import Donate4FunProvider + return Donate4FunProvider() + case _: + raise NotImplementedError + + @classmethod + def from_url(cls, target: furl): + match target.host: + case 'youtube.com' | 'www.youtube.com' | 'youtu.be': + return cls.create(SocialProviderId.youtube) + case 'twitter.com' | 'www.twitter.com': + return cls.create(SocialProviderId.twitter) + case 'github.com' | 'www.github.com': + return cls.create(SocialProviderId.github) + case _: + raise UnsupportedTarget("URL is invalid") + + @classmethod + def from_slug(cls, slug: str): + match slug: + case 'tw': + return cls.create(SocialProviderId.twitter) + case 'gh': + return cls.create(SocialProviderId.github) + case 'yt': + return cls.create(SocialProviderId.youtube) + case _: + raise UnsupportedTarget("Slug is invalid") diff --git a/donate4fun/twitter.py b/donate4fun/twitter.py index b7a53c5f..61fededc 100644 --- a/donate4fun/twitter.py +++ b/donate4fun/twitter.py @@ -1,55 +1,32 @@ -import asyncio import logging -import time -import secrets -import json -import io -import os +import asyncio import re -from uuid import UUID -from base64 import b64encode from contextlib import asynccontextmanager -from dataclasses import dataclass -from datetime import datetime, timedelta -from urllib.parse import urljoin, quote_plus -from typing import Any -from itertools import groupby +from datetime import datetime from functools import partial +from io import BytesIO +from typing import Any -import qrcode -import anyio import httpx -from qrcode.image.styledpil import StyledPilImage -from lnurl.core import _url_encode as lnurl_encode -from starlette.datastructures import URL from authlib.integrations.httpx_client import AsyncOAuth1Client, AsyncOAuth2Client -from furl import furl -from .db import NoResultFound, Database, DbSession, db +from .db import Database, db from .db_twitter import TwitterDbLib -from .models import Donation, TwitterAccount, TwitterTweet, WithdrawalToken, Donator -from .types import ValidationError, EntityTooOld -from .settings import settings -from .core import as_task, register_command, catch_exceptions, app -from .api_utils import scrape_lightning_address, make_absolute_uri +from .models import TwitterAccount +from .types import ValidationError +from .settings import settings, TwitterOAuth +from .core import register_command +from .api_utils import scrape_lightning_address logger = logging.getLogger(__name__) -@dataclass -class TwitterDonatee: - author_handle: str - tweet_id: str | None = None +class APIError(Exception): + pass - async def fetch(self, donation: Donation, db_session: DbSession): - db = TwitterDbLib(db_session) - if self.tweet_id is not None: - tweet = TwitterTweet(tweet_id=self.tweet_id) - # FIXME: we should possibly save link to the tweet author - await db.get_or_create_tweet(tweet) - donation.twitter_tweet = tweet - donation.twitter_account = await query_or_fetch_twitter_account(db=db, handle=self.author_handle) - donation.lightning_address = donation.twitter_account.lightning_address + +MediaID = int +TwitterHandle = str class UnsupportedTwitterUrl(ValidationError): @@ -68,159 +45,181 @@ def parse_twitter_profile_url(url: str) -> str: return match.groups('username') -def validate_twitter_url(parsed) -> TwitterDonatee: - parts = parsed.path.split('/') - if len(parts) in (2, 3): - return TwitterDonatee(author_handle=parts[1]) - elif len(parts) >= 4 and parts[2] == 'status': - return TwitterDonatee(tweet_id=int(parts[3]), author_handle=parts[1]) - else: - raise UnsupportedTwitterUrl - - -async def query_or_fetch_twitter_account(db: TwitterDbLib, **params) -> TwitterAccount: - try: - account: TwitterAccount = await db.query_account(**params) - if account.last_fetched_at is None or account.last_fetched_at < datetime.utcnow() - settings.twitter.refresh_timeout: - raise EntityTooOld - except (NoResultFound, EntityTooOld): - account: TwitterAccount = await fetch_twitter_author(**params) - await db.save_account(account) - return account - +class TwitterApiClient: + def __init__(self, client: httpx.AsyncClient): + self.client: httpx.AsyncClient = client + self.get = partial(self.request, 'GET') + self.post = partial(self.request, 'POST') -@register_command -async def fetch_and_save_twitter_account(handle: str): - async with db.session() as db_session: - account: TwitterAccount = await fetch_twitter_author(handle=handle) - await db_session.save_twitter_account(account) + async def request_raw(self, method: str, api_path: str, **kwargs) -> httpx.Response: + response = await self.client.request( + method=method, + url=api_path, + **kwargs + ) + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + if self.is_json_content_type(exc.response): + body = exc.response.json() + else: + body = exc.response.content + raise APIError(body if body else exc.response.status_code) from exc + return response + def is_json_content_type(self, response): + return response.headers.get('content-type', '').split(';', 1)[0] == 'application/json' -async def api_request_raw(method, client, api_path, **kwargs) -> httpx.Response: - response = await client.request( - method=method, - url=urljoin('https://api.twitter.com/2/', api_path), - **kwargs - ) - response.raise_for_status() - return response + async def request(self, method: str, api_path: str, **kwargs) -> dict[str, Any] | bytes: + response = await self.request_raw(method, api_path, **kwargs) + if response.status_code == 204: + return + elif self.is_json_content_type(response): + return response.json() + else: + return response.content + @asynccontextmanager + async def stream(self, method: str, api_path: str, **kwargs): + async with self.client.stream(method, api_path, **kwargs) as response: + yield response -async def api_request(method, client, api_path, **kwargs) -> dict[str, Any]: - response = await api_request_raw(method, client, api_path, **kwargs) - if response.status_code == 204: - return - elif response.headers['content-type'].split(';', 1)[0] == 'application/json': - data = response.json() - return data.get('data') or data - else: - return response.content + async def get_pages(self, api_path: str, limit: int, params: dict[str, Any], **kwargs) -> list[Any]: + results = [] + params_ = params.copy() + while True: + response = await self.request_raw('GET', api_path, params=params_, **kwargs) + if response.status_code == 204: + break + elif response.headers['content-type'].split(';', 1)[0] == 'application/json': + data = response.json() + results.extend(data.get('data', [])) + if len(results) < limit: + params_['pagination_token'] = data['meta']['next_token'] + else: + break + else: + raise InvalidResponse + return results + async def get_user_by(self, handle: str | None = None, user_id: int | None = None) -> TwitterAccount: + if handle is not None: + path = f'/users/by/username/{handle}' + elif user_id is not None: + path = f'/users/{user_id}' + else: + raise ValueError("One of handle or user_id should be provided") + data: dict = await self.get(path, params={'user.fields': get_user_fields()}) + return api_data_to_twitter_account(data['data']) -api_get = partial(api_request, 'GET') -api_post = partial(api_request, 'POST') + async def get_me(self) -> TwitterAccount: + data: dict = await self.get('/users/me', params={'user.fields': get_user_fields()}) + return api_data_to_twitter_account(data['data']) + async def upload_media(self, image: bytes, mime_type: str, category: str) -> MediaID: + upload_url = 'https://upload.twitter.com/1.1/media/upload.json' + params = dict( + command='INIT', + total_bytes=len(image), + media_type=mime_type, + media_category=category, + ) + media_info = await self.post(upload_url, params=params) + media_id = media_info['media_id'] + await self.post( + upload_url, + data=dict( + command='APPEND', + media_id=media_id, + segment_index=0, + ), + files=dict(media=('image', BytesIO(image), mime_type)), + ) + state_response = await self.post( + upload_url, + params=dict( + command='FINALIZE', + media_id=media_id, + ), + ) + if 'processing_info' in state_response: + while True: + info = state_response['processing_info'] + if info['state'] == 'succeeded': + break + check_after_secs = info['check_after_secs'] + logger.debug("upload_media: state is %s, sleeping for %d seconds", info, check_after_secs) + await asyncio.sleep(check_after_secs) + state_response = await self.post(upload_url, params=dict( + command='STATUS', + media_id=media_id, + )) + logger.trace("Media uploaded: %s", state_response) + return media_id -async def api_get_pages(client, api_path: str, limit: int, params: dict[str, Any], **kwargs): - results = [] - params_ = params.copy() - while True: - response = await api_request_raw('GET', client, api_path, params=params_, **kwargs) - if response.status_code == 204: - break - elif response.headers['content-type'].split(';', 1)[0] == 'application/json': - data = response.json() - results.extend(data.get('data', [])) - if len(results) < limit: - params_['pagination_token'] = data['meta']['next_token'] - else: - break - else: - raise InvalidResponse - return results +@register_command +async def fetch_and_save_twitter_account(handle: str): + async with db.session() as db_session, make_apponly_client(token=settings.twitter.linking_oauth.bearer_token) as client: + account: TwitterAccount = await TwitterApiClient(client).get_user_by(handle=handle) + await db_session.save_twitter_account(account) -async def save_token(db: Database, token: dict[str, Any], refresh_token: str): - logger.debug(f"new token: {token} ({refresh_token})") - async with db.session() as db_session: - await db_session.save_oauth_token('twitter_oauth2', token) +class OAuthTokenLoader: + def __init__(self, name: str): + self.name = name -def make_bot_oauth2_client(token=None, update_token=None): - scope = "tweet.read users.read dm.read dm.write offline.access" - return make_oauth2_client(scope=scope, token=token, update_token=update_token) + @property + def settings(self): + return self.settings_getter() + async def load(self) -> dict: + async with db.session() as db_session: + return await TwitterDbLib(db_session).query_oauth_token(self.name) -def make_link_oauth2_client(token=None): - """ - This client is used to link Twitter account to donator (OAuth2 flow) - """ - return make_oauth2_client(scope='tweet.read users.read', token=token) + async def save(self, token: dict): + async with db.session() as db_session: + await TwitterDbLib(db_session).save_oauth_token(self.name, token) @asynccontextmanager -async def make_oauth2_client(scope: str, token=None, update_token=None): - oauth = settings.twitter.oauth +async def make_oauth2_client(oauth: TwitterOAuth, scope: str, token: dict | None = None, update_token=None, redirect_uri=None): async with AsyncOAuth2Client( client_id=oauth.client_id, client_secret=oauth.client_secret, scope=scope, token_endpoint='https://api.twitter.com/2/oauth2/token', - redirect_uri=make_absolute_uri(app.url_path_for('oauth_redirect', provider='twitter')), + redirect_uri=redirect_uri, code_challenge_method='S256', token=token, update_token=update_token, + base_url='https://api.twitter.com/2', ) as client: yield client @asynccontextmanager -async def make_oauth1_client(**kwargs): - oauth = settings.twitter.oauth +async def make_oauth1_client(oauth: TwitterOAuth, token: dict | None = None, **kwargs): async with AsyncOAuth1Client( client_id=oauth.consumer_key, client_secret=oauth.consumer_secret, + base_url='https://api.twitter.com/2', **kwargs, ) as client: + if token: + client.token = token yield client @asynccontextmanager -async def make_noauth_client(): - token = settings.twitter.bearer_token - async with httpx.AsyncClient(headers=dict(authorization=f'Bearer {token}')) as client: +async def make_apponly_client(token: str): + async with httpx.AsyncClient( + headers=dict(authorization=f'Bearer {token}'), + base_url='https://api.twitter.com/2', + ) as client: yield client -@register_command -async def obtain_twitter_oauth2_token(): - async with make_oauth2_client() as client: - code_verifier = secrets.token_urlsafe(43) - url, state = client.create_authorization_url( - url='https://twitter.com/i/oauth2/authorize', code_verifier=code_verifier, - ) - authorization_response = input(f"Follow this url and enter resulting url after redirect: {url}\n") - token: dict[str, Any] = await client.fetch_token( - authorization_response=authorization_response, - code_verifier=code_verifier, - ) - db = Database(settings.db) - async with db.session() as db_session: - await db_session.save_oauth_token('twitter_oauth2', token) - - -@register_command -async def obtain_twitter_oauth1_token(): - async with make_oauth1_client(redirect_uri='oob') as client: - await client.fetch_request_token('https://api.twitter.com/oauth/request_token') - auth_url = client.create_authorization_url('https://api.twitter.com/oauth/authorize') - pin: str = input(f"Open this url {auth_url} and paste here PIN:\n") - token: dict[str, Any] = await client.fetch_access_token('https://api.twitter.com/oauth/access_token', verifier=pin) - db = Database(settings.db) - async with db.session() as db_session: - await db_session.save_oauth_token('twitter_oauth1', token) - - def get_user_fields(): return 'id,name,profile_image_url,description,verified,entities' @@ -236,323 +235,20 @@ def api_data_to_twitter_account(data: dict): ) -async def fetch_twitter_author(handle: str | None = None, user_id: int | None = None) -> TwitterAccount: - async with make_noauth_client() as client: - if handle is not None: - path = f'users/by/username/{handle}' - elif user_id is not None: - path = f'users/{user_id}' - else: - raise ValueError("One of handle or user_id should be provided") - data: dict = await api_get(client, path, params={'user.fields': get_user_fields()}) - return api_data_to_twitter_account(data) - - -async def fetch_twitter_me(client) -> TwitterAccount: - data: dict = await api_get(client, 'users/me', params={'user.fields': get_user_fields()}) - return api_data_to_twitter_account(data) - - -@dataclass -class DirectMessage: - is_me: bool - text: str - created_at: datetime - - -async def fetch_conversations(client): - logger.trace("fetching new twitter direct messages") - params = { - 'dm_event.fields': 'sender_id,created_at,dm_conversation_id', - 'max_results': 100, - } - events: list[dict[str, Any]] = await api_get_pages(client, 'dm_events', limit=100, params=params) - logger.trace("Fetched direct messages %s", events) - self_id = settings.twitter.self_id - - def keyfunc(event): - return event['dm_conversation_id'] - for dm_conversation_id, conversation_events in groupby(sorted(events, key=keyfunc), keyfunc): - sorted_events = sorted(conversation_events, key=lambda event: event['created_at']) - last_message = sorted_events[-1] - created_at = datetime.fromisoformat(last_message['created_at'][:-1]) # Remove trailing Z - if int(last_message['sender_id']) == self_id and created_at < datetime.utcnow() - timedelta(minutes=2): - # Ignore chats where last message is ours and old - continue - yield dm_conversation_id - - -def get_prove_text_regexp(): - return re.compile(make_prove_message(r'(?P[a-z0-9\-]+)')) - - -def make_prove_message(donator_id: UUID | str): - return f'Hereby I confirm that {donator_id} is my Donate4Fun account id' - - -class Conversation: - def __init__(self, oauth1_client, oauth2_client, db: Database, conversation_id: str): - self.oauth1_client = oauth1_client - self.oauth2_client = oauth2_client - self.db = db - self.is_stale = False - self.conversation_id: str = conversation_id - self.peer_id = int(conversation_id.replace(str(settings.twitter.self_id), '').replace('-', '')) - - async def fetch_messages(self): - events = await api_get( - self.oauth2_client, f'dm_conversations/{self.conversation_id}/dm_events', - params={'dm_event.fields': 'sender_id,created_at,dm_conversation_id', 'max_results': 100}, - ) - return [DirectMessage( - text=event['text'], - created_at=datetime.fromisoformat(event['created_at'][:-1]), # Remove trailing Z - is_me=int(event['sender_id']) != self.peer_id, - ) for event in events] - - async def reply_no_donations(self): - url = ( - "https://twitter.com/intent/tweet?text=" - + quote_plus("Tip me #Bitcoin through a #LightningNetwork using https://donate4.fun") - ) - await self.send_text(f"You have no donations, but if you want any then tweet about us {url}") - - async def send_text(self, text: str, **params): - logger.trace("Sending text to %d: %s", self.peer_id, text) - await api_post( - self.oauth2_client, f'dm_conversations/{self.conversation_id}/messages', - json=dict(text=text, **params), - ) - - async def conversate(self, text: str): - await self.send_text(text) - start = time.time() - while time.time() < start + settings.twitter.answer_timeout: - await api_get(self.oauth2_client, f'dm_conversations/{self.conversation_id}/dm_events') - - @catch_exceptions - async def chat_loop(self): - try: - while True: - history = await self.fetch_messages() - if not history: - break - # Twitter API always returns messages in descending order by created_at - last_message = history[0] - logger.trace("last message %s", last_message) - if last_message.is_me and last_message.created_at < datetime.utcnow() - timedelta(minutes=2): - break - elif not last_message.is_me: - logger.info(f"answering to {last_message}") - for message in history: - if message.is_me: - continue - if match := get_prove_text_regexp().match(last_message.text): - await self.link_twitter_account(match.group('donator_id')) - break - else: - await self.answer_withdraw() - await asyncio.sleep(5) - finally: - self.is_stale = True - - async def link_twitter_account(self, donator_id: str): - logger.info(f"Linking Twitter account {self.peer_id} to {donator_id}") - async with self.db.session() as db_session: - twitter_db = TwitterDbLib(db_session) - author: TwitterAccount = await query_or_fetch_twitter_account(twitter_db, user_id=self.peer_id) - await twitter_db.link_twitter_account(twitter_author=author, donator=Donator(id=donator_id)) - profile_url = furl(settings.base_url) / 'donator' / donator_id - await self.send_text(f"Your account is successefully linked. Go to {profile_url} to claim your donations.") - - async def answer_withdraw(self): - async with self.db.session() as db_session: - await self.send_text(settings.twitter.greeting) - try: - author: TwitterAccount = await db_session.query_twitter_account(user_id=self.peer_id) - except NoResultFound: - await self.reply_no_donations() - else: - if author.total_donated == 0: - await self.reply_no_donations() - elif author.balance == 0: - await self.send_text("All donations have been claimed.") - elif author.balance < settings.min_withdraw: - await self.send_text( - f"You have {author.balance} sats, but minimum withdraw amount is {settings.min_withdraw} sats." - ) - else: - prove_url = f'{settings.base_url}/twitter/prove' - await self.send_text( - f"You have {author.balance} sats! Go to {prove_url}, connect your Twitter account" - " and then you will be able to claim and withdraw your funds." - ) - return - # FIXME: this is old code withdraw directly using Twitter. It's not used anymore - # because withdrawals are only possible from a Donator account - # But if we generate ephemeral Donator account for this and transfer funds there - # then user will not be able to withdraw other way, which could be a bad situation - # Good solution is to allow users to login to a different donator account when linking social account - lnurl: str = await create_withdrawal(db_session, twitter_account=author) - qrcode: bytes = make_qr_code(lnurl) - media_id: int = await upload_media(self.oauth1_client, qrcode, 'image/png') - await self.send_text( - "Here are your withdrawal invoice. Scan it with your Bitcoin Lightning Wallet." - f" Or copy LNURL: {lnurl}", - attachments=[dict(media_id=str(media_id))], - ) - - -async def create_withdrawal(db_session, twitter_account): - # Generate new donator like a new site visitor - donator = Donator() - await db_session.link_twitter_account(twitter_author=twitter_account, donator=donator) - withdrawal_id: UUID = await db_session.create_withdrawal(donator) - token = WithdrawalToken(withdrawal_id=withdrawal_id) - url = urljoin(settings.lnd.lnurl_base_url, '/lnurl/withdraw') - withdraw_url = URL(url).include_query_params(token=token.to_jwt()) - return lnurl_encode(str(withdraw_url)) - - -def make_qr_code(data: str) -> bytes: - qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_H) - qr.add_data(data) - image: StyledPilImage = qr.make_image() - image_data = io.BytesIO() - image.save(image_data, "PNG") - return image_data.getvalue() - - -@register_command -async def make_withdrawal_lnurl(author_id): - async with Database(settings.db).session() as db_session: - lnurl: str = await create_withdrawal(db_session, twitter_author_id=author_id) - print(f"lightning:{lnurl.lower()}") - - -@register_command -async def test_make_qr_code(author_id): - async with Database(settings.db).session() as db_session: - lnurl: str = await create_withdrawal(db_session, twitter_author_id=author_id) - qrcode = make_qr_code(lnurl) - filename = 'tmp-qrcode.png' - with open(filename, 'wb') as f: - f.write(qrcode) - os.system(f'xdg-open {filename}') - - -async def upload_media(client, image: bytes, mime_type: str) -> int: - upload_url = 'https://upload.twitter.com/1.1/media/upload.json' - media_info = await api_post( - client, upload_url, params=dict( - command='INIT', - total_bytes=len(image), - media_type=mime_type, - media_category='dm_image', - ) - ) - media_id = media_info['media_id'] - await api_post( - client, upload_url, - data=dict( - command='APPEND', - media_id=media_id, - segment_index=0, - media_data=b64encode(image).decode(), - ), - ) - state_response = await api_post( - client, upload_url, - params=dict( - command='FINALIZE', - media_id=media_id, - ), - ) - if 'processing_info' in state_response: - while True: - info = state_response['processing_info'] - if info['state'] == 'succeeded': - break - check_after_secs = info['check_after_secs'] - logger.debug("upload_media: state is %s, sleeping for %d seconds", info, check_after_secs) - await asyncio.sleep(check_after_secs) - state_response = await api_post( - client, upload_url, params=dict( - command='STATUS', - media_id=media_id, - ) - ) - logger.trace("Media uploaded: %s", state_response) - return media_id - - @register_command async def get_profile_banner(): + """ + Does it work? + """ async with Database(settings.db).session() as db_session: token: dict = await db_session.query_oauth_token('twitter_oauth1') async with make_oauth1_client() as client: client.token = token url = 'https://api.twitter.com/1.1/users/profile_banner.json' try: - response = await api_get(client, url, params=dict(screen_name='elonmusk')) + response = await client.get(url, params=dict(screen_name='elonmusk')) except httpx.HTTPStatusError as exc: auth_header = exc.request.headers['authorization'] logger.exception("Failed to get banner image:\n%s\n%s\n%s", auth_header, exc.request.headers, exc.response.json()) else: print(response.json()) - - -@register_command -async def test_upload_media(): - async with Database(settings.db).session() as db_session: - token: dict = await db_session.query_oauth_token('twitter_oauth1') - async with make_oauth1_client() as client: - client.token = token - try: - await upload_media(client, open('frontend/public/static/D-16.png', 'rb').read(), 'image/png') - except httpx.HTTPStatusError as exc: - auth_header = exc.request.headers['authorization'] - logger.exception("Failed to upload image:\n%s\n%s\n%s", auth_header, exc.request.headers, exc.response.json()) - - -@register_command -async def fetch_twitter_conversations(token: str): - oauth2_ctx = make_bot_oauth2_client(token=json.loads(token)) - async with oauth2_ctx as oauth2_client: - print([conversation_id async for conversation_id in fetch_conversations(oauth2_client)]) - - -@as_task -async def run_twitter_bot_restarting(db: Database): - while True: - try: - await run_twitter_bot(db) - except Exception: - logger.exception("Exception while running Twitter bot") - await asyncio.sleep(15) - - -async def run_twitter_bot(db: Database): - conversations = {} - async with db.session() as db_session: - oauth1_token = await db_session.query_oauth_token('twitter_oauth1') - oauth2_token = await db_session.query_oauth_token('twitter_oauth2') - logger.debug("Using tokens %s %s", oauth1_token, oauth2_token) - - oauth1_ctx = make_oauth1_client() - oauth2_ctx = make_bot_oauth2_client(token=oauth2_token, update_token=partial(save_token, db)) - async with oauth1_ctx as oauth1_client, oauth2_ctx as oauth2_client, anyio.create_task_group() as tg: - oauth1_client.token = oauth1_token - while True: - async for conversation_id in fetch_conversations(oauth2_client): - if conversation_id not in conversations: - logger.debug("Creating conversation %s", conversation_id) - conversations[conversation_id] = conversation = Conversation( - conversation_id=conversation_id, db=db, oauth2_client=oauth2_client, oauth1_client=oauth1_client, - ) - tg.start_soon(conversation.chat_loop) - for conversation_id, conversation in conversations.copy().items(): - if conversation.is_stale: - logger.debug("Removing conversation %s", conversation_id) - del conversations[conversation_id] - await asyncio.sleep(settings.twitter.dm_check_interval.total_seconds()) diff --git a/donate4fun/twitter_bot.py b/donate4fun/twitter_bot.py new file mode 100644 index 00000000..f2413db2 --- /dev/null +++ b/donate4fun/twitter_bot.py @@ -0,0 +1,479 @@ +import asyncio +import logging +import time +import json +import re +import io +import os +import secrets +from contextlib import asynccontextmanager, AsyncExitStack +from dataclasses import dataclass +from datetime import datetime, timedelta +from functools import wraps +from itertools import groupby +from typing import AsyncIterator +from urllib.parse import quote_plus, urljoin +from uuid import UUID + +import anyio +import httpx +import qrcode +from furl import furl +from qrcode.image.styledpil import StyledPilImage +from lnurl.core import _url_encode as lnurl_encode +from starlette.datastructures import URL + +from .db import NoResultFound, Database, db +from .models import TwitterAccount, Donator, WithdrawalToken, Donation, TwitterAccountOwned, TwitterTweet, PaymentRequest +from .core import as_task, register_command, catch_exceptions +from .settings import settings +from .twitter import ( + make_oauth2_client, TwitterHandle, + make_oauth1_client, OAuthTokenLoader, TwitterApiClient, make_apponly_client, +) +from .twitter_provider import TwitterProvider +from .donation import donate +from .api_utils import make_absolute_uri +from .lnd import lnd, LndClient + +logger = logging.getLogger(__name__) + + +@dataclass +class DirectMessage: + is_me: bool + text: str + created_at: datetime + + +class BaseTwitterBot: + provider = TwitterProvider() + + @classmethod + @asynccontextmanager + async def create(cls) -> AsyncIterator['BaseTwitterBot']: + async with AsyncExitStack() as stack: + obj = cls() + if cls.oauth1_token: + oauth1_token: dict = await cls.oauth1_token.load() + oauth1_ctx = make_oauth1_client(oauth=cls.oauth_settings, token=oauth1_token) + obj.client = TwitterApiClient(await stack.enter_async_context(oauth1_ctx)) + if cls.oauth2_token: + oauth2_token: dict = await cls.oauth2_token.load() + oauth2_ctx = make_oauth2_client(scope=cls.oauth2_scope, oauth=cls.oauth_settings, token=oauth2_token) + obj.oauth2_client = TwitterApiClient(await stack.enter_async_context(oauth2_ctx)) + if cls.oauth_settings.bearer_token: + apponly_ctx = make_apponly_client(token=cls.oauth_settings.bearer_token) + obj.apponly_client = TwitterApiClient(await stack.enter_async_context(apponly_ctx)) + yield obj + + @classmethod + async def obtain_oauth1_token(cls): + async with make_oauth1_client(oauth=cls.oauth_settings, redirect_uri='oob') as client: + await client.fetch_request_token('https://api.twitter.com/oauth/request_token') + auth_url = client.create_authorization_url('https://api.twitter.com/oauth/authorize') + pin: str = input(f"Open this url {auth_url} and paste here PIN:\n") + data: dict = await client.fetch_access_token('https://api.twitter.com/oauth/access_token', verifier=pin) + await cls.oauth1_token.save(data) + + @classmethod + async def obtain_oauth2_token(cls): + client_ctx = make_oauth2_client(scope=cls.oauth2_scope, oauth=cls.oauth_settings, redirect_uri='http://localhost') + async with client_ctx as client: + code_verifier = secrets.token_urlsafe(43) + url, state = client.create_authorization_url( + url='https://twitter.com/i/oauth2/authorize', code_verifier=code_verifier, + ) + authorization_response = input(f"Follow this url and enter resulting url after redirect: {url}\n") + token: dict = await client.fetch_token( + authorization_response=authorization_response, + code_verifier=code_verifier, + ) + await cls.oauth2_token.save(token) + + @classmethod + async def validate_tokens(cls): + async with cls.create() as bot: + donate4fun_user_id = 1572908920485576704 + await bot.client.get_user_by(user_id=donate4fun_user_id) + await bot.oauth2_client.get_user_by(user_id=donate4fun_user_id) + await bot.apponly_client.get_user_by(user_id=donate4fun_user_id) + + +def make_prove_message(donator_id: UUID | str): + return f'Hereby I confirm that {donator_id} is my Donate4Fun account id' + + +class Conversation: + def __init__(self, conversation_id: str, api_client: TwitterApiClient): + self.client: TwitterApiClient = api_client + self.is_stale: bool = False + self.conversation_id: str = conversation_id + self.peer_id: int = int(conversation_id.replace(str(settings.twitter.self_id), '').replace('-', '')) + + async def fetch_messages(self): + events = await self.client.get( + f'dm_conversations/{self.conversation_id}/dm_events', + params={'dm_event.fields': 'sender_id,created_at,dm_conversation_id', 'max_results': 100}, + ) + return [DirectMessage( + text=event['text'], + created_at=datetime.fromisoformat(event['created_at'][:-1]), # Remove trailing Z + is_me=int(event['sender_id']) != self.peer_id, + ) for event in events] + + async def reply_no_donations(self): + url = ( + "https://twitter.com/intent/tweet?text=" + + quote_plus("Tip me #Bitcoin through a #LightningNetwork using https://donate4.fun") + ) + await self.send_text(f"You have no donations, but if you want any then tweet about us {url}") + + async def send_text(self, text: str, **params): + logger.trace("Sending text to %d: %s", self.peer_id, text) + await self.client.post( + f'/dm_conversations/{self.conversation_id}/messages', + json=dict(text=text, **params), + ) + + async def conversate(self, text: str): + await self.send_text(text) + start = time.time() + while time.time() < start + settings.twitter.answer_timeout: + await self.client.get(f'/dm_conversations/{self.conversation_id}/dm_events') + + def get_prove_text_regexp(self): + return re.compile(make_prove_message(r'(?P[a-z0-9\-]+)')) + + @catch_exceptions + async def chat_loop(self): + try: + while True: + history = await self.fetch_messages() + if not history: + break + # Twitter API always returns messages in descending order by created_at + last_message = history[0] + logger.trace("last message %s", last_message) + if last_message.is_me and last_message.created_at < datetime.utcnow() - timedelta(minutes=2): + break + elif not last_message.is_me: + logger.info(f"answering to {last_message}") + for message in history: + if message.is_me: + continue + if match := self.get_prove_text_regexp().match(last_message.text): + await self.link_twitter_account(match.group('donator_id')) + break + else: + await self.answer_withdraw() + await asyncio.sleep(5) + finally: + self.is_stale = True + + async def link_twitter_account(self, donator_id: str): + logger.info(f"Linking Twitter account {self.peer_id} to {donator_id}") + async with db.session() as db_session: + twitter_db = self.provider.wrap_db(db_session) + author: TwitterAccount = await self.provider.query_or_fetch_account(twitter_db, user_id=self.peer_id) + await twitter_db.link_twitter_account(twitter_author=author, donator=Donator(id=donator_id)) + profile_url = furl(settings.base_url) / 'donator' / donator_id + await self.send_text(f"Your account is successefully linked. Go to {profile_url} to claim your donations.") + + async def answer_withdraw(self): + async with db.session() as db_session: + await self.send_text(settings.twitter.conversations_bot.greeting) + try: + author: TwitterAccount = await db_session.query_twitter_account(user_id=self.peer_id) + except NoResultFound: + await self.reply_no_donations() + else: + if author.total_donated == 0: + await self.reply_no_donations() + elif author.balance == 0: + await self.send_text("All donations have been claimed.") + elif author.balance < settings.min_withdraw: + await self.send_text( + f"You have {author.balance} sats, but minimum withdraw amount is {settings.min_withdraw} sats." + ) + else: + prove_url = f'{settings.base_url}/twitter/prove' + await self.send_text( + f"You have {author.balance} sats! Go to {prove_url}, connect your Twitter account" + " and then you will be able to claim and withdraw your funds." + ) + return + # FIXME: this is old code withdraw directly using Twitter. It's not used anymore + # because withdrawals are only possible from a Donator account + # But if we generate ephemeral Donator account for this and transfer funds there + # then user will not be able to withdraw other way, which could be a bad situation + # Good solution is to allow users to login to a different donator account when linking social account + lnurl: str = await create_withdrawal(db_session, twitter_account=author) + qrcode: bytes = make_qr_code(lnurl) + media_id: int = await self.client.upload_media(qrcode, 'image/png', category='dm_image') + await self.send_text( + "Here are your withdrawal invoice. Scan it with your Bitcoin Lightning Wallet." + f" Or copy LNURL: {lnurl}", + attachments=[dict(media_id=str(media_id))], + ) + + +class ConversationsBot(BaseTwitterBot): + oauth1_token = OAuthTokenLoader('twitter_converstaions_oauth1') + oauth2_token = OAuthTokenLoader('twitter_converstaions_oauth2') + oauth2_scope = "tweet.read users.read dm.read dm.write offline.access" + + @classmethod + @property + def oauth_settings(cls): + return settings.twitter.conversations_bot.oauth + + async def run_loop(self): + conversations = {} + async with anyio.create_task_group() as tg: + while True: + async for conversation_id in self.fetch_conversations(): + if conversation_id not in conversations: + logger.debug("Creating conversation %s", conversation_id) + conversation = Conversation(conversation_id=conversation_id, client=self.client) + conversations[conversation_id] = conversation + tg.start_soon(conversation.chat_loop) + for conversation_id, conversation in conversations.copy().items(): + if conversation.is_stale: + logger.debug("Removing conversation %s", conversation_id) + del conversations[conversation_id] + await asyncio.sleep(settings.twitter.dm_check_interval.total_seconds()) + + async def fetch_conversations(self): + logger.trace("fetching new twitter direct messages") + params = { + 'dm_event.fields': 'sender_id,created_at,dm_conversation_id', + 'max_results': 100, + } + events: list[dict] = await self.oauth2_client.get_pages('dm_events', limit=100, params=params) + logger.trace("Fetched direct messages %s", events) + self_id = settings.twitter.self_id + + def keyfunc(event): + return event['dm_conversation_id'] + for dm_conversation_id, conversation_events in groupby(sorted(events, key=keyfunc), keyfunc): + sorted_events = sorted(conversation_events, key=lambda event: event['created_at']) + last_message = sorted_events[-1] + created_at = datetime.fromisoformat(last_message['created_at'][:-1]) # Remove trailing Z + if int(last_message['sender_id']) == self_id and created_at < datetime.utcnow() - timedelta(minutes=2): + # Ignore chats where last message is ours and old + continue + yield dm_conversation_id + + +register_command(ConversationsBot.obtain_oauth1_token, 'obtain_conversations_bot_oauth1_token') +register_command(ConversationsBot.obtain_oauth2_token, 'obtain_conversations_bot_oauth2_token') + + +class MentionsBot(BaseTwitterBot): + oauth1_token = OAuthTokenLoader('twitter_mentions_bot_oauth1') + oauth2_token = OAuthTokenLoader('twitter_mentions_bot_oauth2') + oauth2_scope = "tweet.read tweet.write users.read dm.read dm.write offline.access" + + @classmethod + @property + def oauth_settings(cls): + return settings.twitter.mentions_bot.oauth + + async def run_loop(self, handle: TwitterHandle): + try: + async with anyio.create_task_group() as tg: + async for mention in self.fetch_mentions(handle): + tg.start_soon(self.handle_mention, mention) + except httpx.HTTPStatusError as exc: + print(exc.response.json()) + raise + + async def handle_mention(self, mention: dict): + logger.trace("handling mention %s", mention) + receiver_user_id: int = int(mention['in_reply_to_user_id']) + donator_user_id: int = int(mention['author_id']) + referenced_tweet_id: int = int(mention['referenced_tweets'][0]['id']) + tweet_id: int = int(mention['id']) + text: str = mention['text'] + async with db.session() as db_session: + twitter_db = self.provider.wrap_db(db_session) + donator_account_: TwitterAccount = await self.provider.query_or_fetch_account(db=twitter_db, user_id=donator_user_id) + donator_account: TwitterAccountOwned = await twitter_db.query_account(id=donator_account_.id) + if donator_account.owner_id is None: + donator = Donator() + else: + donator = Donator(id=donator_account.owner_id) + receiver_account: TwitterAccount = await self.provider.query_or_fetch_account(db=twitter_db, user_id=receiver_user_id) + if match := re.search(r' (?P\d+) ?(?P[kK])?', text): + amount = int(match['amount']) + if match['has_k']: + amount *= 1000 + else: + await self.do_not_understand(tweet_id) + return + logger.debug( + "handling donation from @%s to @%s for %d sats via tweet %d", + donator_account.handle, receiver_account.handle, amount, tweet_id, + ) + tweet: TwitterTweet = TwitterTweet(tweet_id=referenced_tweet_id) + await twitter_db.get_or_create_tweet(tweet) + donation = Donation( + donator=donator, + twitter_account=receiver_account, + twitter_tweet=tweet, + donator_twitter_account=donator_account, + amount=amount, + ) + with lnd.assign(LndClient(settings.lnd)): + # Long expiry because this will be posted in Twitter + pay_req, donation = await donate(donation, db_session, expiry=3600) + if pay_req: + await self.send_payreq(tweet_id, receiver_account.handle, pay_req) + elif donation.paid_at: + await self.share_donation_preview(tweet_id, donation) + else: + raise RuntimeError("invalid state") + + async def invite_new_user(self, tweet_id: int): + print("inviting new user") + + async def do_not_understand(self, tweet_id: int): + print("i do not understand") + + async def share_donation_preview(self, tweet_id: int, donation: Donation): + await self.send_tweet(reply_to=tweet_id, text=make_absolute_uri(f'/donation/{donation.id}')) + + async def send_payreq(self, tweet_id: int, handle: TwitterHandle, pay_req: PaymentRequest): + qrcode: bytes = make_qr_code(pay_req) + media_id: int = await self.client.upload_media(qrcode, 'image/png', category='tweet_image') + url: str = make_absolute_uri(f'/lnurlp/twitter/{handle}') + await self.send_tweet(text=f"Invoice: {url}", reply_to=tweet_id, media_id=media_id) + + async def fetch_mentions(self, handle: TwitterHandle) -> AsyncIterator[dict]: + logger.info("fetching new mentions for @%s", handle) + current_rules: list = (await self.apponly_client.get('/tweets/search/stream/rules')).get('data', []) + logger.debug("current rules: %s", current_rules) + expected_rules = [dict(value=f'@{handle} is:reply -from:{handle}')] + if [dict(value=rule['value']) for rule in current_rules] != expected_rules: + logger.debug("current rules differ with expected, overriding with %s", expected_rules) + if current_rules: + current_rule_ids = [rule['id'] for rule in current_rules] + await self.apponly_client.post('/tweets/search/stream/rules', json=dict( + delete=dict(ids=current_rule_ids) + )) + await self.apponly_client.post('/tweets/search/stream/rules', json=dict( + add=expected_rules, + )) + params = { + 'tweet.fields': 'created_at', + 'expansions': 'author_id,referenced_tweets.id,in_reply_to_user_id,referenced_tweets.id.author_id', + } + async with self.apponly_client.stream('GET', '/tweets/search/stream', params=params, timeout=3600) as response: + response.raise_for_status() + async for chunk in response.aiter_text(): + chunk = chunk.strip() + if chunk: + data: dict = json.loads(chunk) + yield data['data'] + + async def send_tweet(self, reply_to: int = None, text: str = None, media_id: int = None): + body = {} + if reply_to is not None: + body['reply'] = dict(in_reply_to_tweet_id=str(reply_to)) + if text is not None: + body['text'] = text + if media_id is not None: + body['media'] = dict(media_ids=[str(media_id)]) + tweet = (await self.client.post('/tweets', json=body))['data'] + logger.trace("tweeted https://twitter.com/status/%s: %s", tweet['id'], tweet['text']) + + +register_command(MentionsBot.obtain_oauth1_token, 'obtain_mentions_bot_oauth1_token') +register_command(MentionsBot.obtain_oauth2_token, 'obtain_mentions_bot_oauth2_token') +register_command(MentionsBot.validate_tokens, 'validate_mentions_bot_tokens') + + +@register_command +async def test_upload_media(): + async with make_oauth1_client(token=await MentionsBot.oauth1_token.load()) as client: + try: + with open('frontend/public/static/D-16.png', 'rb') as f: + await client.upload_media(f.read(), 'image/png', category='tweet_image') + except httpx.HTTPStatusError as exc: + auth_header = exc.request.headers['authorization'] + logger.exception("Failed to upload image:\n%s\n%s\n%s", auth_header, exc.request.headers, exc.response.json()) + + +async def create_withdrawal(db_session, twitter_account): + # Generate new donator like a new site visitor + donator = Donator() + await db_session.link_twitter_account(twitter_author=twitter_account, donator=donator) + withdrawal_id: UUID = await db_session.create_withdrawal(donator) + token = WithdrawalToken(withdrawal_id=withdrawal_id) + url = urljoin(settings.lnd.lnurl_base_url, '/lnurl/withdraw') + withdraw_url = URL(url).include_query_params(token=token.to_jwt()) + return lnurl_encode(str(withdraw_url)) + + +def make_qr_code(data: str) -> bytes: + qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_H) + qr.add_data(data) + image: StyledPilImage = qr.make_image() + image_data = io.BytesIO() + image.save(image_data, "PNG") + return image_data.getvalue() + + +@register_command +async def make_withdrawal_lnurl(author_id): + async with Database(settings.db).session() as db_session: + lnurl: str = await create_withdrawal(db_session, twitter_author_id=author_id) + print(f"lightning:{lnurl.lower()}") + + +@register_command +async def test_make_qr_code(author_id): + async with Database(settings.db).session() as db_session: + lnurl: str = await create_withdrawal(db_session, twitter_author_id=author_id) + qrcode = make_qr_code(lnurl) + filename = 'tmp-qrcode.png' + with open(filename, 'wb') as f: + f.write(qrcode) + os.system(f'xdg-open {filename}') + + +@register_command +async def fetch_twitter_conversations(token: str): + async with ConversationsBot.create() as bot: + print([conversation_id async for conversation_id in bot.fetch_conversations()]) + + +def restarting(func): + @wraps(func) + async def wrapper(*args, **kwargs): + while True: + try: + await func(*args, **kwargs) + except Exception: + delay = 15 + logger.exception("Exception in %s, restarting in %d seconds", func, delay) + await asyncio.sleep(delay) + return wrapper + + +@as_task +@register_command +@restarting +async def run_conversations_bot(): + async with ConversationsBot.create() as bot: + await bot.run_loop() + + +@as_task +@register_command +@restarting +async def run_mentions_bot(): + async with MentionsBot.create() as bot: + me: TwitterAccount = await bot.client.get_me() + await bot.run_loop(me.handle) diff --git a/donate4fun/twitter_provider.py b/donate4fun/twitter_provider.py new file mode 100644 index 00000000..09d192c4 --- /dev/null +++ b/donate4fun/twitter_provider.py @@ -0,0 +1,51 @@ +from datetime import datetime + +from furl import furl +from sqlalchemy.orm.exc import NoResultFound + +from .db import DbSession +from .db_twitter import TwitterDbLib +from .settings import settings +from .social import SocialProvider +from .models import TwitterAccount, Donation, TwitterTweet +from .types import EntityTooOld +from .twitter import make_apponly_client, TwitterApiClient, UnsupportedTwitterUrl + + +class TwitterProvider(SocialProvider): + def wrap_db(self, db_session: DbSession) -> TwitterDbLib: + return TwitterDbLib(db_session) + + async def query_or_fetch_account(self, db: TwitterDbLib, **params) -> TwitterAccount: + try: + account: TwitterAccount = await db.query_account(**params) + if account.last_fetched_at is None or account.last_fetched_at < datetime.utcnow() - settings.twitter.refresh_timeout: + raise EntityTooOld + except (NoResultFound, EntityTooOld): + async with make_apponly_client(token=settings.twitter.linking_oauth.bearer_token) as client: + account: TwitterAccount = await TwitterApiClient(client).get_user_by(**params) + await db.save_account(account) + return account + + def get_account_path(self, account: TwitterAccount) -> str: + return f'/twitter/{account.id}' + + async def apply_target(self, donation: Donation, target: furl, db_session: DbSession): + parts = target.path.segments + if len(parts) in (1, 2): + tweet_id = None + author_handle = parts[0] + elif len(parts) >= 3 and parts[1] == 'status': + tweet_id = int(parts[2]) + author_handle = parts[0] + else: + raise UnsupportedTwitterUrl + + db = TwitterDbLib(db_session) + if tweet_id is not None: + tweet = TwitterTweet(tweet_id=tweet_id) + # FIXME: we should possibly save link to the tweet author + await db.get_or_create_tweet(tweet) + donation.twitter_tweet = tweet + donation.twitter_account = await self.query_or_fetch_account(db=db, handle=author_handle) + donation.lightning_address = donation.twitter_account.lightning_address diff --git a/donate4fun/web.py b/donate4fun/web.py index 80e46114..1aabd32f 100644 --- a/donate4fun/web.py +++ b/donate4fun/web.py @@ -4,24 +4,24 @@ from xml.etree import ElementTree as ET import httpx +from bech32 import bech32_encode from mako.lookup import TemplateLookup from fastapi import Request, Response, FastAPI, Depends from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse from fastapi.exceptions import RequestValidationError +from furl import furl from sqlalchemy.orm.exc import NoResultFound from jwcrypto.jwk import JWK from .api_utils import get_db_session, make_absolute_uri -from .models import YoutubeChannel, TwitterAccount, Donation, Donator, GithubUser -from .youtube import query_or_fetch_youtube_channel -from .twitter import query_or_fetch_twitter_account -from .github import query_or_fetch_github_user +from .models import YoutubeChannel, Donation, Donator, SocialProviderId, SocialAccount +from .social import SocialProvider +from .youtube_provider import YoutubeProvider from .settings import settings from .db import db from .db_models import DonatorDb from .db_youtube import YoutubeDbLib from .db_twitter import TwitterDbLib -from .db_github import GithubDbLib from .db_donations import DonationsDbLib from .lnd import lightning_payment_metadata @@ -114,36 +114,50 @@ async def sitemap(request: Request, db_session=Depends(get_db_session)): @app.get('/d/{channel_id}') async def donate_redirect(request: Request, channel_id: str, db=Depends(get_db_session)): - youtube_channel: YoutubeChannel = await query_or_fetch_youtube_channel(channel_id=channel_id, db=YoutubeDbLib(db)) + youtube_channel: YoutubeChannel = await YoutubeProvider().query_or_fetch_account(channel_id=channel_id, db=YoutubeDbLib(db)) return RedirectResponse(f'{settings.base_url}/donate/{youtube_channel.id}', status_code=302) -@app.get('/tw/{handle}') -async def twitter_account_redirect(request: Request, handle: str, db=Depends(get_db_session)): - account: TwitterAccount = await query_or_fetch_twitter_account(handle=handle, db=TwitterDbLib(db)) - return RedirectResponse(f'{settings.base_url}/twitter/{account.id}', status_code=302) - - -@app.get('/gh/{login}') -async def github_user_redirect(request: Request, login: str, db=Depends(get_db_session)): - user: GithubUser = await query_or_fetch_github_user(login=login, db=GithubDbLib(db)) - return RedirectResponse(f'{settings.base_url}/github/{user.id}', status_code=302) +@app.get('/{provider_slug}/{handle}') +async def social_account_redirect(request: Request, provider_slug: str, handle: str, db=Depends(get_db_session)): + provider: SocialProvider = SocialProvider.from_slug(provider_slug) + account: SocialAccount = await provider.query_or_fetch_account(handle=handle, db=TwitterDbLib(db)) + return RedirectResponse(make_absolute_uri(provider.get_account_path(account)), status_code=302) @app.get('/.well-known/lnurlp/{username}', response_class=JSONResponse) -async def lightning_address(request: Request, username: str, db_session=Depends(get_db_session)): - receiver: Donator = await db_session.find_donator(DonatorDb.lightning_address == f'{username}@{request.headers["host"]}') +@app.get('/.well-known/lnurlp/{provider_id}/{username}', response_class=JSONResponse) +async def lightning_address( + request: Request, username: str, provider_id: SocialProviderId = 'donate4fun', db_session=Depends(get_db_session), +): + if provider_id == 'donate4fun': + account: Donator = await db_session.find_donator(DonatorDb.lightning_address == f'{username}@{request.headers["host"]}') + else: + provider: SocialProvider = SocialProvider.create(provider_id) + account: SocialAccount = await provider.query_or_fetch_account(handle=username) return dict( status='OK', - callback=f'{settings.base_url}/api/v1/lnurl/{receiver.id}/payment-callback', + callback=make_absolute_uri(f'/api/v1/lnurl/{provider_id}/{account.id}/payment-callback'), maxSendable=settings.lnurlp.max_sendable_sats * 1000, minSendable=settings.lnurlp.min_sendable_sats * 1000, - metadata=lightning_payment_metadata(receiver), + metadata=lightning_payment_metadata(account), commentAllowed=255, tag="payRequest", ) +@app.get('/lnurlp/{provider}/{username}') +async def lnurlp_redirect(provider: str, username: str): + url = furl(make_absolute_uri(f'/.well-known/lnurlp/{provider}/{username}'), scheme='lnurlp').url + return RedirectResponse(url, status_code=302) + + +@app.get('/lightning/{provider}/{username}') +async def lightning_redirect(provider: str, username: str): + encoded_url = bech32_encode(make_absolute_uri(f'/.well-known/lnurlp/{provider}/{username}')) + return RedirectResponse(f'lightning:{encoded_url}', status_code=302) + + @app.get("/.well-known/openid-configuration", response_class=JSONResponse) async def openid_configuration(): return dict( diff --git a/donate4fun/youtube.py b/donate4fun/youtube.py index 89ffcc7f..74b9e31d 100644 --- a/donate4fun/youtube.py +++ b/donate4fun/youtube.py @@ -1,21 +1,17 @@ import json import logging from functools import wraps -from dataclasses import dataclass from datetime import datetime -from urllib.parse import parse_qs from aiogoogle import Aiogoogle from aiogoogle.auth.creds import ClientCreds, ServiceAccountCreds from pydantic import BaseModel -from sqlalchemy.orm.exc import NoResultFound from glom import glom from .settings import settings -from .types import UnsupportedTarget, Url, ValidationError, EntityTooOld -from .db import DbSession, Database -from .db_youtube import YoutubeDbLib -from .models import YoutubeVideo, YoutubeChannel, Donation +from .types import Url, ValidationError +from .db import Database +from .models import YoutubeChannel from .core import register_command, app from .api_utils import scrape_lightning_address, make_absolute_uri @@ -24,10 +20,6 @@ logger = logging.getLogger(__name__) -class UnsupportedYoutubeUrl(UnsupportedTarget): - pass - - class YoutubeVideoNotFound(ValidationError): pass @@ -67,49 +59,12 @@ class VideoInfo(BaseModel): default_audio_language: str -@dataclass -class YoutubeDonatee: - channel_id: str | None = None - video_id: str | None = None - handle: str | None = None - - async def fetch(self, donation: Donation, db: DbSession): - youtube_db = YoutubeDbLib(db) - if self.video_id: - donation.youtube_video = await query_or_fetch_youtube_video(video_id=self.video_id, db=youtube_db) - donation.youtube_channel = donation.youtube_video.youtube_channel - elif self.channel_id: - donation.youtube_channel = await query_or_fetch_youtube_channel(channel_id=self.channel_id, db=youtube_db) - elif self.handle: - donation.youtube_channel = await query_or_fetch_youtube_channel(handle=self.handle, db=youtube_db) - else: - raise ValidationError("No YouTube donatee info") - donation.lightning_address = donation.youtube_channel.lightning_address - - -def validate_youtube_url(parsed) -> YoutubeDonatee: - parts = parsed.path.split('/') - if parts[1] == 'watch': - video_id = parse_qs(parsed.query)['v'][0] - return YoutubeDonatee(video_id=video_id) - elif parts[1] == 'shorts': - return YoutubeDonatee(video_id=parts[2]) - elif parts[1] in ('channel', 'c'): - return YoutubeDonatee(handle=parts[2]) - elif parts[1].startswith('@'): - return YoutubeDonatee(handle=parts[1]) - elif parsed.hostname == 'youtu.be': - raise UnsupportedYoutubeUrl("youtu.be urls are not supported") - else: - raise UnsupportedYoutubeUrl("Unrecognized YouTube URL") - - def withyoutube(func): @wraps(func) async def wrapper(*args, **kwargs): async with Aiogoogle(api_key=settings.youtube.api_key) as aiogoogle: youtube_v3 = await aiogoogle.discover("youtube", "v3") - return await func(aiogoogle, youtube_v3, *args, **kwargs) + return await func(*args, aiogoogle=aiogoogle, youtube=youtube_v3, **kwargs) return wrapper @@ -132,54 +87,10 @@ def get_service_account_creds(): ) -async def query_or_fetch_youtube_video(video_id: str, db: YoutubeDbLib) -> YoutubeVideo: - try: - video: YoutubeVideo = await db.query_youtube_video(video_id=video_id) - if should_refresh_channel(video.youtube_channel): - video.youtube_channel = await query_or_fetch_youtube_channel(video.youtube_channel.channel_id, db) - except NoResultFound: - video: YoutubeVideo = await fetch_youtube_video(video_id, db) - await db.save_youtube_video(video) - return video - - -@withyoutube -async def fetch_youtube_video(aiogoogle, youtube, video_id: str, db: DbSession) -> YoutubeVideo: - req = youtube.videos.list(id=video_id, part='snippet') - res = await aiogoogle.as_api_key(req) - items = res['items'] - if not items: - raise YoutubeVideoNotFound - item = items[0] - snippet = item['snippet'] - return YoutubeVideo( - video_id=item['id'], - title=snippet['title'], - thumbnail_url=snippet['thumbnails']['default']['url'], - default_audio_language=snippet.get('defaultAudioLanguage', 'en'), - youtube_channel=await query_or_fetch_youtube_channel(channel_id=snippet['channelId'], db=db), - ) - - def should_refresh_channel(channel: YoutubeChannel): return channel.last_fetched_at is None or channel.last_fetched_at < datetime.utcnow() - settings.youtube.refresh_timeout -async def query_or_fetch_youtube_channel(db: YoutubeDbLib, **params) -> YoutubeChannel: - try: - channel: YoutubeChannel = await db.find_youtube_channel(**params) - if should_refresh_channel(channel): - logger.debug("youtube channel %s is too old, refreshing", channel) - raise EntityTooOld - return channel - except (NoResultFound, EntityTooOld): - channel: YoutubeChannel = ( - await fetch_youtube_channel(**params) if 'channel_id' in params else await search_for_youtube_channel(**params) - ) - await db.save_account(channel) - return channel - - @register_command async def fetch_and_save_youtube_channel(channel_id: str): channel: YoutubeChannel = await fetch_youtube_channel(channel_id) @@ -188,7 +99,7 @@ async def fetch_and_save_youtube_channel(channel_id: str): @withyoutube -async def search_for_youtube_channel(aiogoogle, youtube, handle: str) -> YoutubeChannel: +async def search_for_youtube_channel(handle: str, aiogoogle, youtube) -> YoutubeChannel: """ Until Google updates his YouTube API we ought to search by handle and then list for each result until we find the needed one. """ @@ -207,7 +118,7 @@ async def search_for_youtube_channel(aiogoogle, youtube, handle: str) -> Youtube @register_command @withyoutube -async def fetch_youtube_channel(aiogoogle, youtube, channel_id: str) -> YoutubeChannel: +async def fetch_youtube_channel(channel_id: str, aiogoogle, youtube) -> YoutubeChannel: req = youtube.channels.list(id=channel_id, part='snippet,brandingSettings') res = await aiogoogle.as_api_key(req) total_results = res['pageInfo']['totalResults'] @@ -243,7 +154,7 @@ async def fetch_user_channel(code: str) -> ChannelInfo: @withyoutube -async def find_comment(aiogoogle, youtube, video_id: str, comment: str) -> list[ChannelId]: +async def find_comment(video_id: str, comment: str, aiogoogle, youtube) -> list[ChannelId]: req = youtube.commentThreads.list(part='snippet', videoId=video_id, searchTerms=comment) res = await aiogoogle.as_api_key(req) channels = [] diff --git a/donate4fun/youtube_provider.py b/donate4fun/youtube_provider.py new file mode 100644 index 00000000..dc469d67 --- /dev/null +++ b/donate4fun/youtube_provider.py @@ -0,0 +1,96 @@ +import logging + +from furl import furl +from sqlalchemy.orm.exc import NoResultFound + +from .models import YoutubeChannel, YoutubeVideo, Donation +from .db import DbSession +from .db_youtube import YoutubeDbLib +from .social import SocialProvider +from .types import EntityTooOld, UnsupportedTarget +from .youtube import ( + should_refresh_channel, search_for_youtube_channel, fetch_youtube_channel, + withyoutube, YoutubeVideoNotFound, +) + +logger = logging.getLogger(__name__) + + +class UnsupportedYoutubeUrl(UnsupportedTarget): + pass + + +class YoutubeProvider(SocialProvider): + def wrap_db(self, db_session: DbSession) -> YoutubeDbLib: + return YoutubeDbLib(db_session) + + async def query_or_fetch_account(self, db: YoutubeDbLib, **params) -> YoutubeChannel: + try: + channel: YoutubeChannel = await db.find_youtube_channel(**params) + if should_refresh_channel(channel): + logger.debug("youtube channel %s is too old, refreshing", channel) + raise EntityTooOld + return channel + except (NoResultFound, EntityTooOld): + channel: YoutubeChannel = ( + await fetch_youtube_channel(**params) if 'channel_id' in params else await search_for_youtube_channel(**params) + ) + await db.save_account(channel) + return channel + + async def query_or_fetch_video(self, video_id: str, db: YoutubeDbLib) -> YoutubeVideo: + try: + video: YoutubeVideo = await db.query_youtube_video(video_id=video_id) + if should_refresh_channel(video.youtube_channel): + video.youtube_channel = await YoutubeProvider().query_or_fetch_account(video.youtube_channel.channel_id, db) + except NoResultFound: + video: YoutubeVideo = await self.fetch_video(video_id, db) + await db.save_youtube_video(video) + return video + + @withyoutube + async def fetch_video(self, video_id: str, db: DbSession, aiogoogle, youtube) -> YoutubeVideo: + req = youtube.videos.list(id=video_id, part='snippet') + res = await aiogoogle.as_api_key(req) + items = res['items'] + if not items: + raise YoutubeVideoNotFound + item = items[0] + snippet = item['snippet'] + return YoutubeVideo( + video_id=item['id'], + title=snippet['title'], + thumbnail_url=snippet['thumbnails']['default']['url'], + default_audio_language=snippet.get('defaultAudioLanguage', 'en'), + youtube_channel=await self.query_or_fetch_account(channel_id=snippet['channelId'], db=db), + ) + + async def get_account_path(self, account: YoutubeChannel) -> str: + return f'/youtube/{account.id}' + + async def apply_target(self, donation: Donation, target: furl, db_session: DbSession): + parts = target.path.segments + handle = None + video_id = None + channel_id = None + match parts[0]: + case 'watch': + video_id = target.query.params['v'] + case 'shorts': + video_id = parts[1] + case 'channel' | 'c': + channel_id = parts[1] + case s if s.startswith('@'): + handle = s + case _: + raise UnsupportedYoutubeUrl("Unrecognized YouTube URL") + + youtube_db = self.wrap_db(db_session) + if video_id: + donation.youtube_video = await self.query_or_fetch_video(video_id=video_id, db=youtube_db) + donation.youtube_channel = donation.youtube_video.youtube_channel + elif channel_id: + donation.youtube_channel = await self.query_or_fetch_account(channel_id=channel_id, db=youtube_db) + elif handle: + donation.youtube_channel = await self.query_or_fetch_account(handle=handle, db=youtube_db) + donation.lightning_address = donation.youtube_channel.lightning_address diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 14643ac0..8706aa6a 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -54,12 +54,14 @@ export default defineConfig({ '/api/': httpProxy, '/d/': httpProxy, '/tw/': httpProxy, + '/gh/': httpProxy, '/js/script.js': httpProxy, '/youtube/': httpProxy, '/twitter/': httpProxy, '/donation/': httpProxy, '/preview/': httpProxy, '/.well-known/': httpProxy, + '/lnurlp/': httpProxy, '^/$': httpProxy, }, }, diff --git a/poetry.lock b/poetry.lock index 1a948be2..92d68555 100644 --- a/poetry.lock +++ b/poetry.lock @@ -181,16 +181,23 @@ tests = ["attrs[tests-no-zope]", "zope.interface"] tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] [[package]] -name = "authlib" -version = "1.2.0" +name = "Authlib" +version = "1.1.0" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." category = "main" optional = false python-versions = "*" +develop = false [package.dependencies] cryptography = ">=3.2" +[package.source] +type = "git" +url = "https://github.com/nikicat/authlib" +reference = "HEAD" +resolved_reference = "8001945f9149d6379a1fed27c83a33b0fa62700e" + [[package]] name = "backcall" version = "0.2.0" @@ -1936,7 +1943,7 @@ standard = ["PyYAML (>=5.1)", "colorama (>=0.4)", "httptools (>=0.4.0)", "python name = "vcrpy" version = "4.1.1" description = "Automatically mock your HTTP interactions to simplify and speed up testing" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" develop = false @@ -1951,7 +1958,7 @@ yarl = "*" type = "git" url = "https://github.com/nikicat/vcrpy" reference = "fix-httpx-stub" -resolved_reference = "00a9072e703277a2d6665ace9021f4494b1a9b98" +resolved_reference = "e3eedae31498d264018531e856737c5d82941460" [[package]] name = "watchfiles" @@ -2026,7 +2033,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "c6055e2c017b28b60113fda5e9bfdda691fd866ed6069c26e699a1e082d7e5c7" +content-hash = "2454c0d91070a869ba1671a332930da4a2b44c0d0f7b398434d078acdbcd9470" [metadata.files] aiofiles = [ @@ -2191,10 +2198,7 @@ attrs = [ {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, ] -authlib = [ - {file = "Authlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:4ddf4fd6cfa75c9a460b361d4bd9dac71ffda0be879dbe4292a02e92349ad55a"}, - {file = "Authlib-1.2.0.tar.gz", hash = "sha256:4fa3e80883a5915ef9f5bc28630564bc4ed5b5af39812a3ff130ec76bd631e9d"}, -] +Authlib = [] backcall = [ {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, diff --git a/pyproject.toml b/pyproject.toml index 84ea1a6b..519c2332 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ rollbar = "^0.16.3" pyinstrument = "^4.3.0" alembic = "^1.8.1" furl = "^2.1.3" -authlib = "^1.1.0" +authlib = {git = "https://github.com/nikicat/authlib"} glom = "^22.1.0" playwright = "^1.28.0" async-lru = "^1.0.3" @@ -53,6 +53,8 @@ jwskate = "^0.5.0" sentry-sdk = "^1.13.0" jwcrypto = "^1.4.2" cairosvg = "^2.6.0" +bech32 = "^1.2.0" +vcrpy = {git = "https://github.com/nikicat/vcrpy", rev = "fix-httpx-stub"} [tool.poetry.dev-dependencies] ipython = "^8.3.0" diff --git a/tests/autofixtures/github-account-owned.yaml b/tests/autofixtures/github-account-owned.yaml index 0731dc4d..a7043e7b 100644 --- a/tests/autofixtures/github-account-owned.yaml +++ b/tests/autofixtures/github-account-owned.yaml @@ -1,6 +1,7 @@ json: avatar_url: http://url.com balance: 0 + display_name: title id: 00000000-0000-0000-0000-000000000001 is_my: false last_fetched_at: '2022-02-02T22:22:22' @@ -10,6 +11,7 @@ json: owner_id: null provider: github total_donated: 0 + unique_name: handle user_id: 123 via_oauth: false status_code: 200 diff --git a/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth-relogin.yaml b/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth-relogin.yaml index f8ba56c7..ee378bf1 100644 --- a/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth-relogin.yaml +++ b/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth-relogin.yaml @@ -2,5 +2,5 @@ aud: http://localhost:3000 iss: http://localhost:3000 toasts: - icon: success - message: You've successfully signed in using Nikolay Bryskin Github account + message: You've successfully signed in using nikicat Github account title: Successful sign-in diff --git a/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth.yaml b/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth.yaml index 0ddd860c..2909e46d 100644 --- a/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth.yaml +++ b/tests/autofixtures/oauth-redirect-toasts-github-login-via-oauth.yaml @@ -2,5 +2,5 @@ aud: http://localhost:3000 iss: http://localhost:3000 toasts: - icon: success - message: Github account Nikolay Bryskin was successefully linked + message: Github account nikicat was successefully linked title: Social account is linked diff --git a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth-relogin.yaml b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth-relogin.yaml index d5ad246d..f4d4c367 100644 --- a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth-relogin.yaml +++ b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth-relogin.yaml @@ -2,5 +2,5 @@ aud: http://localhost:3000 iss: http://localhost:3000 toasts: - icon: success - message: You've successfully signed in using @donate4_fun Twitter account + message: You've successfully signed in using @handle Twitter account title: Successful sign-in diff --git a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth.yaml b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth.yaml index b4bf59dd..5b538335 100644 --- a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth.yaml +++ b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth.yaml @@ -2,5 +2,5 @@ aud: http://localhost:3000 iss: http://localhost:3000 toasts: - icon: success - message: Twitter account @donate4_fun was successefully linked + message: Twitter account @handle was successefully linked title: Social account is linked diff --git a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1-relogin.yaml b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1-relogin.yaml index d5ad246d..f4d4c367 100644 --- a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1-relogin.yaml +++ b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1-relogin.yaml @@ -2,5 +2,5 @@ aud: http://localhost:3000 iss: http://localhost:3000 toasts: - icon: success - message: You've successfully signed in using @donate4_fun Twitter account + message: You've successfully signed in using @handle Twitter account title: Successful sign-in diff --git a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1.yaml b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1.yaml index b4bf59dd..5b538335 100644 --- a/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1.yaml +++ b/tests/autofixtures/oauth-redirect-toasts-twitter-login-via-oauth1.yaml @@ -2,5 +2,5 @@ aud: http://localhost:3000 iss: http://localhost:3000 toasts: - icon: success - message: Twitter account @donate4_fun was successefully linked + message: Twitter account @handle was successefully linked title: Social account is linked diff --git a/tests/autofixtures/twitter-account-owned.yaml b/tests/autofixtures/twitter-account-owned.yaml index ecad374e..2d9407cb 100644 --- a/tests/autofixtures/twitter-account-owned.yaml +++ b/tests/autofixtures/twitter-account-owned.yaml @@ -1,15 +1,17 @@ json: balance: 0 + display_name: title handle: handle id: 00000000-0000-0000-0000-000000000001 - is_my: false + is_my: true last_fetched_at: '2022-02-02T22:22:22' lightning_address: null - name: null - owner_id: null + name: title + owner_id: 00000000-0000-0000-0000-000000000000 profile_image_url: null provider: twitter total_donated: 0 + unique_name: '@handle' user_id: 123 - via_oauth: false + via_oauth: true status_code: 200 diff --git a/tests/autofixtures/youtube-channel-owned.yaml b/tests/autofixtures/youtube-channel-owned.yaml index 027ad57e..35d31687 100644 --- a/tests/autofixtures/youtube-channel-owned.yaml +++ b/tests/autofixtures/youtube-channel-owned.yaml @@ -2,6 +2,7 @@ json: balance: 0 banner_url: null channel_id: UCxxx + display_name: title handle: null id: 00000000-0000-0000-0000-000000000001 is_my: true @@ -12,5 +13,6 @@ json: thumbnail_url: null title: title total_donated: 0 + unique_name: '@None' via_oauth: true status_code: 200 diff --git a/tests/cassettes/test_twitter/test_handle_twitter_mention[100-None].yaml b/tests/cassettes/test_twitter/test_handle_twitter_mention[100-None].yaml new file mode 100644 index 00000000..83449cdd --- /dev/null +++ b/tests/cassettes/test_twitter/test_handle_twitter_mention[100-None].yaml @@ -0,0 +1,468 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - secret + connection: + - keep-alive + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: GET + uri: https://api.twitter.com/2/users/1591008723912343553?user.fields=id%2Cname%2Cprofile_image_url%2Cdescription%2Cverified%2Centities + response: + content: '{"data": {"username": "Bryskin2", "verified": false, "name": "Nikolay + Bryskin", "id": "1591008723912343553", "profile_image_url": "https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png", + "description": ""}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '183' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:30 UTC + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587032589662; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:30 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read + x-connection-hash: + - ef82efaf650d60f576b5ec2a13c27454754e0e67c0a0818441809aded54e7efc + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '300' + x-rate-limit-remaining: + - '299' + x-rate-limit-reset: + - '1676836770' + x-response-time: + - '134' + x-transaction-id: + - 70074242276c9831 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - secret + connection: + - keep-alive + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: GET + uri: https://api.twitter.com/2/users/1616757608299339776?user.fields=id%2Cname%2Cprofile_image_url%2Cdescription%2Cverified%2Centities + response: + content: '{"data": {"verified": false, "description": "Bitcoin Lightning tips + - mention me to send a tip. Part of @donate4_fun service.", "id": "1616757608299339776", + "name": "Tip4Fun", "username": "tip4fun", "profile_image_url": "https://pbs.twimg.com/profile_images/1624461699909574663/ko3M0TUK_normal.jpg", + "entities": {"url": {"urls": [{"start": 0, "end": 23, "url": "https://t.co/jkmQZP2YbL", + "expanded_url": "https://donate4.fun", "display_url": "donate4.fun"}]}, "description": + {"mentions": [{"start": 59, "end": 71, "username": "donate4_fun"}]}}}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '336' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:30 UTC + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587066127388; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:30 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read + x-connection-hash: + - b6af3993f33b80d49f21d75c3fd544b86cacaa4ffef08e9ba306eced87a19d66 + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '300' + x-rate-limit-remaining: + - '298' + x-rate-limit-reset: + - '1676836770' + x-response-time: + - '126' + x-transaction-id: + - 0b47aec3c8d1f2bf + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '0' + host: + - upload.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://upload.twitter.com/1.1/media/upload.json?command=INIT&total_bytes=448&media_type=image%2Fpng&media_category=tweet_image + response: + content: '{"media_id": 1627393713390452737, "media_id_string": "1627393713390452737", + "expires_after_secs": 86400, "media_key": "3_1627393713390452737"}' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '111' + content-type: + - application/json;charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:31 GMT + expires: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Sun, 19 Feb 2023 19:44:31 GMT + perf: + - '7626143928' + pragma: + - no-cache + server: + - tsa_o + set-cookie: + - lang=en; Path=/ + - guest_id=v1%3A167683587142729293; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:31 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + status: + - 202 Accepted + strict-transport-security: + - max-age=631138519 + vary: + - Origin + x-access-level: + - read-write-directmessages + x-connection-hash: + - 5a42a5b826b080802cc50c7a5d93c270be5a60afb785d879fdedcdc216ac21e5 + x-frame-options: + - SAMEORIGIN + x-mediaid: + - '1627393713390452737' + x-rate-limit-limit: + - '415' + x-rate-limit-remaining: + - '411' + x-rate-limit-reset: + - '1676837494' + x-response-time: + - '131' + x-transaction: + - c41c2255489b32fd + x-transaction-id: + - c41c2255489b32fd + x-twitter-response-tags: + - BouncerCompliant + x-xss-protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 202 +- request: + body: !!binary | + LS1jMzBlZTdlZGZhMzQ5YjFkZGJiNzQ4YzhlODY1M2VlYg0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJjb21tYW5kIg0KDQpBUFBFTkQNCi0tYzMwZWU3ZWRmYTM0OWIxZGRi + Yjc0OGM4ZTg2NTNlZWINCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ibWVk + aWFfaWQiDQoNCjE2MjczOTM3MTMzOTA0NTI3MzcNCi0tYzMwZWU3ZWRmYTM0OWIxZGRiYjc0OGM4 + ZTg2NTNlZWINCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ic2VnbWVudF9p + bmRleCINCg0KMA0KLS1jMzBlZTdlZGZhMzQ5YjFkZGJiNzQ4YzhlODY1M2VlYg0KQ29udGVudC1E + aXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJtZWRpYSI7IGZpbGVuYW1lPSJpbWFnZSINCkNv + bnRlbnQtVHlwZTogaW1hZ2UvcG5nDQoNColQTkcNChoKAAAADUlIRFIAAAEiAAABIgEAAAAAdcXi + GwAAAYdJREFUeJztmcFtxCAQRf8ESzliKQVsKbizKJ3ZpaSASHC0hPVzALybtaLkwhqvh5vhSf6C + 4TMMQvzdppd/QIBSSimllFKtU5JblzplCKVn2FXXKShHkvQAph4AYEiS/Ek9XtcpqJBjXIYyct0G + 7as/JtVteoKACPX+qNRvFEeYbcq/v65npErcWwI52hchANyuQKvqj03h9kiFIZw3hPOlx5Hk2Kr6 + Y1Mp7q8xTiAib4M9dZ2KWkSGIAIEEY4AMGmeU5dKnsMxJfQRJCOQ3WcdbVX9sans944RcN6Qo81L + AVhS/b4iVfw+9ADsLHCfXRRYD06DlIOgVfXHplbPyal9ye+zBWncV6Ty3K8tuXyyf5+LOjr3dajk + OZK/TARgojjfC2BnUc+pT611TH5cYhkLr1rHrEnh6jTpSgusnqN+X5fa1jHtVyfOvwHTYDTuH0lx + LE4j737Rd6ua1KaO6bgIp8ss5QVrH11noO7rmKQvV1pYao5Zk7qvYwI2Quv3SimllFJPR30DyRHy + jwljaJQAAAAASUVORK5CYIINCi0tYzMwZWU3ZWRmYTM0OWIxZGRiYjc0OGM4ZTg2NTNlZWItLQ0K + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '912' + content-type: + - multipart/form-data; boundary=c30ee7edfa349b1ddbb748c8e8653eeb + cookie: + - guest_id=v1%3A167683587142729293; lang=en + host: + - upload.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://upload.twitter.com/1.1/media/upload.json + response: + content: '' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-security-policy: + - default-src 'self'; connect-src 'self'; font-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com data:; frame-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com; img-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com data:; media-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com; object-src 'none'; script-src + 'self' https://*.twimg.com https://twitter.com https://ton.twitter.com; style-src + 'self' https://*.twimg.com https://twitter.com https://ton.twitter.com; report-uri + https://twitter.com/i/csp_report?a=OBZG6ZTJNRSWE2LSMQ%3D%3D%3D%3D%3D%3D&ro=false; + content-type: + - text/html;charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:31 GMT + expires: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Sun, 19 Feb 2023 19:44:31 GMT + perf: + - '7626143928' + pragma: + - no-cache + server: + - tsa_o + status: + - 204 No Content + strict-transport-security: + - max-age=631138519 + vary: + - Origin + x-access-level: + - read-write-directmessages + x-connection-hash: + - 5a42a5b826b080802cc50c7a5d93c270be5a60afb785d879fdedcdc216ac21e5 + x-frame-options: + - SAMEORIGIN + x-mediaid: + - '1627393713390452737' + x-rate-limit-limit: + - '20000' + x-rate-limit-remaining: + - '19997' + x-rate-limit-reset: + - '1676837494' + x-response-time: + - '138' + x-segmentcount: + - '0' + x-totalbytes: + - '0' + x-transaction: + - 97fdd1a0239bb425 + x-transaction-id: + - 97fdd1a0239bb425 + x-twitter-response-tags: + - BouncerCompliant + x-xss-protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 204 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '0' + cookie: + - guest_id=v1%3A167683587142729293; lang=en + host: + - upload.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://upload.twitter.com/1.1/media/upload.json?command=FINALIZE&media_id=1627393713390452737 + response: + content: '{"media_id": 1627393713390452737, "media_id_string": "1627393713390452737", + "media_key": "3_1627393713390452737", "size": 448, "expires_after_secs": 86400, + "image": {"image_type": "image/png", "w": 290, "h": 290}}' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '149' + content-type: + - application/json;charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:32 GMT + expires: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Sun, 19 Feb 2023 19:44:32 GMT + perf: + - '7626143928' + pragma: + - no-cache + server: + - tsa_o + status: + - 201 Created + strict-transport-security: + - max-age=631138519 + vary: + - Origin + x-access-level: + - read-write-directmessages + x-connection-hash: + - 5a42a5b826b080802cc50c7a5d93c270be5a60afb785d879fdedcdc216ac21e5 + x-frame-options: + - SAMEORIGIN + x-mediaid: + - '1627393713390452737' + x-rate-limit-limit: + - '615' + x-rate-limit-remaining: + - '612' + x-rate-limit-reset: + - '1676837494' + x-response-time: + - '305' + x-transaction: + - 5cd2ca86536bb126 + x-transaction-id: + - 5cd2ca86536bb126 + x-twitter-response-tags: + - BouncerCompliant + x-xss-protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 201 +- request: + body: '{"reply": {"in_reply_to_tweet_id": "1623292724978896896"}, "text": "Invoice: + http://localhost:3000/lnurlp/twitter/tip4fun", "media": {"media_ids": ["1627393713390452737"]}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '172' + content-type: + - application/json + cookie: + - guest_id=v1%3A167683587142729293 + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://api.twitter.com/2/tweets + response: + content: '{"data": {"edit_history_tweet_ids": ["1627393718549467138"], "id": "1627393718549467138", + "text": "@Bryskin2 @donate4_fun Invoice: http://localhost:3000/lnurlp/twitter/tip4fun + https://t.co/7VPO7IzYgi"}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '181' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:32 UTC + location: + - https://api.twitter.com/2/tweets/1627393718549467138 + perf: + - '7626143928' + server: + - tsa_o + strict-transport-security: + - max-age=631138519 + x-access-level: + - read-write-directmessages + x-connection-hash: + - a53756e5a01cce793e21ed201ad9a66b14421647462973f201425c67f65784ad + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '200' + x-rate-limit-remaining: + - '199' + x-rate-limit-reset: + - '1676836772' + x-response-time: + - '362' + x-transaction-id: + - 41187a04f3e04b54 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 201 +version: 1 diff --git a/tests/cassettes/test_twitter/test_handle_twitter_mention[200-100].yaml b/tests/cassettes/test_twitter/test_handle_twitter_mention[200-100].yaml new file mode 100644 index 00000000..cbec956b --- /dev/null +++ b/tests/cassettes/test_twitter/test_handle_twitter_mention[200-100].yaml @@ -0,0 +1,468 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - secret + connection: + - keep-alive + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: GET + uri: https://api.twitter.com/2/users/1591008723912343553?user.fields=id%2Cname%2Cprofile_image_url%2Cdescription%2Cverified%2Centities + response: + content: '{"data": {"description": "", "verified": false, "profile_image_url": + "https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png", + "id": "1591008723912343553", "username": "Bryskin2", "name": "Nikolay Bryskin"}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '182' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:33 UTC + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587357561322; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:33 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read + x-connection-hash: + - e671c977ce744d34700eca273f3db9abc9ce40a0d4528c760ddd31f0441f9fa6 + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '300' + x-rate-limit-remaining: + - '297' + x-rate-limit-reset: + - '1676836770' + x-response-time: + - '127' + x-transaction-id: + - df36471ad1967d71 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - secret + connection: + - keep-alive + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: GET + uri: https://api.twitter.com/2/users/1616757608299339776?user.fields=id%2Cname%2Cprofile_image_url%2Cdescription%2Cverified%2Centities + response: + content: '{"data": {"verified": false, "description": "Bitcoin Lightning tips + - mention me to send a tip. Part of @donate4_fun service.", "id": "1616757608299339776", + "name": "Tip4Fun", "username": "tip4fun", "profile_image_url": "https://pbs.twimg.com/profile_images/1624461699909574663/ko3M0TUK_normal.jpg", + "entities": {"url": {"urls": [{"start": 0, "end": 23, "url": "https://t.co/jkmQZP2YbL", + "expanded_url": "https://donate4.fun", "display_url": "donate4.fun"}]}, "description": + {"mentions": [{"start": 59, "end": 71, "username": "donate4_fun"}]}}}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '336' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:33 UTC + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587392717435; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:33 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read + x-connection-hash: + - 941b19b747d21616789a573486cd9e95276b584c73a8b7fb4a51489d7e161f38 + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '300' + x-rate-limit-remaining: + - '296' + x-rate-limit-reset: + - '1676836770' + x-response-time: + - '125' + x-transaction-id: + - 7edb65d707b375c8 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '0' + host: + - upload.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://upload.twitter.com/1.1/media/upload.json?command=INIT&total_bytes=448&media_type=image%2Fpng&media_category=tweet_image + response: + content: '{"media_id": 1627393725734199297, "media_id_string": "1627393725734199297", + "expires_after_secs": 86399, "media_key": "3_1627393725734199297"}' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '110' + content-type: + - application/json;charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:34 GMT + expires: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Sun, 19 Feb 2023 19:44:34 GMT + perf: + - '7626143928' + pragma: + - no-cache + server: + - tsa_o + set-cookie: + - lang=en; Path=/ + - guest_id=v1%3A167683587437614733; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:34 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + status: + - 202 Accepted + strict-transport-security: + - max-age=631138519 + vary: + - Origin + x-access-level: + - read-write-directmessages + x-connection-hash: + - 2a9d48621a9fa1956f7a8ea13c129d310db0b0439a5f171e9512acd4733d61c0 + x-frame-options: + - SAMEORIGIN + x-mediaid: + - '1627393725734199297' + x-rate-limit-limit: + - '415' + x-rate-limit-remaining: + - '410' + x-rate-limit-reset: + - '1676837494' + x-response-time: + - '130' + x-transaction: + - 819ec13e2b5211db + x-transaction-id: + - 819ec13e2b5211db + x-twitter-response-tags: + - BouncerCompliant + x-xss-protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 202 +- request: + body: !!binary | + LS03Zjc1MDU5NDI3ZTFmMGFlODYyZDY3YmFlZmYzMDYzOA0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJjb21tYW5kIg0KDQpBUFBFTkQNCi0tN2Y3NTA1OTQyN2UxZjBhZTg2 + MmQ2N2JhZWZmMzA2MzgNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ibWVk + aWFfaWQiDQoNCjE2MjczOTM3MjU3MzQxOTkyOTcNCi0tN2Y3NTA1OTQyN2UxZjBhZTg2MmQ2N2Jh + ZWZmMzA2MzgNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ic2VnbWVudF9p + bmRleCINCg0KMA0KLS03Zjc1MDU5NDI3ZTFmMGFlODYyZDY3YmFlZmYzMDYzOA0KQ29udGVudC1E + aXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJtZWRpYSI7IGZpbGVuYW1lPSJpbWFnZSINCkNv + bnRlbnQtVHlwZTogaW1hZ2UvcG5nDQoNColQTkcNChoKAAAADUlIRFIAAAEiAAABIgEAAAAAdcXi + GwAAAYdJREFUeJztmcFtxCAQRf8ESzliKQVsKbizKJ3ZpaSASHC0hPVzALybtaLkwhqvh5vhSf6C + 4TMMQvzdppd/QIBSSimllFKtU5JblzplCKVn2FXXKShHkvQAph4AYEiS/Ek9XtcpqJBjXIYyct0G + 7as/JtVteoKACPX+qNRvFEeYbcq/v65npErcWwI52hchANyuQKvqj03h9kiFIZw3hPOlx5Hk2Kr6 + Y1Mp7q8xTiAib4M9dZ2KWkSGIAIEEY4AMGmeU5dKnsMxJfQRJCOQ3WcdbVX9sans944RcN6Qo81L + AVhS/b4iVfw+9ADsLHCfXRRYD06DlIOgVfXHplbPyal9ye+zBWncV6Ty3K8tuXyyf5+LOjr3dajk + OZK/TARgojjfC2BnUc+pT611TH5cYhkLr1rHrEnh6jTpSgusnqN+X5fa1jHtVyfOvwHTYDTuH0lx + LE4j737Rd6ua1KaO6bgIp8ss5QVrH11noO7rmKQvV1pYao5Zk7qvYwI2Quv3SimllFJPR30DyRHy + jwljaJQAAAAASUVORK5CYIINCi0tN2Y3NTA1OTQyN2UxZjBhZTg2MmQ2N2JhZWZmMzA2MzgtLQ0K + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '912' + content-type: + - multipart/form-data; boundary=7f75059427e1f0ae862d67baeff30638 + cookie: + - guest_id=v1%3A167683587437614733; lang=en + host: + - upload.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://upload.twitter.com/1.1/media/upload.json + response: + content: '' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-security-policy: + - default-src 'self'; connect-src 'self'; font-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com data:; frame-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com; img-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com data:; media-src 'self' https://*.twimg.com + https://twitter.com https://ton.twitter.com; object-src 'none'; script-src + 'self' https://*.twimg.com https://twitter.com https://ton.twitter.com; style-src + 'self' https://*.twimg.com https://twitter.com https://ton.twitter.com; report-uri + https://twitter.com/i/csp_report?a=OBZG6ZTJNRSWE2LSMQ%3D%3D%3D%3D%3D%3D&ro=false; + content-type: + - text/html;charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:34 GMT + expires: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Sun, 19 Feb 2023 19:44:34 GMT + perf: + - '7626143928' + pragma: + - no-cache + server: + - tsa_o + status: + - 204 No Content + strict-transport-security: + - max-age=631138519 + vary: + - Origin + x-access-level: + - read-write-directmessages + x-connection-hash: + - 2a9d48621a9fa1956f7a8ea13c129d310db0b0439a5f171e9512acd4733d61c0 + x-frame-options: + - SAMEORIGIN + x-mediaid: + - '1627393725734199297' + x-rate-limit-limit: + - '20000' + x-rate-limit-remaining: + - '19996' + x-rate-limit-reset: + - '1676837494' + x-response-time: + - '135' + x-segmentcount: + - '0' + x-totalbytes: + - '0' + x-transaction: + - 7dbe420600e4858a + x-transaction-id: + - 7dbe420600e4858a + x-twitter-response-tags: + - BouncerCompliant + x-xss-protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 204 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '0' + cookie: + - guest_id=v1%3A167683587437614733; lang=en + host: + - upload.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://upload.twitter.com/1.1/media/upload.json?command=FINALIZE&media_id=1627393725734199297 + response: + content: '{"media_id": 1627393725734199297, "media_id_string": "1627393725734199297", + "media_key": "3_1627393725734199297", "size": 448, "expires_after_secs": 86400, + "image": {"image_type": "image/png", "w": 290, "h": 290}}' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '151' + content-type: + - application/json;charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:35 GMT + expires: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Sun, 19 Feb 2023 19:44:35 GMT + perf: + - '7626143928' + pragma: + - no-cache + server: + - tsa_o + status: + - 201 Created + strict-transport-security: + - max-age=631138519 + vary: + - Origin + x-access-level: + - read-write-directmessages + x-connection-hash: + - 2a9d48621a9fa1956f7a8ea13c129d310db0b0439a5f171e9512acd4733d61c0 + x-frame-options: + - SAMEORIGIN + x-mediaid: + - '1627393725734199297' + x-rate-limit-limit: + - '615' + x-rate-limit-remaining: + - '611' + x-rate-limit-reset: + - '1676837494' + x-response-time: + - '252' + x-transaction: + - 4ef4783f29b04021 + x-transaction-id: + - 4ef4783f29b04021 + x-twitter-response-tags: + - BouncerCompliant + x-xss-protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 201 +- request: + body: '{"reply": {"in_reply_to_tweet_id": "1623292724978896896"}, "text": "Invoice: + http://localhost:3000/lnurlp/twitter/tip4fun", "media": {"media_ids": ["1627393725734199297"]}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '172' + content-type: + - application/json + cookie: + - guest_id=v1%3A167683587437614733 + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://api.twitter.com/2/tweets + response: + content: '{"data": {"edit_history_tweet_ids": ["1627393729907638273"], "id": "1627393729907638273", + "text": "@Bryskin2 @donate4_fun Invoice: http://localhost:3000/lnurlp/twitter/tip4fun + https://t.co/FIQDjDWkvD"}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '180' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:35 UTC + location: + - https://api.twitter.com/2/tweets/1627393729907638273 + perf: + - '7626143928' + server: + - tsa_o + strict-transport-security: + - max-age=631138519 + x-access-level: + - read-write-directmessages + x-connection-hash: + - 897d0a53038b1aaa4ad865294dc8c5ea1cf1c741afbfc0b2a1f5b48c206d156e + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '200' + x-rate-limit-remaining: + - '198' + x-rate-limit-reset: + - '1676836772' + x-response-time: + - '308' + x-transaction-id: + - fb3dab3dbc2d81f6 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 201 +version: 1 diff --git a/tests/cassettes/test_twitter/test_handle_twitter_mention[300-400].yaml b/tests/cassettes/test_twitter/test_handle_twitter_mention[300-400].yaml new file mode 100644 index 00000000..30e794ff --- /dev/null +++ b/tests/cassettes/test_twitter/test_handle_twitter_mention[300-400].yaml @@ -0,0 +1,210 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - secret + connection: + - keep-alive + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: GET + uri: https://api.twitter.com/2/users/1591008723912343553?user.fields=id%2Cname%2Cprofile_image_url%2Cdescription%2Cverified%2Centities + response: + content: '{"data": {"username": "Bryskin2", "verified": false, "name": "Nikolay + Bryskin", "id": "1591008723912343553", "profile_image_url": "https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png", + "description": ""}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '183' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:36 UTC + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587631943113; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:36 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read + x-connection-hash: + - 86d563595032705bb7b9e20ea94d84f4f3e6ac461070a53910b980103fe666cc + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '300' + x-rate-limit-remaining: + - '295' + x-rate-limit-reset: + - '1676836770' + x-response-time: + - '139' + x-transaction-id: + - 86130e0536e5ad20 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - secret + connection: + - keep-alive + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: GET + uri: https://api.twitter.com/2/users/1616757608299339776?user.fields=id%2Cname%2Cprofile_image_url%2Cdescription%2Cverified%2Centities + response: + content: '{"data": {"verified": false, "id": "1616757608299339776", "description": + "Bitcoin Lightning tips - mention me to send a tip. Part of @donate4_fun service.", + "profile_image_url": "https://pbs.twimg.com/profile_images/1624461699909574663/ko3M0TUK_normal.jpg", + "name": "Tip4Fun", "username": "tip4fun", "entities": {"url": {"urls": [{"start": + 0, "end": 23, "url": "https://t.co/jkmQZP2YbL", "expanded_url": "https://donate4.fun", + "display_url": "donate4.fun"}]}, "description": {"mentions": [{"start": 59, + "end": 71, "username": "donate4_fun"}]}}}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '335' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:36 UTC + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587659109893; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:36 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read + x-connection-hash: + - bfbd4b3dd08d78ec685d40747332bd5db28a1dc276b8fb253b51d082ed9f7c79 + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '300' + x-rate-limit-remaining: + - '294' + x-rate-limit-reset: + - '1676836770' + x-response-time: + - '133' + x-transaction-id: + - 474ed5eb28caf4d3 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"reply": {"in_reply_to_tweet_id": "1623292724978896896"}, "text": "http://localhost:3000/donation/03d9613c-72af-440e-8ea1-bdc5fb5e8c2d"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '137' + content-type: + - application/json + host: + - api.twitter.com + user-agent: + - python-httpx/0.23.2 + method: POST + uri: https://api.twitter.com/2/tweets + response: + content: '{"data": {"edit_history_tweet_ids": ["1627393736819851264"], "id": "1627393736819851264", + "text": "@Bryskin2 @donate4_fun http://localhost:3000/donation/03d9613c-72af-440e-8ea1-bdc5fb5e8c2d"}}' + headers: + api-version: + - '2.61' + cache-control: + - no-cache, no-store, max-age=0 + content-disposition: + - attachment; filename=json.json + content-encoding: + - gzip + content-length: + - '170' + content-type: + - application/json; charset=utf-8 + date: + - Sun, 19 Feb 2023 19:44:37 UTC + location: + - https://api.twitter.com/2/tweets/1627393736819851264 + perf: + - '7626143928' + server: + - tsa_o + set-cookie: + - guest_id=v1%3A167683587700387819; Max-Age=34214400; Expires=Thu, 21 Mar 2024 + 19:44:37 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-access-level: + - read-write-directmessages + x-connection-hash: + - 18907e1506e12b4ca4d7e2be0be1002d302022f87eab80a0df68c7171c7cc4ea + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + x-rate-limit-limit: + - '200' + x-rate-limit-remaining: + - '197' + x-rate-limit-reset: + - '1676836772' + x-response-time: + - '248' + x-transaction-id: + - 06c1323769d6b3d9 + x-xss-protection: + - '0' + http_version: HTTP/1.1 + status_code: 201 +version: 1 diff --git a/tests/fixtures.py b/tests/fixtures.py index 8f8a200a..9bfd7405 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -198,7 +198,7 @@ async def pubsub(db): @pytest.fixture async def app(db, settings, pubsub): posthog.disabled = True - async with create_app(settings) as app, anyio.create_task_group() as tg: + async with create_app() as app, anyio.create_task_group() as tg: lnd = get_alice_lnd() with app_var.assign(app), lnd_var.assign(lnd), pubsub_var.assign(pubsub), task_group.assign(tg): yield app diff --git a/tests/test_twitter.py b/tests/test_twitter.py index 930fe181..edbfda59 100644 --- a/tests/test_twitter.py +++ b/tests/test_twitter.py @@ -10,14 +10,16 @@ DonateRequest, TwitterAccount, TwitterAccountOwned, Donator, TwitterTweet, DonateResponse, DonationPaidRequest, PayInvoiceResult, DonationPaidRouteInfo, Donation, ) -from donate4fun.twitter import query_or_fetch_twitter_account +from donate4fun.twitter_bot import MentionsBot +from donate4fun.twitter_provider import TwitterProvider from donate4fun.lnd import lnd, monitor_invoices_step, PayInvoiceError from donate4fun.types import PaymentRequest, LightningAddress from donate4fun.db import db as db_var from donate4fun.db_twitter import TwitterDbLib from donate4fun.db_donations import DonationsDbLib -from donate4fun.settings import settings +from donate4fun.settings import settings, TwitterOAuth from donate4fun.jobs import refetch_twitter_authors +from donate4fun import twitter_bot from tests.test_util import ( verify_response, mark_vcr, check_notification, login_to, check_response, freeze_time, follow_oauth_flow, load_or_ask, verify_oauth_redirect, @@ -125,6 +127,8 @@ async def app_wrapper(*args): ).dict(), )).json()) donation = donate_response.donation + # Donation should be paid from balance (r_hash is not None) or be requested to be paid from client's wallet + # directly to a lightning address (donate_response.payment_request is not None) assert (donation.r_hash is None) != (donate_response.payment_request is None) if donation.r_hash is not None: assert donation.paid_at != None # noqa @@ -197,7 +201,7 @@ async def test_link_twitter_account(twitter_db): @mark_vcr @pytest.mark.parametrize('params', [dict(handle='donate4_fun'), dict(user_id=12345), dict(handle='twiteis')]) async def test_query_or_fetch_twitter_account(twitter_db, params): - await query_or_fetch_twitter_account(db=twitter_db, **params) + await TwitterProvider().query_or_fetch_account(db=twitter_db, **params) async def test_transfer_from_twitter(client, db, rich_donator, settings): @@ -253,12 +257,12 @@ async def test_login_via_oauth(client, settings, monkeypatch, db, freeze_uuids, monkeypatch.setattr('secrets.token_urlsafe', lambda size: urlsafe_b64encode(b'\x00' * size)) monkeypatch.setattr('secrets.token_bytes', lambda size: b'\x00' * size) - async def patched_fetch_twitter_me(client) -> TwitterAccount: + async def patched_get_me(self) -> TwitterAccount: return account - monkeypatch.setattr('donate4fun.twitter.fetch_twitter_me', patched_fetch_twitter_me) + monkeypatch.setattr('donate4fun.twitter.TwitterApiClient.get_me', patched_get_me) account = TwitterAccount( user_id=123, - title='title', + name='title', handle='handle', description='description', last_fetched_at=datetime.utcnow(), @@ -297,6 +301,7 @@ async def follow_oauth1_flow(client, name: str): headers=dict(referer=settings.base_url), )).json() auth_url: str = response['url'] + # FIXME: rename .code to .link redirect_url: str = load_or_ask(f'{name}.code', f"Open this url {auth_url}, authorize and paste url here:") params = dict(furl(redirect_url).query.params) response = await client.get( @@ -311,12 +316,12 @@ async def test_login_via_oauth1(client, settings, monkeypatch, db, freeze_uuids, monkeypatch.setattr('secrets.token_urlsafe', lambda size: urlsafe_b64encode(b'\x00' * size)) monkeypatch.setattr('secrets.token_bytes', lambda size: b'\x00' * size) - async def patched_fetch_twitter_me(client) -> TwitterAccount: + async def patched_get_me(self) -> TwitterAccount: return account - monkeypatch.setattr('donate4fun.twitter.fetch_twitter_me', patched_fetch_twitter_me) + monkeypatch.setattr('donate4fun.twitter.TwitterApiClient.get_me', patched_get_me) account = TwitterAccount( user_id=123, - title='title', + name='title', handle='handle', description='description', last_fetched_at=time_of_freeze, @@ -369,3 +374,68 @@ async def test_unlink_twitter_account(client, db): await twitter_db.link_account(account, donator, via_oauth=True) login_to(client, settings, donator) check_response(await client.post(f'/api/v1/social/twitter/{account.id}/unlink')) + + +@mark_vcr +@pytest.mark.parametrize('text,balance', [ + ('100', None), + ('200', 100), + ('300', 400), +]) +async def test_handle_twitter_mention(db, app, text: str, balance: int | None, monkeypatch): + mention = { + 'author_id': '1591008723912343553', + 'created_at': '2023-02-08T12:08:39.000Z', + 'edit_history_tweet_ids': ['1623292724978896896'], + 'id': '1623292724978896896', + 'in_reply_to_user_id': '1616757608299339776', + 'referenced_tweets': [{'id': '1622708944996114432', 'type': 'replied_to'}], + 'text': f'@tip4fun @donate4_fun {text}', + } + if balance is not None: + async with db.session() as db_session: + twitter_db = TwitterDbLib(db_session) + account = TwitterAccount( + user_id=int(mention['author_id']), + handle='tip4fun', + ) + await twitter_db.save_account(account) + donator = Donator(id=UUID(int=0), balance=balance) + await db_session.save_donator(donator) + await twitter_db.link_account(account, donator, via_oauth=True) + + orig_make_qr_code = twitter_bot.make_qr_code + monkeypatch.setattr('donate4fun.twitter_bot.make_qr_code', lambda data: orig_make_qr_code(b'123qwe')) + settings.twitter.linking_oauth.bearer_token = 'secret' + + class FixtureTokenLoader: + def __init__(self, token: dict): + self.token = token + + async def load(self) -> dict: + return self.token + + class PatchedMentionsBot(MentionsBot): + oauth1_token = FixtureTokenLoader(dict( + oauth_token="secret", + oauth_token_secret="secret", + )) + oauth2_token = FixtureTokenLoader(dict( + access_token="secret", + token_type="bearer", + expires_at=1676058441, + )) + + @classmethod + @property + def oauth_settings(cls): + return TwitterOAuth( + consumer_key='JqyhuOUhGJNO4GLeb3t3PcySP', + consumer_secret='secret', + # @bryskin2 + client_id='MWlyTVlGbUdXRkY2Q09ncGxkbzc6MTpjaQ', + client_secret='secret', + ) + + async with PatchedMentionsBot.create() as bot: + await bot.handle_mention(mention) diff --git a/tests/test_util.py b/tests/test_util.py index 758e7812..0aee233a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -18,7 +18,8 @@ from vcr.filters import replace_query_parameters from furl import furl -from donate4fun.models import Donator, Credentials, SocialProvider +from donate4fun.models import Donator, Credentials +from donate4fun.social import SocialProvider from donate4fun.db import Notification from donate4fun.settings import Settings, settings from donate4fun.core import to_base64 diff --git a/tests/test_youtube.py b/tests/test_youtube.py index de6633d8..3c08d510 100644 --- a/tests/test_youtube.py +++ b/tests/test_youtube.py @@ -7,7 +7,8 @@ from donate4fun.lnd import LndClient, lnd, monitor_invoices_step from donate4fun.models import Donation, YoutubeChannel, Donator, YoutubeChannelOwned, OAuthState, DonateRequest, DonateResponse from donate4fun.types import PaymentRequest -from donate4fun.youtube import query_or_fetch_youtube_video, ChannelInfo +from donate4fun.youtube import ChannelInfo +from donate4fun.youtube_provider import YoutubeProvider from donate4fun.jobs import refetch_youtube_channels from donate4fun.db_youtube import YoutubeDbLib from donate4fun.db_donations import DonationsDbLib @@ -388,7 +389,7 @@ async def test_query_or_fetch_youtube_video(db_session, video_id): """ One channel has banner, other not """ - await query_or_fetch_youtube_video(video_id='VOG-fFhq7kk', db=YoutubeDbLib(db_session)) + await YoutubeProvider().query_or_fetch_video(video_id='VOG-fFhq7kk', db=YoutubeDbLib(db_session)) @mark_vcr