diff --git a/justfile b/justfile index 2c31feea..60c71b0e 100644 --- a/justfile +++ b/justfile @@ -109,13 +109,18 @@ flask +CMD: #!/bin/bash echo "FLASK_APP: ${FLASK_APP-None}" echo "FLASK_ENV: ${FLASK_ENV-None}" - # op run --env-file='./.env' -- flask {{ CMD }} - # export OP_ACCOUNT="my.1password.com" - # export DIGITAL_MEMBERSHIP_SECRETS_JSON="op://Los Verdes/digital-membership_local_dev_secrets/value" - # export DIGITAL_MEMBERSHIP_SECRETS_JSON="op://Los Verdes/digital-membership_gcp-secrets-manager/value" - # op run -- flask {{ CMD }} flask {{ CMD }} + +secret-flask +CMD: + #!/bin/bash + echo "FLASK_APP: ${FLASK_APP-None}" + echo "FLASK_ENV: ${FLASK_ENV-None}" + export OP_ACCOUNT="my.1password.com" + export DIGITAL_MEMBERSHIP_SECRETS_JSON="op://Los Verdes/digital-membership_local_dev_secrets/value" + # export DIGITAL_MEMBERSHIP_SECRETS_JSON="op://Los Verdes/digital-membership_gcp-secrets-manager/value" + op run -- flask {{ CMD }} + ensure-db-schemas: just flask ensure-db-schemas diff --git a/member_card/commands.py b/member_card/commands.py index 67b9051e..17458d74 100755 --- a/member_card/commands.py +++ b/member_card/commands.py @@ -12,7 +12,7 @@ from member_card.db import db from member_card.gcp import get_bucket, publish_message from member_card.image import generate_card_image -from member_card.minibc import Minibc, parse_subscriptions +from member_card.minibc import Minibc, parse_subscriptions, find_missing_shipping from member_card.models import AnnualMembership, User from member_card.models.membership_card import get_or_create_membership_card from member_card.models.user import add_role_to_user_by_email, edit_user_name @@ -254,6 +254,27 @@ def minibc(): pass +@minibc.command("sync-subscriptions") +def minibc_sync_subscriptions(): + etl_results = worker.sync_minibc_subscriptions_etl( + message=dict(type="cli-sync-minibc-subscriptions"), + ) + logger.info(f"minibc_sync_subscriptions() => {etl_results=}") + + +@minibc.command("find-missing-shipping") +def minibc_cmd_find_missing_shipping(): + minibc_client = Minibc(api_key=app.config["MINIBC_API_KEY"]) + missing_shipping_subs = find_missing_shipping( + minibc_client=minibc_client, + skus=app.config["MINIBC_MEMBERSHIP_SKUS"], + ) + print(f"{len(missing_shipping_subs)}=") + from pprint import pprint + + pprint({sub["id"]: sub["customer"] for sub in missing_shipping_subs}) + + @minibc.command("list-incoming-webhooks") def list_incoming_webhooks(): minibc = Minibc(api_key=app.config["MINIBC_API_KEY"]) diff --git a/member_card/minibc.py b/member_card/minibc.py index c74c367c..abfac703 100644 --- a/member_card/minibc.py +++ b/member_card/minibc.py @@ -1,14 +1,13 @@ import logging -from datetime import timedelta, timezone +from datetime import timezone from time import sleep from typing import TYPE_CHECKING import requests -from dateutil.parser import parse, ParserError +from dateutil.parser import ParserError, parse from member_card.db import db, get_or_update from member_card.models import table_metadata -from member_card.models.user import ensure_user # from member_card.models import MinibcWebhook, table_metadata @@ -162,126 +161,133 @@ def get_profile_by_email(self, email): return self.get(path="profiles/", args=dict(filter=f"email,{email}")) -def insert_order_as_membership(order, skus): - from member_card.models import AnnualMembership - - membership_orders = [] - products = order.get("products", []) - subscription_line_items = [p for p in products if p["sku"] in skus] - ignored_line_items = [p for p in products if p["sku"] not in skus] - logger.debug(f"{ignored_line_items=}") - for subscription_line_item in subscription_line_items: - fulfilled_on = None - if fulfilled_on := order.get("fulfilledOn"): - fulfilled_on = parse(fulfilled_on).replace(tzinfo=timezone.utc) - - customer_email = order["customer"]["email"] - # logger.debug(f"{order=}") - - weird_dates_keys = [ - "created_time", - "last_modified", - "signup_date", - "next_payment_date", - ] - weird_dates = {} - for weird_dates_key in weird_dates_keys: - order[weird_dates_key] = order[weird_dates_key].strip("-") - if order[weird_dates_key] == "0": - weird_dates[weird_dates_key] = None - else: - try: - weird_dates[weird_dates_key] = parse( - order[weird_dates_key] - ).replace(tzinfo=timezone.utc) - except ParserError as err: - logger.warning( - f"Unable to parse {weird_dates_key} for {customer_email}: {err=}" - ) - weird_dates[weird_dates_key] = None - - created_on = weird_dates["signup_date"] - if weird_dates["next_payment_date"] is not None: - created_on = weird_dates["next_payment_date"] - timedelta(days=365) - - logger.debug(f"{weird_dates['next_payment_date']=} => {created_on=}") - membership_kwargs = dict( - order_id=f"minibc_{str(order['id'])}", - order_number=f"minibc_{order['order_id'] or order['id']}_{order['customer']['store_customer_id']}", - channel="minibc", - channel_name="minibc", - billing_address_first_name=order["customer"]["first_name"], - billing_address_last_name=order["customer"]["last_name"], - external_order_reference=order["customer"]["store_customer_id"], - created_on=created_on, - modified_on=weird_dates["last_modified"], - fulfilled_on=fulfilled_on, - customer_email=customer_email, - fulfillment_status=None, - test_mode=False, - line_item_id=subscription_line_item["order_product_id"], - sku=subscription_line_item["sku"], - variant_id=subscription_line_item["name"], - product_id=subscription_line_item["store_product_id"], - product_name=subscription_line_item["name"], +def parse_subscriptions(subscriptions): + logger.info(f"{len(subscriptions)=} retrieved from Minibc...") + + # Insert oldest orders first (so our internal membership ID generally aligns with order IDs...) + subscriptions.reverse() + + # Loop over all the raw order data and do the ETL bits + subscription_objs = [] + from member_card.models import Subscription + + for subscription in subscriptions: + product_name = ",".join([p["name"] for p in subscription["products"]]) + shipping_address = " ".join(subscription["shipping_address"].values()) + subscription_kwargs = dict( + subscription_id=subscription["id"], + order_id=subscription["order_id"], + customer_id=subscription["customer"]["id"], + customer_first_name=subscription["customer"]["first_name"], + customer_last_name=subscription["customer"]["last_name"], + customer_email=subscription["customer"]["email"], + product_name=product_name, + status=subscription["status"], + shipping_address=shipping_address, + signup_date=parse_weird_dates(subscription["signup_date"]), + pause_date=parse_weird_dates(subscription["pause_date"]), + cancellation_date=parse_weird_dates(subscription["cancellation_date"]), + next_payment_date=parse_weird_dates(subscription["next_payment_date"]), + created_time=parse_weird_dates(subscription["created_time"]), + last_modified=parse_weird_dates(subscription["last_modified"]), ) - membership = get_or_update( + subscription_obj = get_or_update( session=db.session, - model=AnnualMembership, - filters=["order_id"], - kwargs=membership_kwargs, + model=Subscription, + filters=["subscription_id"], + kwargs=subscription_kwargs, ) - membership_orders.append(membership) + subscription_objs.append(subscription_obj) + + for subscription_obj in subscription_objs: + db.session.add(subscription_obj) + db.session.commit() + return subscription_objs + + +def find_missing_shipping(minibc_client: Minibc, skus): + start_page_num = 1 + max_pages = 1000 + + missing_shipping_subs = list() + inactive_missing_shipping_subs = list() - membership_user = ensure_user( - email=membership.customer_email, - first_name=membership.billing_address_first_name, - last_name=membership.billing_address_last_name, + last_page_num = start_page_num + end_page_num = start_page_num + max_pages + 1 + + logger.debug( + f"find_missing_shipping() => starting to paginate subscriptions and such: {start_page_num=} {end_page_num=}" + ) + total_subs_num = 0 + total_subs_missing_shipping = 0 + total_inactive_subs_missing_shipping = 0 + for page_num in range(start_page_num, end_page_num): + logger.info(f"Sync at {page_num=}") + subscriptions = minibc_client.search_subscriptions( + product_sku=skus[0], + page_num=page_num, ) - membership_user_id = membership_user.id - if not membership.user_id: + if subscriptions is None: logger.debug( - f"No user_id set for {membership=}! Setting to: {membership_user_id=}" + f"find_missing_shipping() => {last_page_num=} returned no results!. Setting `last_page_num` back to 1" ) - setattr(membership, "user_id", membership_user_id) - return membership_orders + last_page_num = 1 + break + last_page_num = page_num + for subscription in subscriptions: + if subscription["shipping_address"]["street_1"] == "": + logger.debug( + f"{subscription['customer']['email']} has no shipping address set!" + ) + if subscription["status"] == "inactive": + inactive_missing_shipping_subs.append(subscriptions) + missing_shipping_subs.append(subscription) -def parse_subscriptions(skus, subscriptions): - logger.info(f"{len(subscriptions)=} retrieved from Minibc...") + logger.debug( + f"find_missing_shipping() => after {page_num=} sleeping for 1 second..." + ) + total_subs_num += len(subscriptions) + total_subs_missing_shipping = len(missing_shipping_subs) + total_inactive_subs_missing_shipping = len(inactive_missing_shipping_subs) + logger.debug( + f"{total_subs_num=}:: {total_subs_missing_shipping=} ({total_inactive_subs_missing_shipping=})" + ) + sleep(1) - # Insert oldest orders first (so our internal membership ID generally aligns with order IDs...) - subscriptions.reverse() + logger.debug( + f"{total_subs_num=}:: {total_subs_missing_shipping=} ({total_inactive_subs_missing_shipping=})" + ) + return missing_shipping_subs - # Loop over all the raw order data and do the ETL bits - memberships = [] - for subscription in subscriptions: - membership_orders = insert_order_as_membership( - order=subscription, - skus=skus, - ) - for membership_order in membership_orders: - db.session.add(membership_order) - db.session.commit() - memberships += membership_orders - return memberships +def parse_weird_dates(date_str): + date_str = date_str.strip("-") + if date_str == "0": + return None + + try: + return parse(date_str).replace(tzinfo=timezone.utc) + except ParserError as err: + logger.warning(f"Unable to parse {date_str}: {err=}") + return None -def minibc_orders_etl(minibc_client: Minibc, skus, load_all): - from member_card import models - # etl_start_time = datetime.now(tz=ZoneInfo("UTC")) +def minibc_subscriptions_etl(minibc_client: Minibc, skus, load_all=False): + from member_card import models - membership_table_name = models.AnnualMembership.__tablename__ + subscriptions_table_name = models.Subscription.__tablename__ if load_all: start_page_num = 1 max_pages = 1000 else: - start_page_num = table_metadata.get_last_run_start_page(membership_table_name) + start_page_num = table_metadata.get_last_run_start_page( + subscriptions_table_name + ) max_pages = 20 - memberships = list() + subscription_objs = list() last_page_num = start_page_num end_page_num = start_page_num + max_pages + 1 @@ -304,19 +310,19 @@ def minibc_orders_etl(minibc_client: Minibc, skus, load_all): break last_page_num = page_num - memberships += parse_subscriptions(skus, subscriptions) + subscription_objs += parse_subscriptions(subscriptions) logger.debug(f"after {page_num=} sleeping for 1 second...") sleep(1) if not load_all: logger.debug( - f"Setting start_page_num metadata on {membership_table_name=} to {last_page_num=}" + f"Setting start_page_num metadata on {subscriptions_table_name=} to {last_page_num=}" ) table_metadata.set_last_run_start_page( - membership_table_name, max(1, last_page_num - 1) + subscriptions_table_name, max(1, last_page_num - 1) ) - return memberships + return subscription_objs def load_single_subscription(minibc_client: Minibc, skus, order_id): diff --git a/member_card/models/__init__.py b/member_card/models/__init__.py index 663cc571..d2eba4cc 100644 --- a/member_card/models/__init__.py +++ b/member_card/models/__init__.py @@ -1,14 +1,16 @@ # from member_card.db import Model # Base, Table +from social_flask_sqlalchemy import models + from member_card.models.annual_membership import AnnualMembership from member_card.models.apple_device_registration import AppleDeviceRegistration from member_card.models.membership_card import MembershipCard from member_card.models.slack_user import SlackUser from member_card.models.squarespace_webhook import SquarespaceWebhook -from member_card.models.table_metadata import TableMetadata -from member_card.models.user import Role, User from member_card.models.store import Store from member_card.models.store_user import StoreUser -from social_flask_sqlalchemy import models +from member_card.models.subscription import Subscription +from member_card.models.table_metadata import TableMetadata +from member_card.models.user import Role, User __all__ = ( "AnnualMembership", @@ -20,6 +22,7 @@ "SquarespaceWebhook", "Store", "StoreUser", + "Subscription", "TableMetadata", "models", ) diff --git a/member_card/models/subscription.py b/member_card/models/subscription.py new file mode 100644 index 00000000..93df9aef --- /dev/null +++ b/member_card/models/subscription.py @@ -0,0 +1,26 @@ +import logging + +from member_card.db import db + +logger = logging.getLogger(__name__) + + +class Subscription(db.Model): + __tablename__ = "subscription" + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + subscription_id = db.Column(db.Integer) + order_id = db.Column(db.Integer) + customer_id = db.Column(db.String(32)) + customer_first_name = db.Column(db.String(64)) + customer_last_name = db.Column(db.String(64)) + customer_email = db.Column(db.String(120)) + product_name = db.Column(db.String(200)) + status = db.Column(db.String(12)) + shipping_address = db.Column(db.Text()) + signup_date = db.Column(db.DateTime) + pause_date = db.Column(db.DateTime) + cancellation_date = db.Column(db.DateTime) + next_payment_date = db.Column(db.DateTime) + created_time = db.Column(db.DateTime) + last_modified = db.Column(db.DateTime) diff --git a/member_card/worker.py b/member_card/worker.py index 4dcd81c4..9b73cc50 100644 --- a/member_card/worker.py +++ b/member_card/worker.py @@ -4,6 +4,7 @@ from flask import Blueprint, current_app, request +from member_card import minibc from member_card import bigcommerce, slack from member_card.db import db from member_card.image import ensure_uploaded_card_image @@ -224,6 +225,26 @@ def sync_customers_etl(message): ) +def sync_minibc_subscriptions_etl(message): + log_extra = dict(pubsub_message=message) + logger.debug( + f"sync_minibc_subscriptions_etl(): Processing message: {message}", + extra=log_extra, + ) + + minibc_client = minibc.Minibc(api_key=current_app.config["MINIBC_API_KEY"]) + membership_skus = current_app.config["BIGCOMMERCE_MEMBERSHIP_SKUS"] + + etl_result = minibc.minibc_subscriptions_etl( + minibc_client=minibc_client, + skus=membership_skus, + ) + logger.debug( + f"sync_minibc_subscriptions_etl(): {len(etl_result)=}", + extra=log_extra, + ) + + @worker_bp.route("/pubsub", methods=["POST"]) def pubsub_ingress(): try: @@ -234,6 +255,7 @@ def pubsub_ingress(): MESSAGE_TYPE_HANDLERS = { "email_distribution_request": process_email_distribution_request, "sync_subscriptions_etl": sync_subscriptions_etl, + "sync_minibc_subscriptions_etl": sync_minibc_subscriptions_etl, "sync_customers_etl": sync_customers_etl, "sync_squarespace_order": sync_squarespace_order, "sync_bigcommerce_order": sync_bigcommerce_order, diff --git a/migrations/versions/897b8492d02b_add_subscription_model.py b/migrations/versions/897b8492d02b_add_subscription_model.py new file mode 100644 index 00000000..633bb0b6 --- /dev/null +++ b/migrations/versions/897b8492d02b_add_subscription_model.py @@ -0,0 +1,51 @@ +"""Add subscription model + +Revision ID: 897b8492d02b +Revises: 2d15cb59b42d +Create Date: 2024-03-08 13:57:12.431748 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "897b8492d02b" +down_revision = "2d15cb59b42d" +branch_labels = None +depends_on = None + + +def upgrade(): + # jscpd:ignore-start + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "subscription", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("subscription_id", sa.Integer(), nullable=True), + sa.Column("order_id", sa.Integer(), nullable=True), + sa.Column("customer_id", sa.String(length=32), nullable=True), + sa.Column("customer_first_name", sa.String(length=64), nullable=True), + sa.Column("customer_last_name", sa.String(length=64), nullable=True), + sa.Column("customer_email", sa.String(length=120), nullable=True), + sa.Column("product_name", sa.String(length=200), nullable=True), + sa.Column("status", sa.String(length=12), nullable=True), + sa.Column("shipping_address", sa.Text(), nullable=True), + sa.Column("signup_date", sa.DateTime(), nullable=True), + sa.Column("pause_date", sa.DateTime(), nullable=True), + sa.Column("cancellation_date", sa.DateTime(), nullable=True), + sa.Column("next_payment_date", sa.DateTime(), nullable=True), + sa.Column("created_time", sa.DateTime(), nullable=True), + sa.Column("last_modified", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + # jscpd:ignore-end + sql = 'REASSIGN OWNED BY current_user TO "read_write"' + op.execute(sql) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("subscription") + # ### end Alembic commands ### diff --git a/terraform/scheduler.tf b/terraform/scheduler.tf index cb042b29..6792dcb2 100644 --- a/terraform/scheduler.tf +++ b/terraform/scheduler.tf @@ -21,7 +21,14 @@ locals { data = { type = "sync_subscriptions_etl", } - + } + sync_minibc_subscriptions_etl = { + description = "Regularly recurring MiniBC subscriptions into membership database ETL task" + schedule = "30 */12 * * *" + paused = true + data = { + type = "sync_minibc_subscriptions_etl", + } } } } diff --git a/tests/test_commands.py b/tests/test_commands.py index b9db7c6a..09f5c931 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,7 +3,7 @@ from member_card.db import db from flask import url_for -from member_card.commands import bigcomm +from member_card.commands import bigcomm, minibc if TYPE_CHECKING: from flask import Flask @@ -396,3 +396,35 @@ def test_ensure_widget_placement( mock_bigcomm_client.get_all_widgets.assert_called_once() mock_bigcomm_client.create_a_placement.assert_called() assert result.exit_code == 0 + + +class TestMinibcCommands: + def test_sync_subscriptions( + self, runner: "FlaskCliRunner", mocker: "MockerFixture" + ): + mock_sync_subs = mocker.patch( + "member_card.commands.worker" + ).sync_minibc_subscriptions_etl + + result = runner.invoke( + cli=minibc, + args=["sync-subscriptions"], + ) + + assert result.exit_code == 0 + + mock_sync_subs.assert_called_once_with( + message=dict(type="cli-sync-minibc-subscriptions"), + ) + + def test_minibc_cmd_find_missing_shipping( + self, app: "Flask", runner: "FlaskCliRunner", mocker: "MockerFixture" + ): + _ = app.config["BIGCOMMERCE_MEMBERSHIP_SKUS"] + _ = mocker.patch("member_card.commands.Minibc") + _ = mocker.patch("member_card.commands.find_missing_shipping") + result = runner.invoke( + cli=minibc, + args=["find-missing-shipping"], + ) + assert result.exit_code == 0 diff --git a/tests/test_minibc.py b/tests/test_minibc.py new file mode 100644 index 00000000..25e5e0e7 --- /dev/null +++ b/tests/test_minibc.py @@ -0,0 +1,92 @@ +from typing import TYPE_CHECKING + +import pytest + +from member_card import minibc + +if TYPE_CHECKING: + from flask import Flask + + +@pytest.fixture() +def mock_subscriptions(): + return [ + { + "id": 1022, + "order_id": 500001, + "customer": { + "id": 501, + "first_name": "John", + "last_name": "Doe", + "email": "john.doe@example.com", + }, + "payment_method": { + "id": 13741, + "method": "credit_card", + "credit_card": { + "type": "Visa", + "last_digits": "string", + "expiry_month": 7, + "expiry_year": 2024, + }, + "billing_address": { + "first_name": "John", + "last_name": "Jones", + "street_1": "200 Yorkland Blvd.", + "street_2": "Unit 210", + "city": "Toronto", + "state": "Ontario", + "country_iso2": "CA", + "zip": "M2J5C1", + }, + "config_id": 122, + }, + "products": [ + { + "store_product_id": 4002, + "order_product_id": 12901, + "name": "T-Shirt of the Month", + "sku": "TEST-001", + "options": [ + { + "option_id": 43, + "value": "112", + "display_name": "Size", + "display_value": "M", + } + ], + "quantity": 2, + "price": 6.49, + "total": 12.98, + } + ], + "periodicity": {"frequency": 1, "unit": "month"}, + "subtotal": 12.98, + "shipping_cost": 5, + "total": 17.98, + "status": "active", + "shipping_address": { + "street_1": "200 Yorkland Blvd.", + "street_2": "Unit 210", + "city": "Toronto", + "state": "Ontario", + "country_iso2": "CA", + "zip": "M2J5C1,", + "shipping_method": "Free Shipping", + }, + "signup_date": "string", + "pause_date": "string", + "cancellation_date": "string", + "next_payment_date": "2022-05-12", + "created_time": "string", + "last_modified": "string", + } + ] + + +def test_parse_subscriptions(app: "Flask", mock_subscriptions, mocker): + with app.app_context(): + returned_subscriptions = minibc.parse_subscriptions( + subscriptions=mock_subscriptions + ) + assert len(returned_subscriptions) == 1 diff --git a/tests/test_worker.py b/tests/test_worker.py index f16aafc4..4be007f7 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -355,3 +355,17 @@ def test_worker_run_slack_members_etl(mocker): assert return_value mock_slack.slack_members_etl.assert_called_once() + + +def test_sync_minibc_subscriptions_etl(app: "Flask", mocker): + mock_minibc = mocker.patch("member_card.worker.minibc") + test_message = dict( + type="sync_minibc_subscriptions_etl", + ) + + with app.app_context(): + worker.sync_minibc_subscriptions_etl( + message=test_message, + ) + + mock_minibc.minibc_subscriptions_etl.assert_called_once()