diff --git a/.github/workflows/deploy_process_file.yml b/.github/workflows/deploy_process_file.yml index 1fdda1e..4987e46 100644 --- a/.github/workflows/deploy_process_file.yml +++ b/.github/workflows/deploy_process_file.yml @@ -32,7 +32,7 @@ jobs: entry_point: "run" source_dir: "process_file" memory_mb: 4096 - secret_environment_variables: "MYSQL_CONFIG=projects/greg-finley/secrets/MYSQL_CONFIG/versions/latest" + secret_environment_variables: "NEON_DATABASE_URL=projects/greg-finley/secrets/NEON_DATABASE_URL/versions/latest" timeout: 540 event_trigger_type: "providers/cloud.pubsub/eventTypes/topic.publish" event_trigger_resource: "projects/greg-finley/topics/dropbox-backup" diff --git a/.github/workflows/deploy_queue_files.yml b/.github/workflows/deploy_queue_files.yml index b92133d..8f03e74 100644 --- a/.github/workflows/deploy_queue_files.yml +++ b/.github/workflows/deploy_queue_files.yml @@ -32,7 +32,7 @@ jobs: entry_point: "run" source_dir: "queue_files" memory_mb: 256 - secret_environment_variables: "MYSQL_CONFIG=projects/greg-finley/secrets/MYSQL_CONFIG/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest" + secret_environment_variables: "NEON_DATABASE_URL=projects/greg-finley/secrets/NEON_DATABASE_URL/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest" timeout: 540 event_trigger_type: "providers/cloud.pubsub/eventTypes/topic.publish" event_trigger_resource: "projects/greg-finley/topics/dropbox-queue-files" diff --git a/.github/workflows/deploy_requeue_failed_files.yml b/.github/workflows/deploy_requeue_failed_files.yml index b4772de..8e5c669 100644 --- a/.github/workflows/deploy_requeue_failed_files.yml +++ b/.github/workflows/deploy_requeue_failed_files.yml @@ -32,7 +32,7 @@ jobs: entry_point: "run" source_dir: "requeue_failed_files" memory_mb: 256 - secret_environment_variables: "MYSQL_CONFIG=projects/greg-finley/secrets/MYSQL_CONFIG/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest" + secret_environment_variables: "NEON_DATABASE_URL=projects/greg-finley/secrets/NEON_DATABASE_URL/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest" timeout: 540 min_instances: 0 max_instances: 1 diff --git a/ddl.sql b/ddl.sql index 467cdc5..6ada1d2 100644 --- a/ddl.sql +++ b/ddl.sql @@ -1,21 +1,21 @@ -CREATE TABLE "dropbox" ( - "id" int NOT NULL AUTO_INCREMENT, - "desktop_path" varchar(300) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - "status" enum('pending','done','deleted') DEFAULT 'done', - "filename" varchar(300) NOT NULL, - PRIMARY KEY ("id"), - UNIQUE KEY "desktop_path" ("desktop_path"), - KEY "filename_index" ("filename"), - KEY "status_index" ("status"), - KEY "created_at_desc_idx" ("created_at" DESC) -) +CREATE TYPE status_type AS ENUM ('pending', 'done', 'deleted'); -CREATE TABLE "dropbox_cursors" ( - "id" int NOT NULL AUTO_INCREMENT, - "dropbox_cursor" varchar(500) NOT NULL, - "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY ("id"), - UNIQUE KEY "dropbox_cursor" ("dropbox_cursor"), - KEY "idx_created_at_desc" ("created_at" DESC) -) +CREATE TABLE dropbox ( + id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + desktop_path VARCHAR(300) NOT NULL UNIQUE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + status status_type DEFAULT 'done', + filename VARCHAR(300) NOT NULL +); + +CREATE INDEX filename_index ON dropbox (filename); +CREATE INDEX status_index ON dropbox (status); +CREATE INDEX created_at_desc_idx ON dropbox (created_at DESC); + +CREATE TABLE dropbox_cursors ( + id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + dropbox_cursor VARCHAR(500) NOT NULL UNIQUE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_created_at_desc ON dropbox_cursors (created_at DESC); diff --git a/process_file/main.py b/process_file/main.py index fc69b39..556fe54 100644 --- a/process_file/main.py +++ b/process_file/main.py @@ -1,27 +1,15 @@ import base64 -import json import os import dropbox -import mysql.connector +import psycopg from google.cloud import secretmanager, storage gcs_client = storage.Client() gcs_bucket = gcs_client.get_bucket("greg-finley-dropbox-backup") -mysql_config_str = os.environ["MYSQL_CONFIG"] -mysql_config_dict = json.loads(mysql_config_str) secret_client = secretmanager.SecretManagerServiceClient() -mysql_connection = mysql.connector.connect( - unix_socket=mysql_config_dict["MYSQL_SOCKET"], - user=mysql_config_dict["MYSQL_USERNAME"], - passwd=mysql_config_dict["MYSQL_PASSWORD"], - database=mysql_config_dict["MYSQL_DATABASE"], -) -mysql_connection.autocommit = True - - def run(event, context): dropbox_access_token = secret_client.access_secret_version( name="projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest" @@ -40,7 +28,7 @@ def run(event, context): VALUES (%s, SUBSTRING_INDEX(%s, '/', -1), 'done') ON DUPLICATE KEY UPDATE status = 'done' """ - cursor = mysql_connection.cursor() - cursor.execute(query, (filename, filename)) - cursor.close() + with psycopg.connect(os.environ["NEON_DATABASE_URL"]) as conn: + with conn.cursor() as cursor: + cursor.execute(query, (filename, filename)) print(f"Uploaded {filename} - {content_type}") diff --git a/process_file/poetry.lock b/process_file/poetry.lock index 52ce8f7..6584669 100644 --- a/process_file/poetry.lock +++ b/process_file/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "cachetools" version = "5.3.0" description = "Extensible memoizing collections and decorators" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -16,7 +15,6 @@ files = [ name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -28,7 +26,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -113,7 +110,6 @@ files = [ name = "dropbox" version = "11.36.0" description = "Official Dropbox API Client" -category = "main" optional = false python-versions = "*" files = [ @@ -131,7 +127,6 @@ stone = ">=2" name = "google-api-core" version = "1.34.0" description = "Google API client core library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -156,7 +151,6 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-api-core" version = "2.11.0" description = "Google API client core library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -169,7 +163,7 @@ google-auth = ">=2.14.1,<3.0dev" googleapis-common-protos = ">=1.56.2,<2.0dev" grpcio = [ {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\""}, + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] grpcio-status = {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""} protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -184,7 +178,6 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-auth" version = "2.18.1" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ @@ -210,7 +203,6 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"] name = "google-cloud-core" version = "2.3.2" description = "Google Cloud API client core library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -219,7 +211,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] @@ -229,7 +221,6 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] name = "google-cloud-pubsub" version = "2.17.1" description = "Google Cloud Pub/Sub API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -238,7 +229,7 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.0,<2.0.0 || >=2.11.0,<3.0.0dev", extras = ["grpc"]} +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" grpcio = ">=1.51.3,<2.0dev" grpcio-status = ">=1.33.2" @@ -255,7 +246,6 @@ libcst = ["libcst (>=0.3.10)"] name = "google-cloud-secret-manager" version = "2.16.1" description = "Google Cloud Secret Manager API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -264,7 +254,7 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.0,<2.0.0 || >=2.11.0,<3.0.0dev", extras = ["grpc"]} +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = [ {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, @@ -276,7 +266,6 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4 name = "google-cloud-storage" version = "2.9.0" description = "Google Cloud Storage API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -285,7 +274,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-resumable-media = ">=2.3.2" @@ -298,7 +287,6 @@ protobuf = ["protobuf (<5.0.0dev)"] name = "google-crc32c" version = "1.5.0" description = "A python wrapper of the C library 'Google CRC32C'" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -379,7 +367,6 @@ testing = ["pytest"] name = "google-resumable-media" version = "2.5.0" description = "Utilities for Google Media Downloads and Resumable Uploads" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -398,7 +385,6 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] name = "googleapis-common-protos" version = "1.59.0" description = "Common protobufs used in Google APIs" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -417,7 +403,6 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] name = "grpc-google-iam-v1" version = "0.12.6" description = "IAM API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -434,7 +419,6 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 name = "grpcio" version = "1.54.2" description = "HTTP/2-based RPC framework" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -492,7 +476,6 @@ protobuf = ["grpcio-tools (>=1.54.2)"] name = "grpcio-status" version = "1.48.2" description = "Status proto mapping for gRPC" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -509,7 +492,6 @@ protobuf = ">=3.12.0" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -517,54 +499,10 @@ files = [ {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] -[[package]] -name = "mysql-connector-python" -version = "8.0.33" -description = "MySQL driver written in Python" -category = "main" -optional = false -python-versions = "*" -files = [ - {file = "mysql-connector-python-8.0.33.tar.gz", hash = "sha256:9775331fa60b5d5a6925781d77eee4384e2b54a12dea694ffdefd1cf1a9c0fdb"}, - {file = "mysql_connector_python-8.0.33-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:241483065ad062256985e082e3cbb3e7d1d6d2275cee17c66d22525b09096201"}, - {file = "mysql_connector_python-8.0.33-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:9f5eb33e29742c5f8ef23df2d3f0de0e46f4325e4324016e15aba7f8665a68c0"}, - {file = "mysql_connector_python-8.0.33-cp310-cp310-manylinux1_i686.whl", hash = "sha256:4c82fb70f44f2469c0879434c1d8ee3162f56a40cc8f5ca1cc4d97f06c84cd43"}, - {file = "mysql_connector_python-8.0.33-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:016662c6252f2c5f47805d9168187be1316d0c1d7109f9fe668482c3d6e5711d"}, - {file = "mysql_connector_python-8.0.33-cp310-cp310-win_amd64.whl", hash = "sha256:46ff8a10c13f39996d60f45c30cf2ea15e883bc71d58259ed2fea0a5a6fb93a3"}, - {file = "mysql_connector_python-8.0.33-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:db422b19347c5d00e078dd64e281e5b1e5a19a2d972dc2d9733b136d79c34798"}, - {file = "mysql_connector_python-8.0.33-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:37eace5b7eb676a41ff1edc5cf6ce4ae1c28406d1a6fe84941e6aa396d688195"}, - {file = "mysql_connector_python-8.0.33-cp311-cp311-manylinux1_i686.whl", hash = "sha256:753d07fb39a67f7f35fe6e6a4fac12008287661de59f9d5c0bf4da3359d83eb8"}, - {file = "mysql_connector_python-8.0.33-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:984f5649e6abee04461d6f52fbc77387d7137b8fd003c54bac66505006f17183"}, - {file = "mysql_connector_python-8.0.33-cp311-cp311-win_amd64.whl", hash = "sha256:f324233af7ec9fcb19c23096af27662459708c0465886cb017d78ff3f5b78b55"}, - {file = "mysql_connector_python-8.0.33-cp37-cp37m-macosx_12_0_x86_64.whl", hash = "sha256:41db4452a99ee28494313eab1aa7749475d3e39bed7b24a0868aee45bd0d9c73"}, - {file = "mysql_connector_python-8.0.33-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:34d5c5f6ec7c1e75bf972def40d097138e097dc694e36dec89a5dd604ef7aada"}, - {file = "mysql_connector_python-8.0.33-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a632d7b0e569a46e6d44e6cd3f8db747995a787a081870697dbfd3ae18949339"}, - {file = "mysql_connector_python-8.0.33-cp37-cp37m-win_amd64.whl", hash = "sha256:e853e12c00e3beabc581f4e039222708ee606fef80a3bac6b1f497ed89a31aea"}, - {file = "mysql_connector_python-8.0.33-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:96f7fb0ccfe96e6e478e5f0f034c99bda961b99ffa1c746cee39cfea45b0c04d"}, - {file = "mysql_connector_python-8.0.33-cp38-cp38-manylinux1_i686.whl", hash = "sha256:7318f416b9defe84b2bd025304bab62b68f8d8fcbe479af5593161eff12ef169"}, - {file = "mysql_connector_python-8.0.33-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0004426e964856148e1cde31e9b8be63ae3013715b048ff0f2ede69a6ddd36f7"}, - {file = "mysql_connector_python-8.0.33-cp38-cp38-win_amd64.whl", hash = "sha256:d8167868ebad8d78ba69babd028626e96a51365cab76edf735b2559731759b62"}, - {file = "mysql_connector_python-8.0.33-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:f403ff22d3514d08028590fef463d17dc107ac72ea27a49429614949d82fda40"}, - {file = "mysql_connector_python-8.0.33-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:af9feec311d8ea51261e1ef1f959a442708e30f0024d08d0fb537b07a1271634"}, - {file = "mysql_connector_python-8.0.33-cp39-cp39-manylinux1_i686.whl", hash = "sha256:7266d7b2550f9fe0cdcea1647aa6aade352e14095042b6a3921c9152cf8543e8"}, - {file = "mysql_connector_python-8.0.33-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:ea05590cb972b114efa027c343b4b7110d8e8450493984ebfb9a651e27674636"}, - {file = "mysql_connector_python-8.0.33-cp39-cp39-win_amd64.whl", hash = "sha256:3bedf8265fb31698e4144ca54e241e3386802c3a437745b1a536a74cbe7e4fb9"}, - {file = "mysql_connector_python-8.0.33-py2.py3-none-any.whl", hash = "sha256:c20a85a69af41d2d7d5cf52106f0b9473775819d189487c6ff3d3f3946931ca2"}, -] - -[package.dependencies] -protobuf = ">=3.11.0,<=3.20.3" - -[package.extras] -compression = ["lz4 (>=2.1.6,<=4.3.2)", "zstandard (>=0.12.0,<=0.19.0)"] -dns-srv = ["dnspython (>=1.16.0,<=2.3.0)"] -gssapi = ["gssapi (>=1.6.9,<=1.8.2)"] - [[package]] name = "ply" version = "3.11" description = "Python Lex & Yacc" -category = "main" optional = false python-versions = "*" files = [ @@ -576,7 +514,6 @@ files = [ name = "proto-plus" version = "1.22.2" description = "Beautiful, Pythonic protocol buffers." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -594,7 +531,6 @@ testing = ["google-api-core[grpc] (>=1.31.5)"] name = "protobuf" version = "3.20.3" description = "Protocol Buffers" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -622,11 +558,33 @@ files = [ {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"}, ] +[[package]] +name = "psycopg" +version = "3.1.18" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg-3.1.18-py3-none-any.whl", hash = "sha256:4d5a0a5a8590906daa58ebd5f3cfc34091377354a1acced269dd10faf55da60e"}, + {file = "psycopg-3.1.18.tar.gz", hash = "sha256:31144d3fb4c17d78094d9e579826f047d4af1da6a10427d91dfcfb6ecdf6f12b"}, +] + +[package.dependencies] +typing-extensions = ">=4.1" +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.1.18)"] +c = ["psycopg-c (==3.1.18)"] +dev = ["black (>=24.1.0)", "codespell (>=2.2)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.4.1)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=3.6.2,<4.0)", "mypy (>=1.4.1)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -638,7 +596,6 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -653,7 +610,6 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -668,7 +624,6 @@ cli = ["click (>=5.0)"] name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -690,7 +645,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -705,7 +659,6 @@ pyasn1 = ">=0.1.3" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -717,7 +670,6 @@ files = [ name = "stone" version = "3.3.1" description = "Stone is an interface description language (IDL) for APIs." -category = "main" optional = false python-versions = "*" files = [ @@ -730,11 +682,32 @@ files = [ ply = ">=3.4" six = ">=1.12.0" +[[package]] +name = "typing-extensions" +version = "4.11.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + [[package]] name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -750,4 +723,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "619d22851fd40043973e9fb2508c464b4d59af14c5eb1001860489e5f71f326e" +content-hash = "67d0ee095eab63ae774e18a26e982dd197a21ca2752f621e3f70ad2e4a737402" diff --git a/process_file/pyproject.toml b/process_file/pyproject.toml index ba1cc40..1d11545 100644 --- a/process_file/pyproject.toml +++ b/process_file/pyproject.toml @@ -11,9 +11,9 @@ python = "^3.9" dropbox = "^11.36.0" python-dotenv = "^1.0.0" google-cloud-storage = "^2.8.0" -mysql-connector-python = "^8.0.32" google-cloud-pubsub = "^2.16.0" google-cloud-secret-manager = "^2.16.1" +psycopg = "^3.1.18" [build-system] diff --git a/process_file/requirements.txt b/process_file/requirements.txt index 059a604..a51c35f 100644 --- a/process_file/requirements.txt +++ b/process_file/requirements.txt @@ -15,12 +15,12 @@ googleapis-common-protos==1.59.0 ; python_version >= "3.9" and python_version < googleapis-common-protos[grpc]==1.59.0 ; python_version >= "3.9" and python_version < "4.0" grpc-google-iam-v1==0.12.6 ; python_version >= "3.9" and python_version < "4.0" grpcio-status==1.48.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio==1.54.2 ; python_version < "4.0" and python_version >= "3.9" +grpcio==1.54.2 ; python_version >= "3.9" and python_version < "4.0" idna==3.4 ; python_version >= "3.9" and python_version < "4.0" -mysql-connector-python==8.0.33 ; python_version >= "3.9" and python_version < "4.0" ply==3.11 ; python_version >= "3.9" and python_version < "4.0" proto-plus==1.22.2 ; python_version >= "3.9" and python_version < "4.0" -protobuf==3.20.3 ; python_version < "4.0" and python_version >= "3.9" +protobuf==3.20.3 ; python_version >= "3.9" and python_version < "4.0" +psycopg==3.1.18 ; python_version >= "3.9" and python_version < "4.0" pyasn1-modules==0.3.0 ; python_version >= "3.9" and python_version < "4.0" pyasn1==0.5.0 ; python_version >= "3.9" and python_version < "4.0" python-dotenv==1.0.0 ; python_version >= "3.9" and python_version < "4.0" @@ -28,4 +28,6 @@ requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" rsa==4.9 ; python_version >= "3.9" and python_version < "4" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" stone==3.3.1 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.11.0 ; python_version >= "3.9" and python_version < "4.0" +tzdata==2024.1 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" urllib3==1.26.16 ; python_version >= "3.9" and python_version < "4.0" diff --git a/queue_files/main.py b/queue_files/main.py index 839a465..ee2db8b 100644 --- a/queue_files/main.py +++ b/queue_files/main.py @@ -3,7 +3,7 @@ from time import sleep import dropbox -import mysql.connector +import psycopg import requests from google.cloud import pubsub_v1, secretmanager, storage @@ -12,35 +12,17 @@ QUEUE_TOPIC_NAME = "projects/greg-finley/topics/dropbox-queue-files" secret_client = secretmanager.SecretManagerServiceClient() -mysql_config = json.loads(os.environ["MYSQL_CONFIG"]) - -mysql_connection = mysql.connector.connect( - unix_socket=mysql_config["MYSQL_SOCKET"], - user=mysql_config["MYSQL_USERNAME"], - passwd=mysql_config["MYSQL_PASSWORD"], - database=mysql_config["MYSQL_DATABASE"], -) -mysql_connection.autocommit = True - def run(event, context): - """Responds to any HTTP request. - Args: - request (flask.Request): HTTP request object. - Returns: - The response text or any set of values that can be turned into a - Response object using - `make_response `. - """ dbx = dropbox.Dropbox(os.getenv("DROPBOX_ACCESS_TOKEN")) query = ( "SELECT dropbox_cursor from dropbox_cursors order by created_at desc limit 1" ) - cursor = mysql_connection.cursor() - cursor.execute(query) - old_cursor = cursor.fetchone()[0] - print("Old cursor: ", old_cursor) - cursor.close() + conn = psycopg.connect(os.environ["NEON_DATABASE_URL"]) + with conn.cursor() as cursor: + cursor.execute(query) + old_cursor = cursor.fetchone()[0] + print("Old cursor: ", old_cursor) try: dropbox_result = dbx.files_list_folder_continue(cursor=old_cursor) @@ -57,10 +39,9 @@ def run(event, context): # Immediately write the new cursor to the database so further requests use # it and cut down on duplicated work print("New cursor: ", dropbox_result.cursor) - query = "INSERT IGNORE INTO dropbox_cursors (dropbox_cursor) VALUES (%s)" - cursor = mysql_connection.cursor() - cursor.execute(query, (dropbox_result.cursor,)) - cursor.close() + query = "INSERT INTO dropbox_cursors (dropbox_cursor) VALUES (%s) ON CONFLICT (dropbox_cursor) DO NOTHING" + with conn.cursor() as cursor: + cursor.execute(query, (dropbox_result.cursor,)) publisher = pubsub_v1.PublisherClient() gcs_client = storage.Client() @@ -78,25 +59,20 @@ def run(event, context): query = """ INSERT INTO dropbox (desktop_path, filename, status) VALUES (%s, SUBSTRING_INDEX(%s, '/', -1), 'pending') + ON CONFLICT (desktop_path) DO NOTHING """ - cursor = mysql_connection.cursor() - try: + with conn.cursor() as cursor: cursor.execute(query, (clean_name, clean_name)) print(f"Queued {clean_name}") - cursor.close() - # Maybe we already enqueued this file; continue instead of enqueueing again - except mysql.connector.Error as err: - print(f"Failed to insert entry: {err}") - cursor.close() - continue + future = publisher.publish(TOPIC_NAME, clean_name.encode("utf-8")) futures.append(future) elif isinstance(entry, dropbox.files.DeletedMetadata): print(f"Deleting {clean_name}") query = "UPDATE dropbox SET status = 'deleted' WHERE desktop_path = %s" - cursor = mysql_connection.cursor() - cursor.execute(query, (clean_name,)) - cursor.close() + with conn.cursor() as cursor: + cursor.execute(query, (clean_name,)) + try: gcs_bucket.delete_blob(clean_name) except Exception as err: @@ -109,6 +85,8 @@ def run(event, context): if dropbox_result.has_more: publisher.publish(QUEUE_TOPIC_NAME, "Hi".encode("utf-8")) + conn.close() + def refresh_token(): dropbox_config = json.loads(os.environ["DROPBOX_CONFIG"]) diff --git a/queue_files/requirements.txt b/queue_files/requirements.txt index 2e3e9c7..667ca1d 100644 --- a/queue_files/requirements.txt +++ b/queue_files/requirements.txt @@ -2,5 +2,7 @@ dropbox>=11.36.0 google-cloud-secret-manager>=2.16.1 google-cloud-pubsub>=2.17.1 requests>=2.31.0 -mysql-connector-python>=8.0.33 +psycopg==3.1.18 +typing-extensions==4.11.0 +tzdata==2024.1 google-cloud-storage>=2.9.0 diff --git a/requeue_failed_files/main.py b/requeue_failed_files/main.py index 5ed3e1c..d8f9313 100644 --- a/requeue_failed_files/main.py +++ b/requeue_failed_files/main.py @@ -3,7 +3,7 @@ from time import sleep import dropbox -import mysql.connector +import psycopg import requests from flask import Response from google.cloud import pubsub_v1, secretmanager @@ -11,15 +11,6 @@ TOPIC_NAME = "projects/greg-finley/topics/dropbox-backup" ACCESS_TOKEN_SECRET_NAME = "projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN" -mysql_config = json.loads(os.environ["MYSQL_CONFIG"]) - -mysql_connection = mysql.connector.connect( - unix_socket=mysql_config["MYSQL_SOCKET"], - user=mysql_config["MYSQL_USERNAME"], - passwd=mysql_config["MYSQL_PASSWORD"], - database=mysql_config["MYSQL_DATABASE"], -) -mysql_connection.autocommit = True secret_client = secretmanager.SecretManagerServiceClient() @@ -31,11 +22,10 @@ def run(request): except dropbox.exceptions.AuthError: refresh_token() - query = "SELECT desktop_path from dropbox where status = 'pending';" - cursor = mysql_connection.cursor() - cursor.execute(query) - desktop_paths = [row[0] for row in cursor.fetchall()] - cursor.close() + with psycopg.connect(os.environ["NEON_DATABASE_URL"]) as conn: + with conn.cursor() as cursor: + cursor.execute("SELECT desktop_path from dropbox where status = 'pending';") + desktop_paths = [row[0] for row in cursor.fetchall()] publisher = pubsub_v1.PublisherClient() diff --git a/requeue_failed_files/requirements.txt b/requeue_failed_files/requirements.txt index 66c5d55..5f07578 100644 --- a/requeue_failed_files/requirements.txt +++ b/requeue_failed_files/requirements.txt @@ -1,5 +1,7 @@ dropbox>=11.36.0 google-cloud-secret-manager>=2.16.1 google-cloud-pubsub>=2.17.1 +psycopg==3.1.18 +typing-extensions==4.11.0 +tzdata==2024.1 requests>=2.31.0 -mysql-connector-python>=8.0.33