Skip to content

Commit

Permalink
Move to Neon Postgres
Browse files Browse the repository at this point in the history
  • Loading branch information
greg-finley committed Apr 11, 2024
1 parent 7df4a9f commit 1c5a23c
Show file tree
Hide file tree
Showing 12 changed files with 113 additions and 178 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/deploy_process_file.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
entry_point: "run"
source_dir: "process_file"
memory_mb: 4096
secret_environment_variables: "MYSQL_CONFIG=projects/greg-finley/secrets/MYSQL_CONFIG/versions/latest"
secret_environment_variables: "NEON_DATABASE_URL=projects/greg-finley/secrets/NEON_DATABASE_URL/versions/latest"
timeout: 540
event_trigger_type: "providers/cloud.pubsub/eventTypes/topic.publish"
event_trigger_resource: "projects/greg-finley/topics/dropbox-backup"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/deploy_queue_files.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
entry_point: "run"
source_dir: "queue_files"
memory_mb: 256
secret_environment_variables: "MYSQL_CONFIG=projects/greg-finley/secrets/MYSQL_CONFIG/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest"
secret_environment_variables: "NEON_DATABASE_URL=projects/greg-finley/secrets/NEON_DATABASE_URL/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest"
timeout: 540
event_trigger_type: "providers/cloud.pubsub/eventTypes/topic.publish"
event_trigger_resource: "projects/greg-finley/topics/dropbox-queue-files"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/deploy_requeue_failed_files.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
entry_point: "run"
source_dir: "requeue_failed_files"
memory_mb: 256
secret_environment_variables: "MYSQL_CONFIG=projects/greg-finley/secrets/MYSQL_CONFIG/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest"
secret_environment_variables: "NEON_DATABASE_URL=projects/greg-finley/secrets/NEON_DATABASE_URL/versions/latest,DROPBOX_ACCESS_TOKEN=projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest,DROPBOX_CONFIG=projects/greg-finley/secrets/DROPBOX_CONFIG/versions/latest"
timeout: 540
min_instances: 0
max_instances: 1
40 changes: 20 additions & 20 deletions ddl.sql
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
CREATE TABLE "dropbox" (
"id" int NOT NULL AUTO_INCREMENT,
"desktop_path" varchar(300) NOT NULL,
"created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
"status" enum('pending','done','deleted') DEFAULT 'done',
"filename" varchar(300) NOT NULL,
PRIMARY KEY ("id"),
UNIQUE KEY "desktop_path" ("desktop_path"),
KEY "filename_index" ("filename"),
KEY "status_index" ("status"),
KEY "created_at_desc_idx" ("created_at" DESC)
)
CREATE TYPE status_type AS ENUM ('pending', 'done', 'deleted');

CREATE TABLE "dropbox_cursors" (
"id" int NOT NULL AUTO_INCREMENT,
"dropbox_cursor" varchar(500) NOT NULL,
"created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY ("id"),
UNIQUE KEY "dropbox_cursor" ("dropbox_cursor"),
KEY "idx_created_at_desc" ("created_at" DESC)
)
CREATE TABLE dropbox (
id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
desktop_path VARCHAR(300) NOT NULL UNIQUE,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
status status_type DEFAULT 'done',
filename VARCHAR(300) NOT NULL
);

CREATE INDEX filename_index ON dropbox (filename);
CREATE INDEX status_index ON dropbox (status);
CREATE INDEX created_at_desc_idx ON dropbox (created_at DESC);

CREATE TABLE dropbox_cursors (
id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
dropbox_cursor VARCHAR(500) NOT NULL UNIQUE,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);

CREATE INDEX idx_created_at_desc ON dropbox_cursors (created_at DESC);
20 changes: 4 additions & 16 deletions process_file/main.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,15 @@
import base64
import json
import os

import dropbox
import mysql.connector
import psycopg
from google.cloud import secretmanager, storage

gcs_client = storage.Client()
gcs_bucket = gcs_client.get_bucket("greg-finley-dropbox-backup")
mysql_config_str = os.environ["MYSQL_CONFIG"]
mysql_config_dict = json.loads(mysql_config_str)
secret_client = secretmanager.SecretManagerServiceClient()


mysql_connection = mysql.connector.connect(
unix_socket=mysql_config_dict["MYSQL_SOCKET"],
user=mysql_config_dict["MYSQL_USERNAME"],
passwd=mysql_config_dict["MYSQL_PASSWORD"],
database=mysql_config_dict["MYSQL_DATABASE"],
)
mysql_connection.autocommit = True


def run(event, context):
dropbox_access_token = secret_client.access_secret_version(
name="projects/greg-finley/secrets/DROPBOX_ACCESS_TOKEN/versions/latest"
Expand All @@ -40,7 +28,7 @@ def run(event, context):
VALUES (%s, SUBSTRING_INDEX(%s, '/', -1), 'done')
ON DUPLICATE KEY UPDATE status = 'done'
"""
cursor = mysql_connection.cursor()
cursor.execute(query, (filename, filename))
cursor.close()
with psycopg.connect(os.environ["NEON_DATABASE_URL"]) as conn:
with conn.cursor() as cursor:
cursor.execute(query, (filename, filename))
print(f"Uploaded {filename} - {content_type}")
Loading

0 comments on commit 1c5a23c

Please sign in to comment.