-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: enable Postgres connections * feat: implement migration templates * feat: convert sqlite-specific migrations to migration templates; adjust SQL syntax * fix: add migration to fix postgres incompatibilities in the DB schema * fix: correct SQL syntax for comparisons in the transactions service * feat: database migration tool * chore: update tests for testing with Postgres * fix: update test * fix: explicitly store mock timestamps in UTC * chore: add txdb to go.mod and run tidy * fix: serialize concurrent transactions when running tests with txdb * chore: update README to mention postgres * fix: use timestamptz in Postgres * test: add Github action for testing the backend with Postgres * chore: fix migration name and add comments * docs: add db migration notes to README * fix: address PR feedback * fix: rename the cmd/migrate tool to cmd/db_migrate * feat: add schema check in the DB migration tool * test: use testdb instead of txdb for testing with Postgresql * fix: fix tests after merge * chore: comment out unused VACUUM in migrations * fix: reset Postgres sequences after migrating data * docs: update migrate command in readme * fix: do not allow backup for migration when postgres is enabled * chore: add debug logs for db backend type * chore: use Logrus instead of slog for db_migrate * fix: initialize logger in the keys tests * test: add db_migrate test * fix: use transaction object in db_migrate function * fix: txlock in db migration test --------- Co-authored-by: Roland Bewick <[email protected]>
- Loading branch information
Showing
54 changed files
with
1,234 additions
and
459 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| name: Backend testing with Postgres | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| pull_request: | ||
| types: [opened, synchronize] | ||
|
|
||
| jobs: | ||
| test-postgres: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| services: | ||
| postgres: | ||
| image: postgres:17.2 | ||
| ports: | ||
| - 5432:5432 | ||
| env: | ||
| POSTGRES_DB: albyhub | ||
| POSTGRES_USER: alby | ||
| POSTGRES_PASSWORD: albytest123 | ||
| options: >- | ||
| --health-cmd pg_isready | ||
| --health-interval 10s | ||
| --health-timeout 5s | ||
| --health-retries 5 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| name: Check out code | ||
|
|
||
| - name: Setup GoLang | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: "./go.mod" | ||
|
|
||
| - name: Get dependencies | ||
| run: go get -v -t -d ./... | ||
|
|
||
| - name: Run tests | ||
| env: | ||
| TEST_DATABASE_URI: "postgresql://alby:albytest123@localhost:5432/albyhub" | ||
| TEST_DB_MIGRATE_POSTGRES_URI: "postgresql://alby:albytest123@localhost:5432/albyhub" | ||
| run: mkdir frontend/dist && touch frontend/dist/tmp && go test ./... |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,242 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "os" | ||
| "slices" | ||
| "strconv" | ||
|
|
||
| "github.com/sirupsen/logrus" | ||
| "gorm.io/gorm" | ||
|
|
||
| "github.com/getAlby/hub/db" | ||
| "github.com/getAlby/hub/logger" | ||
| ) | ||
|
|
||
| var expectedTables = []string{ | ||
| "apps", | ||
| "app_permissions", | ||
| "request_events", | ||
| "response_events", | ||
| "transactions", | ||
| "user_configs", | ||
| "migrations", | ||
| } | ||
|
|
||
| func main() { | ||
| var fromDSN, toDSN string | ||
|
|
||
| logger.Init(strconv.Itoa(int(logrus.DebugLevel))) | ||
|
|
||
| flag.StringVar(&fromDSN, "from", "", "source DSN") | ||
| flag.StringVar(&toDSN, "to", "", "destination DSN") | ||
|
|
||
| flag.Parse() | ||
|
|
||
| if fromDSN == "" || toDSN == "" { | ||
| flag.Usage() | ||
| logger.Logger.Error("missing DSN") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| stopDB := func(d *gorm.DB) { | ||
| if err := db.Stop(d); err != nil { | ||
| logger.Logger.WithError(err).Error("failed to close database") | ||
| } | ||
| } | ||
|
|
||
| logger.Logger.Info("opening source DB...") | ||
| fromDB, err := db.NewDB(fromDSN, false) | ||
| if err != nil { | ||
| logger.Logger.WithError(err).Error("failed to open source database") | ||
| os.Exit(1) | ||
| } | ||
| defer stopDB(fromDB) | ||
|
|
||
| logger.Logger.Info("opening destination DB...") | ||
| toDB, err := db.NewDB(toDSN, false) | ||
| if err != nil { | ||
| logger.Logger.WithError(err).Error("failed to open destination database") | ||
| os.Exit(1) | ||
| } | ||
| defer stopDB(toDB) | ||
|
|
||
| // Migrations are applied to both the source and the target DB, so | ||
| // schemas should be equal at this point. | ||
| err = checkSchema(fromDB) | ||
| if err != nil { | ||
| logger.Logger.WithError(err).Error("database schema check failed; the migration tool may be outdated") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| logger.Logger.Info("migrating...") | ||
| err = migrateDB(fromDB, toDB) | ||
| if err != nil { | ||
| logger.Logger.WithError(err).Error("failed to migrate database") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| logger.Logger.Info("migration complete") | ||
| } | ||
|
|
||
| func migrateDB(from, to *gorm.DB) error { | ||
| tx := to.Begin() | ||
| defer tx.Rollback() | ||
|
|
||
| if err := tx.Error; err != nil { | ||
| return fmt.Errorf("failed to start transaction: %w", err) | ||
| } | ||
|
|
||
| // Table migration order matters: referenced tables must be migrated | ||
| // before referencing tables. | ||
|
|
||
| logger.Logger.Info("migrating apps...") | ||
| if err := migrateTable[db.App](from, tx); err != nil { | ||
| return fmt.Errorf("failed to migrate apps: %w", err) | ||
| } | ||
|
|
||
| logger.Logger.Info("migrating app_permissions...") | ||
| if err := migrateTable[db.AppPermission](from, tx); err != nil { | ||
| return fmt.Errorf("failed to migrate app_permissions: %w", err) | ||
| } | ||
|
|
||
| logger.Logger.Info("migrating request_events...") | ||
| if err := migrateTable[db.RequestEvent](from, tx); err != nil { | ||
| return fmt.Errorf("failed to migrate request_events: %w", err) | ||
| } | ||
|
|
||
| logger.Logger.Info("migrating response_events...") | ||
| if err := migrateTable[db.ResponseEvent](from, tx); err != nil { | ||
| return fmt.Errorf("failed to migrate response_events: %w", err) | ||
| } | ||
|
|
||
| logger.Logger.Info("migrating transactions...") | ||
| if err := migrateTable[db.Transaction](from, tx); err != nil { | ||
| return fmt.Errorf("failed to migrate transactions: %w", err) | ||
| } | ||
|
|
||
| logger.Logger.Info("migrating user_configs...") | ||
| if err := migrateTable[db.UserConfig](from, tx); err != nil { | ||
| return fmt.Errorf("failed to migrate user_configs: %w", err) | ||
| } | ||
|
|
||
| if to.Dialector.Name() == "postgres" { | ||
| logger.Logger.Info("resetting sequences...") | ||
| if err := resetSequences(to); err != nil { | ||
| return fmt.Errorf("failed to reset sequences: %w", err) | ||
| } | ||
| } | ||
|
|
||
| tx.Commit() | ||
| if err := tx.Error; err != nil { | ||
| return fmt.Errorf("failed to commit transaction: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func migrateTable[T any](from, to *gorm.DB) error { | ||
| var data []T | ||
| if err := from.Find(&data).Error; err != nil { | ||
| return fmt.Errorf("failed to fetch data: %w", err) | ||
| } | ||
|
|
||
| if len(data) == 0 { | ||
| return nil | ||
| } | ||
|
|
||
| if err := to.Create(data).Error; err != nil { | ||
| return fmt.Errorf("failed to insert data: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func checkSchema(db *gorm.DB) error { | ||
| tables, err := listTables(db) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to list database tables: %w", err) | ||
| } | ||
|
|
||
| for _, table := range expectedTables { | ||
| if !slices.Contains(tables, table) { | ||
| return fmt.Errorf("table missing from the database: %q", table) | ||
| } | ||
| } | ||
|
|
||
| for _, table := range tables { | ||
| if !slices.Contains(expectedTables, table) { | ||
| return fmt.Errorf("unexpected table found in the database: %q", table) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func listTables(db *gorm.DB) ([]string, error) { | ||
| var query string | ||
|
|
||
| switch db.Dialector.Name() { | ||
| case "sqlite": | ||
| query = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';" | ||
| case "postgres": | ||
| query = "SELECT tablename FROM pg_tables WHERE schemaname = 'public';" | ||
| default: | ||
| return nil, fmt.Errorf("unsupported database: %q", db.Dialector.Name()) | ||
| } | ||
|
|
||
| rows, err := db.Raw(query).Rows() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to query table names: %w", err) | ||
| } | ||
| defer func() { | ||
| if err := rows.Close(); err != nil { | ||
| logger.Logger.WithError(err).Error("failed to close rows") | ||
| } | ||
| }() | ||
|
|
||
| var tables []string | ||
| for rows.Next() { | ||
| var table string | ||
| if err := rows.Scan(&table); err != nil { | ||
| return nil, fmt.Errorf("failed to scan table name: %w", err) | ||
| } | ||
| tables = append(tables, table) | ||
| } | ||
|
|
||
| return tables, nil | ||
| } | ||
|
|
||
| func resetSequences(db *gorm.DB) error { | ||
| type resetReq struct { | ||
| table string | ||
| seq string | ||
| } | ||
|
|
||
| resetReqs := []resetReq{ | ||
| {"apps", "apps_2_id_seq"}, | ||
| {"app_permissions", "app_permissions_2_id_seq"}, | ||
| {"request_events", "request_events_id_seq"}, | ||
| {"response_events", "response_events_id_seq"}, | ||
| {"transactions", "transactions_id_seq"}, | ||
| {"user_configs", "user_configs_id_seq"}, | ||
| } | ||
|
|
||
| for _, req := range resetReqs { | ||
| if err := resetPostgresSequence(db, req.table, req.seq); err != nil { | ||
| return fmt.Errorf("failed to reset sequence %q for %q: %w", req.seq, req.table, err) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func resetPostgresSequence(db *gorm.DB, table string, seq string) error { | ||
| query := fmt.Sprintf("SELECT setval('%s', (SELECT MAX(id) FROM %s));", seq, table) | ||
| if err := db.Exec(query).Error; err != nil { | ||
| return fmt.Errorf("failed to execute setval(): %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.