From 3830572f885ee90e1c16880aaa8a8f740678b10a Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 09:12:53 +0100 Subject: [PATCH 01/13] add column check and .bruin.yml to template. --- .gitignore | 1 + templates/chess/.bruin.yml | 12 ++++++++++++ templates/chess/assets/player_summary.sql | 7 +++++++ 3 files changed, 20 insertions(+) create mode 100644 templates/chess/.bruin.yml diff --git a/.gitignore b/.gitignore index 9a4b7937..cd8af0e6 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,7 @@ bin __pycache__/ .bruin.yml !templates/duckdb/.bruin.yml +!templates/chess/.bruin.yml dist/ *.pyc diff --git a/templates/chess/.bruin.yml b/templates/chess/.bruin.yml new file mode 100644 index 00000000..7ff22a6d --- /dev/null +++ b/templates/chess/.bruin.yml @@ -0,0 +1,12 @@ +default_environment: default +environments: + default: + connections: + duckdb: + - name: "duckdb-default" + path: "duckdb.db" + chess: + - name: "chess-default" + players: + - "MagnusCarlsen" + - "Hikaru" \ No newline at end of file diff --git a/templates/chess/assets/player_summary.sql b/templates/chess/assets/player_summary.sql index d6b637de..2b723f0f 100644 --- a/templates/chess/assets/player_summary.sql +++ b/templates/chess/assets/player_summary.sql @@ -9,6 +9,13 @@ depends: - chess_playground.games - chess_playground.profiles +columns: + - name: total_games + type: integer + description: "the games" + checks: + - name: positive + @bruin */ WITH game_results AS ( From a75b77403d16295c2c7dd0471e820712b3176b21 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 09:15:15 +0100 Subject: [PATCH 02/13] run chess template instead --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b8b6313d..23adee77 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ build: deps duckdb-test: build @echo "$(OK_COLOR)==> Testing with duck db...$(NO_COLOR)" - @./bin/bruin init duckdb test + @./bin/bruin init chess test @cd test && git init @./bin/bruin run test From 8bb1bbabc58c2daf08bb80589d658ec55b3a4544 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 09:38:07 +0100 Subject: [PATCH 03/13] use whatever shell is being used --- pkg/python/uv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/python/uv.go b/pkg/python/uv.go index 316dc03a..c2c6aecc 100644 --- a/pkg/python/uv.go +++ b/pkg/python/uv.go @@ -109,7 +109,7 @@ func (u *UvChecker) installUvCommand(ctx context.Context, dest string) error { if runtime.GOOS == "windows" { commandInstance = exec.Command(Shell, ShellSubcommandFlag, fmt.Sprintf("winget install --accept-package-agreements --accept-source-agreements --silent --id=astral-sh.uv --version %s --location %s -e", UvVersion, dest)) //nolint:gosec } else { - commandInstance = exec.Command(Shell, ShellSubcommandFlag, fmt.Sprintf(" set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 sh", UvVersion, dest)) //nolint:gosec + commandInstance = exec.Command(fmt.Sprintf(" set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 sh", UvVersion, dest)) //nolint:gosec } err := u.cmd.RunAnyCommand(ctx, commandInstance) From 1c70f2b5fb8060579a1d1ca689ffa041d2cae0f7 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 09:43:48 +0100 Subject: [PATCH 04/13] fullpath sh --- pkg/python/uv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/python/uv.go b/pkg/python/uv.go index c2c6aecc..b2c486fa 100644 --- a/pkg/python/uv.go +++ b/pkg/python/uv.go @@ -109,7 +109,7 @@ func (u *UvChecker) installUvCommand(ctx context.Context, dest string) error { if runtime.GOOS == "windows" { commandInstance = exec.Command(Shell, ShellSubcommandFlag, fmt.Sprintf("winget install --accept-package-agreements --accept-source-agreements --silent --id=astral-sh.uv --version %s --location %s -e", UvVersion, dest)) //nolint:gosec } else { - commandInstance = exec.Command(fmt.Sprintf(" set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 sh", UvVersion, dest)) //nolint:gosec + commandInstance = exec.Command(fmt.Sprintf(" set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 /bin/sh", UvVersion, dest)) //nolint:gosec } err := u.cmd.RunAnyCommand(ctx, commandInstance) From e4a05e11451ee9c4eada43864ee4b21f904d8bb5 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 09:54:54 +0100 Subject: [PATCH 05/13] fullpath sh --- pkg/python/uv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/python/uv.go b/pkg/python/uv.go index b2c486fa..8208c1b3 100644 --- a/pkg/python/uv.go +++ b/pkg/python/uv.go @@ -109,7 +109,7 @@ func (u *UvChecker) installUvCommand(ctx context.Context, dest string) error { if runtime.GOOS == "windows" { commandInstance = exec.Command(Shell, ShellSubcommandFlag, fmt.Sprintf("winget install --accept-package-agreements --accept-source-agreements --silent --id=astral-sh.uv --version %s --location %s -e", UvVersion, dest)) //nolint:gosec } else { - commandInstance = exec.Command(fmt.Sprintf(" set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 /bin/sh", UvVersion, dest)) //nolint:gosec + commandInstance = exec.Command("/bin/bash", "-c", fmt.Sprintf("set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 bash", UvVersion, dest)) //nolint:gosec } err := u.cmd.RunAnyCommand(ctx, commandInstance) From 6d8968fab118ca654c7817127df9658d7537d7a0 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 11:38:26 +0100 Subject: [PATCH 06/13] make path absolute and run chess first --- .github/workflows/build-test.yml | 3 +-- templates/chess/.bruin.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index a11d8e9e..2b8a4440 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -49,10 +49,9 @@ jobs: with: go-version-file: 'go.mod' cache: ${{ matrix.cache }} - - run: make test - run: make duckdb-test shell: bash - + - run: make test lint: runs-on: ubuntu-latest diff --git a/templates/chess/.bruin.yml b/templates/chess/.bruin.yml index 7ff22a6d..9ebd967e 100644 --- a/templates/chess/.bruin.yml +++ b/templates/chess/.bruin.yml @@ -4,7 +4,7 @@ environments: connections: duckdb: - name: "duckdb-default" - path: "duckdb.db" + path: "/tmp/duckdb.db" chess: - name: "chess-default" players: From 67341436aabd636a0482612ff4ef3c2a58a35edf Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 11:48:56 +0100 Subject: [PATCH 07/13] fix duckdb paths to be absolute --- pkg/config/manager.go | 17 +++++++++++++++++ templates/chess/.bruin.yml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/config/manager.go b/pkg/config/manager.go index a4a3f17a..c5c5852a 100644 --- a/pkg/config/manager.go +++ b/pkg/config/manager.go @@ -333,6 +333,23 @@ func LoadFromFile(fs afero.Fs, path string) (*Config, error) { config.DefaultEnvironmentName = "default" } + absoluteConfigPath, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path: %w", err) + } + configLocation := filepath.Dir(absoluteConfigPath) + + // Make duckdb paths absolute + for _, env := range config.Environments { + for _, conn := range env.Connections.DuckDB { + if filepath.IsAbs(conn.Path) { + continue + } + + conn.Path = filepath.Join(configLocation, conn.Path) + } + } + err = config.SelectEnvironment(config.DefaultEnvironmentName) if err != nil { return nil, fmt.Errorf("failed to select default environment: %w", err) diff --git a/templates/chess/.bruin.yml b/templates/chess/.bruin.yml index 9ebd967e..7ff22a6d 100644 --- a/templates/chess/.bruin.yml +++ b/templates/chess/.bruin.yml @@ -4,7 +4,7 @@ environments: connections: duckdb: - name: "duckdb-default" - path: "/tmp/duckdb.db" + path: "duckdb.db" chess: - name: "chess-default" players: From 784223a5170670a88cdf5627e800b44a823d7d1e Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 12:18:02 +0100 Subject: [PATCH 08/13] modify actual slice not copy --- pkg/config/manager.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/config/manager.go b/pkg/config/manager.go index c5c5852a..05730043 100644 --- a/pkg/config/manager.go +++ b/pkg/config/manager.go @@ -341,12 +341,12 @@ func LoadFromFile(fs afero.Fs, path string) (*Config, error) { // Make duckdb paths absolute for _, env := range config.Environments { - for _, conn := range env.Connections.DuckDB { + for i, conn := range env.Connections.DuckDB { if filepath.IsAbs(conn.Path) { continue } - - conn.Path = filepath.Join(configLocation, conn.Path) + fmt.Printf("Fixing %s to %s", conn.Path, filepath.Join(configLocation, conn.Path)) + env.Connections.DuckDB[i].Path = filepath.Join(configLocation, conn.Path) } } From 904961d721eb85d0916cb8b3dc21827764b324f1 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 13:38:38 +0100 Subject: [PATCH 09/13] install winget on wondows --- .github/workflows/build-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2b8a4440..f98376af 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -49,6 +49,9 @@ jobs: with: go-version-file: 'go.mod' cache: ${{ matrix.cache }} + - name: Install winget + if: matrix.platform == 'windows-latest' + uses: Cyberboss/install-winget@v1 - run: make duckdb-test shell: bash - run: make test From 2ed6fdc6d8df191dcc9f1a402473e56cd3890159 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 14:02:39 +0100 Subject: [PATCH 10/13] fix test for windows --- pkg/config/manager_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/config/manager_test.go b/pkg/config/manager_test.go index 1b8c6a1f..4bb7d124 100644 --- a/pkg/config/manager_test.go +++ b/pkg/config/manager_test.go @@ -1,6 +1,7 @@ package config import ( + "runtime" "testing" "github.com/bruin-data/bruin/pkg/pipeline" @@ -12,6 +13,13 @@ import ( func TestLoadFromFile(t *testing.T) { t.Parallel() + var duckPath string + if runtime.GOOS == "windows" { + duckPath = "C:\\path\\to\\duck.db" + } else { + duckPath = "/path/to/duck.db" + } + devEnv := Environment{ Connections: &Connections{ GoogleCloudPlatform: []GoogleCloudPlatformConnection{ @@ -208,7 +216,7 @@ func TestLoadFromFile(t *testing.T) { DuckDB: []DuckDBConnection{ { Name: "conn20", - Path: "/path/to/duck.db", + Path: duckPath, }, }, Hubspot: []HubspotConnection{ From f7ea774adba45eb8014b27ba862663743ce1868a Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 14:10:03 +0100 Subject: [PATCH 11/13] change release to test chess pipeline instead of duckdb --- .github/workflows/release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d704026..db323181 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -146,9 +146,11 @@ jobs: - name: Use install script shell: bash run: curl -LsSf https://raw.githubusercontent.com/bruin-data/bruin/refs/heads/main/install.sh | sh -s -- -d ${GITHUB_REF##*/} + - name: Install winget + uses: Cyberboss/install-winget@v1 - name: Test Pipeline shell: bash - run: /c/Users/runneradmin/.local/bin/bruin.exe init duckdb test-pipeline && cd test-pipeline && git init && /c/Users/runneradmin/.local/bin/bruin.exe run . + run: /c/Users/runneradmin/.local/bin/bruin.exe init chess test-pipeline && cd test-pipeline && git init && /c/Users/runneradmin/.local/bin/bruin.exe run . install-unix: runs-on: ubuntu-latest @@ -167,7 +169,7 @@ jobs: - name: Use install script run: curl -LsSf https://raw.githubusercontent.com/bruin-data/bruin/refs/heads/main/install.sh | sh -s -- -d ${{ env.tag }} - name: Test Pipeline - run: bruin init duckdb test-pipeline && cd test-pipeline && git init && bruin run . + run: bruin init chess test-pipeline && cd test-pipeline && git init && bruin run . release: runs-on: ubuntu-latest From f2828948d87ef8f7b954a0004f7e73e9d526afa8 Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 14:20:26 +0100 Subject: [PATCH 12/13] use different file --- pkg/config/manager_test.go | 6 +- pkg/config/testdata/simple_win.yml | 183 +++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 pkg/config/testdata/simple_win.yml diff --git a/pkg/config/manager_test.go b/pkg/config/manager_test.go index 4bb7d124..c71c148a 100644 --- a/pkg/config/manager_test.go +++ b/pkg/config/manager_test.go @@ -13,10 +13,12 @@ import ( func TestLoadFromFile(t *testing.T) { t.Parallel() - var duckPath string + var duckPath, configFile string if runtime.GOOS == "windows" { + configFile = "simple_win.yml" duckPath = "C:\\path\\to\\duck.db" } else { + configFile = "simple.yml" duckPath = "/path/to/duck.db" } @@ -294,7 +296,7 @@ func TestLoadFromFile(t *testing.T) { { name: "read simple connection", args: args{ - path: "testdata/simple.yml", + path: "testdata/" + configFile, }, want: &Config{ DefaultEnvironmentName: "dev", diff --git a/pkg/config/testdata/simple_win.yml b/pkg/config/testdata/simple_win.yml new file mode 100644 index 00000000..d3879ab6 --- /dev/null +++ b/pkg/config/testdata/simple_win.yml @@ -0,0 +1,183 @@ +default_environment: dev +environments: + dev: + connections: + google_cloud_platform: + - name: conn1 + service_account_json: "{\"key1\": \"value1\"}" + service_account_file: "/path/to/service_account.json" + project_id: "my-project" + + snowflake: + - name: conn2 + username: "user" + password: "pass" + account: "account" + database: "db" + warehouse: "wh" + schema: "schema" + role: "role" + region: "region" + + postgres: + - name: conn3 + username: "pguser" + password: "pgpass" + host: "somehost" + port: 5432 + database: "pgdb" + schema: "non_public_schema" + pool_max_conns: 5 + ssl_mode: "require" + + redshift: + - name: conn4 + username: "rsuser" + password: "rspass" + host: "someredshift" + port: 5433 + database: "rsdb" + pool_max_conns: 4 + ssl_mode: "disable" + + mssql: + - name: conn5 + username: "msuser" + password: "mspass" + host: "somemssql" + port: 1433 + database: "mssqldb" + + databricks: + - name: conn55 + host: "hostbricks" + path: "sql" + token: "aaaaaaaa" + port: 443 + + synapse: + - name: conn6 + username: "syuser" + password: "sypass" + host: "somemsynapse" + port: 1434 + database: "sydb" + + mongo: + - name: conn7 + username: "mongouser" + password: "mongopass" + host: "mongohost" + port: 27017 + database: "mongodb" + + mysql: + - name: conn8 + username: "mysqluser" + password: "mysqlpass" + host: "mysqlhost" + port: 3306 + database: "mysqldb" + + notion: + - name: conn9 + api_key: "XXXXYYYYZZZZ" + + hana: + - name: conn10 + username: "hanauser" + password: "hanapass" + host: "hanahost" + port: 39013 + database: "hanadb" + + shopify: + - name: conn11 + api_key: "shopifykey" + url: "shopifyurl" + + gorgias: + - name: conn12 + api_key: "gorgiaskey" + domain: "gorgiasurl" + email: "gorgiasemail" + + aws: + - name: conn13 + access_key: "awskey" + secret_key: "awssecret" + athena: + - name: conn14 + access_key_id: "athena_key" + secret_access_key: "athena_secret" + query_results_path: "s3://bucket/prefix" + region: "us-west-2" + database: "athena_db" + klaviyo: + - name: conn15 + api_key: "klaviyokey" + adjust: + - name: conn16 + api_key: "adjustokey" + facebookads: + - name: conn17 + access_token: "Facebookkey" + account_id: "Id123" + stripe: + - name: conn18 + api_key: "stripekey" + appsflyer: + - name: conn19 + api_key: "appsflyerkey" + kafka: + - name: conn20 + bootstrap_servers: "localhost:9093" + group_id: "kafka123" + duckdb: + - name: conn20 + path: "C:\\path\\to\\duck.db" + hubspot: + - name: conn21 + api_key: "hubspotkey" + google_sheets: + - name: conn22 + credentials_path: "/path/to/service_account.json" + chess: + - name: conn24 + players: + - "Max" + - "Peter" + airtable: + - name: conn23 + base_id: "123" + access_token: "accessKey" + zendesk: + - name: conn25 + api_token: "zendeskKey" + email: "zendeskemail" + subdomain: "zendeskUrl" + + - name: conn25-1 + oauth_token: "zendeskToken" + subdomain: "zendeskUrl" + s3: + - name: conn25 + bucket_name: "my-bucket" + path_to_file: "/folder1/file.csv" + access_key_id: "123Key" + secret_access_key: "secretKey123" + slack: + - name: conn26 + api_key: "slackkey" + generic: + - name: key1 + value: value1 + - name: key2 + value: value2 + + prod: + connections: + google_cloud_platform: + - name: conn1 + service_account_file: "/path/to/service_account.json" + project_id: "my-project" From 408b934aef11dbd6bb0d9919cd1e23516e68f3ce Mon Sep 17 00:00:00 2001 From: "Alberto J. Gomez" Date: Thu, 14 Nov 2024 14:42:02 +0100 Subject: [PATCH 13/13] change order --- .github/workflows/build-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f98376af..1fbea199 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -49,12 +49,13 @@ jobs: with: go-version-file: 'go.mod' cache: ${{ matrix.cache }} + - run: make test - name: Install winget if: matrix.platform == 'windows-latest' uses: Cyberboss/install-winget@v1 - run: make duckdb-test shell: bash - - run: make test + lint: runs-on: ubuntu-latest