Merge pull request #208 from bruin-data/patch/run-ingestr-on-ci

Patch/run ingestr on ci

.github/workflows/build-test.yml

@@ -50,6 +50,9 @@ jobs:
           go-version-file: 'go.mod'
           cache: ${{ matrix.cache }}
       - run: make test
+      - name: Install winget
+        if: matrix.platform == 'windows-latest'
+        uses: Cyberboss/install-winget@v1
       - run: make duckdb-test
         shell: bash
 

.github/workflows/release.yml

@@ -146,9 +146,11 @@ jobs:
       - name: Use install script
         shell: bash
         run: curl -LsSf https://raw.githubusercontent.com/bruin-data/bruin/refs/heads/main/install.sh | sh -s -- -d ${GITHUB_REF##*/}
+      - name: Install winget
+        uses: Cyberboss/install-winget@v1
       - name: Test Pipeline
         shell: bash
-        run: /c/Users/runneradmin/.local/bin/bruin.exe init duckdb test-pipeline && cd test-pipeline && git init && /c/Users/runneradmin/.local/bin/bruin.exe run .
+        run: /c/Users/runneradmin/.local/bin/bruin.exe init chess test-pipeline && cd test-pipeline && git init && /c/Users/runneradmin/.local/bin/bruin.exe run .
 
   install-unix:
     runs-on: ubuntu-latest
@@ -167,7 +169,7 @@ jobs:
       - name: Use install script
         run: curl -LsSf https://raw.githubusercontent.com/bruin-data/bruin/refs/heads/main/install.sh | sh -s -- -d ${{ env.tag }}
       - name: Test Pipeline
-        run: bruin init duckdb test-pipeline && cd test-pipeline && git init && bruin run .
+        run: bruin init chess test-pipeline && cd test-pipeline && git init && bruin run .
 
   release:
     runs-on: ubuntu-latest

.gitignore

@@ -126,6 +126,7 @@ bin
 __pycache__/
 .bruin.yml
 !templates/duckdb/.bruin.yml
+!templates/chess/.bruin.yml
 dist/
 *.pyc
 

Makefile

@@ -21,7 +21,7 @@ build: deps
 
 duckdb-test: build
 	@echo "$(OK_COLOR)==> Testing with duck db...$(NO_COLOR)"
-	@./bin/bruin init duckdb test
+	@./bin/bruin init chess test
 	@cd test && git init
 	@./bin/bruin run test
 

pkg/config/manager.go

@@ -333,6 +333,23 @@ func LoadFromFile(fs afero.Fs, path string) (*Config, error) {
 		config.DefaultEnvironmentName = "default"
 	}
 
+	absoluteConfigPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get absolute path: %w", err)
+	}
+	configLocation := filepath.Dir(absoluteConfigPath)
+
+	// Make duckdb paths absolute
+	for _, env := range config.Environments {
+		for i, conn := range env.Connections.DuckDB {
+			if filepath.IsAbs(conn.Path) {
+				continue
+			}
+			fmt.Printf("Fixing %s to %s", conn.Path, filepath.Join(configLocation, conn.Path))
+			env.Connections.DuckDB[i].Path = filepath.Join(configLocation, conn.Path)
+		}
+	}
+
 	err = config.SelectEnvironment(config.DefaultEnvironmentName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to select default environment: %w", err)

pkg/config/manager_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"runtime"
 	"testing"
 
 	"github.com/bruin-data/bruin/pkg/pipeline"
@@ -12,6 +13,15 @@ import (
 func TestLoadFromFile(t *testing.T) {
 	t.Parallel()
 
+	var duckPath, configFile string
+	if runtime.GOOS == "windows" {
+		configFile = "simple_win.yml"
+		duckPath = "C:\\path\\to\\duck.db"
+	} else {
+		configFile = "simple.yml"
+		duckPath = "/path/to/duck.db"
+	}
+
 	devEnv := Environment{
 		Connections: &Connections{
 			GoogleCloudPlatform: []GoogleCloudPlatformConnection{
@@ -208,7 +218,7 @@ func TestLoadFromFile(t *testing.T) {
 			DuckDB: []DuckDBConnection{
 				{
 					Name: "conn20",
-					Path: "/path/to/duck.db",
+					Path: duckPath,
 				},
 			},
 			Hubspot: []HubspotConnection{
@@ -286,7 +296,7 @@ func TestLoadFromFile(t *testing.T) {
 		{
 			name: "read simple connection",
 			args: args{
-				path: "testdata/simple.yml",
+				path: "testdata/" + configFile,
 			},
 			want: &Config{
 				DefaultEnvironmentName:  "dev",

pkg/config/testdata/simple_win.yml

@@ -0,0 +1,183 @@
+default_environment: dev
+environments:
+  dev:
+    connections:
+      google_cloud_platform:
+        - name: conn1
+          service_account_json: "{\"key1\": \"value1\"}"
+          service_account_file: "/path/to/service_account.json"
+          project_id: "my-project"
+
+      snowflake:
+        - name: conn2
+          username: "user"
+          password: "pass"
+          account: "account"
+          database: "db"
+          warehouse: "wh"
+          schema: "schema"
+          role: "role"
+          region: "region"
+
+      postgres:
+        - name: conn3
+          username: "pguser"
+          password: "pgpass"
+          host: "somehost"
+          port: 5432
+          database: "pgdb"
+          schema: "non_public_schema"
+          pool_max_conns: 5
+          ssl_mode: "require"
+
+      redshift:
+        - name: conn4
+          username: "rsuser"
+          password: "rspass"
+          host: "someredshift"
+          port: 5433
+          database: "rsdb"
+          pool_max_conns: 4
+          ssl_mode: "disable"
+
+      mssql:
+        - name: conn5
+          username: "msuser"
+          password: "mspass"
+          host: "somemssql"
+          port: 1433
+          database: "mssqldb"
+
+      databricks:
+        - name: conn55
+          host: "hostbricks"
+          path: "sql"
+          token: "aaaaaaaa"
+          port: 443
+
+      synapse:
+        - name: conn6
+          username: "syuser"
+          password: "sypass"
+          host: "somemsynapse"
+          port: 1434
+          database: "sydb"
+
+      mongo:
+        - name: conn7
+          username: "mongouser"
+          password: "mongopass"
+          host: "mongohost"
+          port: 27017
+          database: "mongodb"
+
+      mysql:
+        - name: conn8
+          username: "mysqluser"
+          password: "mysqlpass"
+          host: "mysqlhost"
+          port: 3306
+          database: "mysqldb"
+
+      notion:
+        - name: conn9
+          api_key: "XXXXYYYYZZZZ"
+
+      hana:
+        - name: conn10
+          username: "hanauser"
+          password: "hanapass"
+          host: "hanahost"
+          port: 39013
+          database: "hanadb"
+
+      shopify:
+        - name: conn11
+          api_key: "shopifykey"
+          url: "shopifyurl"
+
+      gorgias:
+        - name: conn12
+          api_key: "gorgiaskey"
+          domain: "gorgiasurl"
+          email: "gorgiasemail"
+
+      aws:
+        - name: conn13
+          access_key: "awskey"
+          secret_key: "awssecret"
+      athena:
+        - name: conn14
+          access_key_id: "athena_key"
+          secret_access_key: "athena_secret"
+          query_results_path: "s3://bucket/prefix"
+          region: "us-west-2"
+          database: "athena_db"
+      klaviyo:
+        - name: conn15
+          api_key: "klaviyokey"
+      adjust:
+        - name: conn16
+          api_key: "adjustokey"
+      facebookads:
+        - name: conn17
+          access_token: "Facebookkey"
+          account_id: "Id123"
+      stripe:
+        - name: conn18
+          api_key: "stripekey"
+      appsflyer:
+        - name: conn19
+          api_key: "appsflyerkey"
+      kafka:
+        - name: conn20
+          bootstrap_servers: "localhost:9093"
+          group_id: "kafka123"
+      duckdb:
+        - name: conn20
+          path: "C:\\path\\to\\duck.db"
+      hubspot:
+        - name: conn21
+          api_key: "hubspotkey"
+      google_sheets:
+        - name: conn22
+          credentials_path: "/path/to/service_account.json"
+      chess:
+        - name: conn24
+          players:
+            - "Max"
+            - "Peter"
+      airtable:
+        - name: conn23
+          base_id: "123"
+          access_token: "accessKey"
+      zendesk:
+        - name: conn25
+          api_token: "zendeskKey"
+          email: "zendeskemail"
+          subdomain: "zendeskUrl"
+
+        - name: conn25-1
+          oauth_token: "zendeskToken"
+          subdomain: "zendeskUrl"
+      s3:
+        - name: conn25
+          bucket_name:  "my-bucket"
+          path_to_file:  "/folder1/file.csv"
+          access_key_id: "123Key"
+          secret_access_key: "secretKey123"
+      slack:
+        - name: conn26
+          api_key: "slackkey"
+      generic:
+        - name: key1
+          value: value1
+        - name: key2
+          value: value2
+
+  prod:
+    connections:
+      google_cloud_platform:
+        - name: conn1
+          service_account_file: "/path/to/service_account.json"
+          project_id: "my-project"

pkg/python/uv.go

@@ -109,7 +109,7 @@ func (u *UvChecker) installUvCommand(ctx context.Context, dest string) error {
 	if runtime.GOOS == "windows" {
 		commandInstance = exec.Command(Shell, ShellSubcommandFlag, fmt.Sprintf("winget install --accept-package-agreements --accept-source-agreements --silent --id=astral-sh.uv --version %s --location %s -e", UvVersion, dest)) //nolint:gosec
 	} else {
-		commandInstance = exec.Command(Shell, ShellSubcommandFlag, fmt.Sprintf(" set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 sh", UvVersion, dest)) //nolint:gosec
+		commandInstance = exec.Command("/bin/bash", "-c", fmt.Sprintf("set -o pipefail; curl -LsSf https://astral.sh/uv/%s/install.sh | UV_INSTALL_DIR=\"%s\" NO_MODIFY_PATH=1 bash", UvVersion, dest)) //nolint:gosec
 	}
 
 	err := u.cmd.RunAnyCommand(ctx, commandInstance)

templates/chess/.bruin.yml

@@ -0,0 +1,12 @@
+default_environment: default
+environments:
+  default:
+    connections:
+      duckdb:
+        - name: "duckdb-default"
+          path: "duckdb.db"
+      chess:
+        - name: "chess-default"
+          players:
+            - "MagnusCarlsen"
+            - "Hikaru"

templates/chess/assets/player_summary.sql

@@ -9,6 +9,13 @@ depends:
    - chess_playground.games
    - chess_playground.profiles
 
+columns:
+  - name: total_games
+    type: integer
+    description: "the games"
+    checks:
+      - name: positive
+
 @bruin */
 
 WITH game_results AS (