Release - Fix path traversal bug for sqlite, new clickhouse options, and support for OpenAI-like URLs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,6 @@
 ---
 name: Bug report
-about: Create a report to help us improve
+about: Create a report to help us improve WhoDB.
 title: "[BUG] - {your title here}"
 labels: bug
 assignees: ''
@@ -24,15 +24,16 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+ - OS that WhoDB is running on: [e.g. Ubuntu]
+ - How you're running WhoDB: [e.g, Docker, exe, manually]
+ - Browser: [e.g. chrome, safari]
+ - Version: [e.g. 22]
 
 **Smartphone (please complete the following information):**
  - Device: [e.g. iPhone6]
  - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
+ - Browser: [e.g. stock browser, safari]
+ - Version: [e.g. 22]
 
 **Additional context**
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,6 @@
 ---
 name: Feature request
-about: Suggest an idea for this project
+about: Suggest an idea for this project.
 title: "[FR] - {your title here}"
 labels: enhancement
 assignees: ''

README.md

@@ -68,12 +68,17 @@ Get up and running with WhoDB quickly using Docker:
 docker run -it -p 8080:8080 clidey/whodb
 ```
 
+
+To run WhoDB with an OpenAI compatible service, you should assign some environments.
+```sh
+docker run -it -e USE_CUSTOM_MODELS=1 -e CUSTOM_MODELS=gpt-4o,gpt-3.5,others -e OPENAI_BASE_URL=http://your_base_url/v1 -p 8080:8080 clidey/whodb
+```
+
 If you are using a remote Ollama server, please start Docker with Ollama Environments like this:
 ```sh
 docker run -it -e WHODB_OLLAMA_HOST=YOUR_OLLAMA_HOST -e WHODB_OLLAMA_PORT=YOUR_OLLAMA_PORT -p 8080:8080 clidey/whodb
 ```
 
-
 Or, use Docker Compose:
 ```sh
 version: "3.8"

core/src/llm/chatgpt_client.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 )
 
-const chatGPTEndpoint = "https://api.openai.com/v1"
 
 func prepareChatGPTRequest(c *LLMClient, prompt string, model LLMModel, receiverChan *chan string) (string, []byte, map[string]string, error) {
 	requestBody, err := json.Marshal(map[string]interface{}{
@@ -20,7 +19,7 @@ func prepareChatGPTRequest(c *LLMClient, prompt string, model LLMModel, receiver
 	if err != nil {
 		return "", nil, nil, err
 	}
-	url := fmt.Sprintf("%v/chat/completions", chatGPTEndpoint)
+	url := fmt.Sprintf("%v/chat/completions", getOpenAICompatibleBaseURL())
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", c.APIKey),
 		"Content-Type":  "application/json",
@@ -29,7 +28,7 @@ func prepareChatGPTRequest(c *LLMClient, prompt string, model LLMModel, receiver
 }
 
 func prepareChatGPTModelsRequest(apiKey string) (string, map[string]string) {
-	url := fmt.Sprintf("%v/models", chatGPTEndpoint)
+	url := fmt.Sprintf("%v/models", getOpenAICompatibleBaseURL())
 	headers := map[string]string{
 		"Authorization": fmt.Sprintf("Bearer %s", apiKey),
 		"Content-Type":  "application/json",

core/src/llm/env.go

@@ -2,7 +2,8 @@ package llm
 
 import (
 	"fmt"
-
+    "strings"
+    "os"
 	"github.com/clidey/whodb/core/src/common"
 	"github.com/clidey/whodb/core/src/env"
 )
@@ -24,3 +25,31 @@ func getOllamaEndpoint() string {
 
 	return fmt.Sprintf("http://%v:%v/api", host, port)
 }
+
+func getOpenAICompatibleBaseURL() string {
+	defaultBaseURL := "https://api.openai.com/v1"
+	baseURL := os.Getenv("OPENAI_BASE_URL")
+	if baseURL == "" {
+		baseURL = defaultBaseURL
+	}
+	return baseURL
+}
+
+func getCustomModels() ([]string, error) {
+    modelsStr := os.Getenv("CUSTOM_MODELS")
+	if modelsStr == "" {
+		return []string{}, nil
+	}
+
+	models := strings.Split(modelsStr, ",")
+
+	for i := range models {
+		models[i] = strings.TrimSpace(models[i])
+	}
+    return models, nil
+}
+
+func ShouldUseCustomModels() bool {
+	useCustomModels := os.Getenv("USE_CUSTOM_MODELS")
+	return useCustomModels == "1"
+}

core/src/llm/llm_client.go

@@ -66,6 +66,9 @@ func (c *LLMClient) GetSupportedModels() ([]string, error) {
 		url, headers = prepareOllamaModelsRequest()
 	case ChatGPT_LLMType:
 		url, headers = prepareChatGPTModelsRequest(c.APIKey)
+		if ShouldUseCustomModels() {
+		    return getCustomModels()
+		}
 	case Anthropic_LLMType:
 		return getAnthropicModels(c.APIKey)
 	default:

core/src/plugins/clickhouse/add.go

@@ -72,7 +72,7 @@
 		query += fmt.Sprintf("\nSETTINGS %s", strings.Join(settingsClauses, ", "))
 	}
 
-	err = conn.Exec(context.Background(), query)
+	_, err = conn.ExecContext(context.Background(), query)
 	if err != nil {
 		return false, fmt.Errorf("failed to create table: %w (query: %s)", err, query)
 	}
@@ -100,6 +100,6 @@
 	query := fmt.Sprintf("INSERT INTO %s.%s (%s) VALUES (%s)",
 		schema, storageUnit, strings.Join(columns, ", "), strings.Join(placeholders, ", "))
 
-	err = conn.Exec(context.Background(), query, args...)
+	_, err = conn.ExecContext(context.Background(), query, args...)
 	return err == nil, err
 }

core/src/plugins/clickhouse/clickhouse.go

@@ -2,8 +2,8 @@
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
-	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"strconv"
 
 	"github.com/clidey/whodb/core/src/engine"
@@ -17,7 +17,7 @@
 		return false
 	}
 	defer conn.Close()
-	return conn.Ping(context.Background()) == nil
+	return conn.PingContext(context.Background()) == nil
 }
 
 func (p *ClickHousePlugin) GetDatabases(config *engine.PluginConfig) ([]string, error) {
@@ -27,7 +27,7 @@
 	}
 	defer conn.Close()
 
-	rows, err := conn.Query(context.Background(), "SHOW DATABASES")
+	rows, err := conn.QueryContext(context.Background(), "SHOW DATABASES")
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@
 		WHERE database = '%s'
 	`, schema)
 
-	rows, err := conn.Query(context.Background(), query)
+	rows, err := conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return nil, err
 	}
@@ -102,7 +102,7 @@
 	return storageUnits, nil
 }
 
-func getTableSchema(conn driver.Conn, schema string, tableName string) ([]engine.Record, error) {
+func getTableSchema(conn *sql.DB, schema string, tableName string) ([]engine.Record, error) {
 	query := fmt.Sprintf(`
 		SELECT 
 			name,
@@ -112,7 +112,7 @@
 		ORDER BY position
 	`, schema, tableName)
 
-	rows, err := conn.Query(context.Background(), query)
+	rows, err := conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return nil, err
 	}

core/src/plugins/clickhouse/db.go

@@ -2,7 +2,11 @@ package clickhouse
 
 import (
 	"context"
+	"crypto/tls"
+	"database/sql"
 	"fmt"
+	"github.com/clidey/whodb/core/src/log"
+	"strings"
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -11,29 +15,61 @@ import (
 	"github.com/clidey/whodb/core/src/engine"
 )
 
-func DB(config *engine.PluginConfig) (driver.Conn, error) {
-	port := common.GetRecordValueOrDefault(config.Credentials.Advanced, "Port", "9000")
+const (
+	portKey         = "Port"
+	sslModeKey      = "SSL Mode"
+	httpProtocolKey = "HTTP Protocol"
+	readOnlyKey     = "Readonly"
+	debugKey        = "Debug"
+)
+
+func DB(config *engine.PluginConfig) (*sql.DB, error) {
+	port := common.GetRecordValueOrDefault(config.Credentials.Advanced, portKey, "9000")
+	sslMode := common.GetRecordValueOrDefault(config.Credentials.Advanced, sslModeKey, "disable")
+	httpProtocol := common.GetRecordValueOrDefault(config.Credentials.Advanced, httpProtocolKey, "disable")
+	readOnly := common.GetRecordValueOrDefault(config.Credentials.Advanced, readOnlyKey, "disable")
+	debug := common.GetRecordValueOrDefault(config.Credentials.Advanced, debugKey, "disable")
 	options := &clickhouse.Options{
 		Addr: []string{fmt.Sprintf("%s:%s", config.Credentials.Hostname, port)},
 		Auth: clickhouse.Auth{
 			Database: config.Credentials.Database,
 			Username: config.Credentials.Username,
 			Password: config.Credentials.Password,
 		},
-		Settings: clickhouse.Settings{
-			"max_execution_time": 60,
-		},
 		DialTimeout:      time.Second * 30,
-		MaxOpenConns:     5,
-		MaxIdleConns:     5,
-		ConnMaxLifetime:  time.Hour,
 		ConnOpenStrategy: clickhouse.ConnOpenInOrder,
-		Compression: &clickhouse.Compression{
+	}
+	if debug != "disable" {
+		options.Debug = true
+	}
+	if readOnly == "disable" {
+		options.Settings = clickhouse.Settings{
+			"max_execution_time": 60,
+		}
+	}
+	if strings.HasPrefix(port, "8") || httpProtocol != "disable" {
+		options.Protocol = clickhouse.HTTP
+		options.Compression = &clickhouse.Compression{
+			Method: clickhouse.CompressionGZIP,
+		}
+	} else {
+		options.Compression = &clickhouse.Compression{
 			Method: clickhouse.CompressionLZ4,
-		},
+		}
+		options.MaxOpenConns = 5
+		options.MaxIdleConns = 5
+		options.ConnMaxLifetime = time.Hour
+	}
+	if sslMode != "disable" {
+		options.TLS = &tls.Config{InsecureSkipVerify: sslMode == "relaxed" || sslMode == "none"}
 	}
 
-	return clickhouse.Open(options)
+	conn := clickhouse.OpenDB(options)
+	err := conn.PingContext(context.Background())
+	if err != nil {
+		log.Logger.Warnf("clickhouse.Ping() error: %v", err)
+	}
+	return conn, err
 }
 
 func getTableColumns(conn driver.Conn, schema, table string) ([]engine.Record, error) {

core/src/plugins/clickhouse/delete.go

@@ -3,12 +3,17 @@
 import (
 	"context"
 	"fmt"
+	"github.com/clidey/whodb/core/src/common"
 	"strings"
 
 	"github.com/clidey/whodb/core/src/engine"
 )
 
 func (p *ClickHousePlugin) DeleteRow(config *engine.PluginConfig, schema string, storageUnit string, values map[string]string) (bool, error) {
+	readOnly := common.GetRecordValueOrDefault(config.Credentials.Advanced, readOnlyKey, "disable")
+	if readOnly != "disable" {
+		return false, fmt.Errorf("readonly mode don't allow DeleteRow")
+	}
 	conn, err := DB(config)
 	if err != nil {
 		return false, err
@@ -68,7 +73,7 @@
 		strings.Join(whereClauses, " AND "))
 
 	// Execute the query
-	err = conn.Exec(context.Background(), query, args...)
+	_, err = conn.ExecContext(context.Background(), query, args...)
 	if err != nil {
 		return false, fmt.Errorf("delete failed: %w (query: %s, args: %+v)", err, query, args)
 	}

core/src/plugins/clickhouse/query.go

@@ -28,13 +28,16 @@
 	}
 	defer conn.Close()
 
-	rows, err := conn.Query(context.Background(), query, params)
+	rows, err := conn.QueryContext(context.Background(), query, params)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
 
-	columnTypes := rows.ColumnTypes()
+	columnTypes, err := rows.ColumnTypes()
+	if err != nil {
+		return nil, err
+	}
 	result := &engine.GetRowsResult{
 		Columns: make([]engine.Column, len(columnTypes)),
 		Rows:    [][]string{},

core/src/plugins/clickhouse/update.go

@@ -3,12 +3,17 @@
 import (
 	"context"
 	"fmt"
+	"github.com/clidey/whodb/core/src/common"
 	"github.com/clidey/whodb/core/src/engine"
 	"strings"
 	"time"
 )
 
 func (p *ClickHousePlugin) UpdateStorageUnit(config *engine.PluginConfig, schema string, storageUnit string, values map[string]string) (bool, error) {
+	readOnly := common.GetRecordValueOrDefault(config.Credentials.Advanced, readOnlyKey, "disable")
+	if readOnly != "disable" {
+		return false, fmt.Errorf("readonly mode don't allow UpdateStorageUnit")
+	}
 	conn, err := DB(config)
 	if err != nil {
 		return false, err
@@ -99,7 +104,7 @@
 		strings.Join(setClauses, ", "),
 		strings.Join(whereClauses, " AND "))
 
-	err = conn.Exec(context.Background(), query, args...)
+	_, err = conn.ExecContext(context.Background(), query, args...)
 	if err != nil {
 		return false, fmt.Errorf("update failed: %w", err)
 	}