From ad831d0c206e2fe0db3b2f09e313863a69951014 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Wed, 10 Jan 2024 16:18:28 +0800 Subject: [PATCH] gjson --- go.mod | 3 +- util/json/json.go | 354 ++++++++++++++++++++++++++++++++++++++- util/json/json_test.go | 149 ++++++++++++++++ util/value/value.go | 27 +++ util/value/value_test.go | 10 ++ 5 files changed, 538 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 064c61129..0168ae11f 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,11 @@ require ( github.com/klauspost/pgzip v1.2.5 github.com/magiconair/properties v1.8.1 github.com/mattn/go-sqlite3 v1.14.16 - github.com/panjf2000/ants/v2 v2.9.0 github.com/pkg/errors v0.8.1 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.8.2 + github.com/tidwall/gjson v1.17.0 + github.com/tidwall/sjson v1.2.5 github.com/ulikunitz/xz v0.5.8 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/net v0.0.0-20201224014010-6772e930b67b diff --git a/util/json/json.go b/util/json/json.go index 4c58afc19..06ff49698 100644 --- a/util/json/json.go +++ b/util/json/json.go @@ -2,25 +2,351 @@ package gjson import ( "encoding/json" + "errors" "fmt" gcore "github.com/snail007/gmc/core" gcast "github.com/snail007/gmc/util/cast" + gvalue "github.com/snail007/gmc/util/value" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" "io" + "strconv" + "strings" ) +var ( + AddModifier = gjson.AddModifier + ModifierExists = gjson.ModifierExists + Escape = gjson.Escape + ForEachLine = gjson.ForEachLine + Parse = gjson.Parse + ParseBytes = gjson.ParseBytes + Valid = gjson.Valid + ValidBytes = gjson.ValidBytes +) + +type Options = sjson.Options + +type Result struct { + gjson.Result + path string + paths []string +} + +func (s Result) Path() string { + return s.path +} + +func (s Result) Paths() []string { + return s.paths +} + +func (s Result) AsJSONObject() *JSONObject { + obj := NewJSONObject(nil) + obj.json = s.Raw + return obj +} + +func (s Result) AsJSONArray() *JSONArray { + obj := NewJSONArray(nil) + obj.json = s.Raw + return obj +} + +type Builder struct { + json string +} + +func NewBuilderE(v interface{}) (*Builder, error) { + str, err := getJSONStr(v, "") + if err != nil { + return nil, err + } + return &Builder{json: str}, nil +} + +func NewBuilder(v interface{}) *Builder { + obj, _ := NewBuilderE(v) + return obj +} + +// Delete deletes a value from json for the specified path. +// path syntax: https://github.com/tidwall/sjson?tab=readme-ov-file#path-syntax +func (s *Builder) Delete(path string) error { + j, err := sjson.Delete(s.json, path) + if err == nil { + s.json = j + } + return err +} + +// Set sets a json value for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +// +// A path is a series of keys separated by a dot. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children.1" >> "Alex" +// +// path syntax: https://github.com/tidwall/sjson?tab=readme-ov-file#path-syntax +func (s *Builder) Set(path string, value interface{}) error { + j, err := sjson.Set(s.json, path, value) + if err == nil { + s.json = j + } + return err +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func (s *Builder) SetOptions(path string, value interface{}, opts *Options) error { + j, err := sjson.SetOptions(s.json, path, value, opts) + if err == nil { + s.json = j + } + return err +} + +// SetRaw sets a raw json value for the specified path. +// This function works the same as Set except that the value is set as a +// raw block of json. This allows for setting premarshalled json objects. +func (s *Builder) SetRaw(path, value string) error { + j, err := sjson.SetRaw(s.json, path, value) + if err == nil { + s.json = j + } + return err +} + +// SetRawOptions sets a raw json value for the specified path with options. +// This furnction works the same as SetOptions except that the value is set +// as a raw block of json. This allows for setting premarshalled json objects. +func (s *Builder) SetRawOptions(path, value string, opts *Options) error { + j, err := sjson.SetRawOptions(s.json, path, value, opts) + if err == nil { + s.json = j + } + return err +} + +// Get searches json for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// When the value is found it's returned immediately. +// +// A path is a series of keys separated by a dot. +// A key may contain special wildcard characters '*' and '?'. +// To access an array value use the index as the key. +// To get the number of elements in an array or to access a child path, use +// the '#' character. +// The dot and wildcard character can be escaped with '\'. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children" >> ["Sara","Alex","Jack"] +// "children.#" >> 3 +// "children.1" >> "Alex" +// "child*.2" >> "Jack" +// "c?ildren.0" >> "Sara" +// "friends.#.first" >> ["James","Roger"] +// +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// If you are consuming JSON from an unpredictable source then you may want to +// use the Valid function first. +// path syntax: https://github.com/tidwall/gjson/blob/master/SYNTAX.md +func (s *Builder) Get(path string) Result { + r := gjson.Get(s.json, path) + return Result{ + paths: r.Paths(s.json), + path: r.Path(s.json), + Result: r, + } +} + +// String get the json string of *Builder +func (s *Builder) String() string { + return s.json +} + +// AsJSONObject convert the *Builder to *JSONObject, +// if the *Builder is not a json object, nil returned. +func (s *Builder) AsJSONObject() *JSONObject { + if s.json != "" && !strings.HasPrefix(s.json, "{") { + return nil + } + return NewJSONObject(s.json) +} + +// AsJSONArray convert the *Builder to *AsJSONArray, +// if the *Builder is not a json array, nil returned. +func (s *Builder) AsJSONArray() *JSONArray { + if s.json != "" && !strings.HasPrefix(s.json, "[") { + return nil + } + return NewJSONArray(s.json) +} + +// GetMany batch of Get +func (s *Builder) GetMany(path ...string) []Result { + rs1 := gjson.GetMany(s.json, path...) + var rs []Result + for _, r := range rs1 { + rs = append(rs, + Result{ + Result: r, + path: r.Path(s.json), + paths: r.Paths(s.json), + }) + } + return rs +} + +type JSONObject struct { + *Builder +} + +// NewJSONObjectE create a *JSONObject form v, returned error(if have) +// v can be json object content of []byte and string, or any data which json.Marshal can be processed. +func NewJSONObjectE(v interface{}) (*JSONObject, error) { + str, err := getJSONStr(v, "{}") + if err != nil { + return nil, err + } + if str != "" && !strings.HasPrefix(str, "{") { + return nil, errors.New("fail to convert v to json array") + } + return &JSONObject{ + Builder: NewBuilder(str), + }, nil +} + +// NewJSONObject create a *JSONObject form v, if error occurred nil returned. +// v can be json object content of []byte and string, or any data which json.Marshal can be processed. +func NewJSONObject(v interface{}) *JSONObject { + obj, _ := NewJSONObjectE(v) + return obj +} + +type JSONArray struct { + *Builder +} + +// NewJSONArrayE create a *JSONArray form v, returned error(if have) +// v can be json array content of []byte and string, or any data which json.Marshal can be processed. +func NewJSONArrayE(v interface{}) (*JSONArray, error) { + str, err := getJSONStr(v, "[]") + if err != nil { + return nil, err + } + if str != "" && !strings.HasPrefix(str, "[") { + return nil, errors.New("fail to convert v to json array") + } + return &JSONArray{ + Builder: NewBuilder(str), + }, nil +} + +// NewJSONArray create a *JSONArray form v, if error occurred nil returned. +// v can be json array content of []byte and string, or any data which json.Marshal can be processed. +func NewJSONArray(v interface{}) *JSONArray { + obj, _ := NewJSONArrayE(v) + return obj +} + +// Merge *JSONArray, JSONArray or any valid slice to s +func (s *JSONArray) Merge(arr interface{}) (err error) { + var merge = func(a *JSONArray) { + a.Get("@this").ForEach(func(key, value gjson.Result) bool { + err = s.Append(value.Value()) + return err == nil + }) + } + switch a := arr.(type) { + case *JSONArray: + merge(a) + case JSONArray: + merge(&a) + default: + err = s.Append(gvalue.NewAny(arr).Slice()...) + } + return +} + +func (s *JSONArray) Append(values ...interface{}) (err error) { + for _, value := range values { + switch v := value.(type) { + case *JSONObject: + err = s.SetRaw("-1", v.json) + case JSONObject: + err = s.SetRaw("-1", v.json) + case *JSONArray: + err = s.SetRaw("-1", v.json) + case JSONArray: + err = s.SetRaw("-1", v.json) + default: + err = s.Set("-1", value) + } + if err != nil { + return + } + } + return nil +} + +func (s *JSONArray) Len() int64 { + return s.Get("#").Int() +} + +func (s *JSONArray) Last() Result { + idx := s.Get("#").Int() - 1 + if idx < 0 { + idx = 0 + } + return s.Get(strconv.FormatInt(idx, 10)) +} + +func (s *JSONArray) First() Result { + return s.Get("0") +} + type JSONResult struct { data map[string]interface{} ctx gcore.Ctx } -//NewResultCtx Optional args: code int, message string, data interface{} +// NewResultCtx Optional args: code int, message string, data interface{} func NewResultCtx(ctx gcore.Ctx, d ...interface{}) *JSONResult { r := NewResult(d...) r.ctx = ctx return r } -//NewResult Optional args: code int, message string, data interface{} +// NewResult Optional args: code int, message string, data interface{} func NewResult(d ...interface{}) *JSONResult { if len(d) == 1 { var b []byte @@ -109,7 +435,7 @@ func (s *JSONResult) WriteToCtx(ctx gcore.Ctx) (err error) { return } -//Success only worked with NewResultCtx() +// Success only worked with NewResultCtx() func (s *JSONResult) Success(d ...interface{}) (err error) { var data interface{} if len(d) == 1 { @@ -118,7 +444,27 @@ func (s *JSONResult) Success(d ...interface{}) (err error) { return s.SetData(data).WriteToCtx(s.ctx) } -//Fail only worked with NewResultCtx() +// Fail only worked with NewResultCtx() func (s *JSONResult) Fail(format string, v ...interface{}) (err error) { return s.SetCode(1).SetMessage(format, v...).WriteToCtx(s.ctx) } + +func getJSONStr(v interface{}, nilValue string) (string, error) { + if gvalue.IsNil(v) { + return nilValue, nil + } + var str string + switch val := v.(type) { + case string: + str = val + case []byte: + str = string(val) + default: + b, _ := json.Marshal(v) + str = string(b) + } + if !Valid(str) { + return "", errors.New("fail to convert to invalid json") + } + return strings.Trim(str, " \r\n\t"), nil +} diff --git a/util/json/json_test.go b/util/json/json_test.go index d09babf35..1f167678b 100644 --- a/util/json/json_test.go +++ b/util/json/json_test.go @@ -174,3 +174,152 @@ func TestJSONResult_Fail(t *testing.T) { expected := `{"code":1,"data":null,"message":"fail"}` assert.Equal(t, expected, recorder.Body.String()) } + +func TestBuilderOperations(t *testing.T) { + // 创建一个新的 Builder 实例 + builder := NewBuilder(`{"name": "John", "age": 30, "city": "New York"}`) + + // 测试 Set 方法 + err := builder.Set("name", "Doe") + if err != nil { + t.Errorf("Set method failed: %v", err) + } + + // 测试 Delete 方法 + err = builder.Delete("age") + if err != nil { + t.Errorf("Delete method failed: %v", err) + } + + // 测试 SetRaw 方法 + err = builder.SetRaw("country", `"USA"`) + if err != nil { + t.Errorf("SetRaw method failed: %v", err) + } + + // 测试 Get 方法 + result := builder.Get("name") + if result.String() != "Doe" { + t.Errorf("Get method failed, expected 'Doe', got '%s'", result.String()) + } + assert.Equal(t, result.Path(), "name") + + // 测试 GetMany 方法 + results := builder.GetMany("name", "country") + if len(results) != 2 { + t.Errorf("GetMany method failed, expected 2 results, got %d", len(results)) + } + if results[1].String() != "USA" { + t.Errorf("GetMany method failed, expected 'USA', got '%s'", results[1].String()) + } + assert.Nil(t, result.Paths()) +} + +func TestBuilderAdditionalOperations(t *testing.T) { + // 创建一个新的 Builder 实例 + builder := NewBuilder(`{"name": "John", "age": 30, "city": "New York"}`) + + // 测试 SetOptions 方法 + opts := &Options{Optimistic: false} + err := builder.SetOptions("address", "123 Main St", opts) + if err != nil { + t.Errorf("SetOptions method failed: %v", err) + } + + // 测试 SetRawOptions 方法 + rawOpts := &Options{Optimistic: false} + err = builder.SetRawOptions("info", `{"key": "value"}`, rawOpts) + if err != nil { + t.Errorf("SetRawOptions method failed: %v", err) + } +} + +func TestJSONArray_Append(t *testing.T) { + arr := NewJSONArray("[123]") + assert.Equal(t, "123", arr.Get("0").String()) + + obj := NewJSONObject(map[string]string{"name": "456"}) + arr.Append(obj) + assert.Equal(t, "456", arr.Get("1.name").String()) + assert.Equal(t, "456", arr.Get("1").AsJSONObject().Get("name").String()) + + obj = NewJSONObject(nil) + obj.Set("name", "789") + arr.Append(*obj) + assert.Equal(t, "789", arr.Get("2.name").String()) + + obja := NewJSONArray(nil) + obja.Append("000", "111") + arr.Append(obja) + assert.Equal(t, "000", arr.Get("3.0").String()) + assert.Equal(t, "111", arr.Get("3.1").String()) + assert.Equal(t, "000", arr.Get("3").AsJSONArray().Get("0").String()) + + assert.Equal(t, int64(4), arr.Len()) + + obja = NewJSONArray([]string{"0000", "1111"}) + arr.Append(*obja) + assert.Equal(t, "0000", arr.Get("4.0").String()) + assert.Equal(t, "1111", arr.Get("4.1").String()) + assert.Equal(t, "0000", arr.Get("4").AsJSONArray().Get("0").String()) + + obj = NewJSONObject(`{"name":"111"}`) + arr.Append(obj) + assert.Equal(t, "111", arr.Get("5.name").String()) + + obj = NewJSONObject([]byte(`{"name":"222"}`)) + arr.Append(obj) + assert.Equal(t, "222", arr.Get("6.name").String()) + + assert.Nil(t, NewJSONObject("{,abc")) + assert.Nil(t, NewJSONArray("{,abc")) + assert.Nil(t, NewBuilder("{,abc")) +} + +func TestJSONArray_Merge(t *testing.T) { + a := NewJSONArray([]int{123}) + arr := NewJSONArray(nil) + assert.Nil(t, arr.Merge(a)) + assert.Nil(t, arr.Merge(*a)) + assert.Nil(t, arr.Merge([]string{"abc", "111"})) + assert.Equal(t, int64(123), arr.Get("0").Int()) + assert.Equal(t, int64(123), arr.Get("1").Int()) + assert.Equal(t, "abc", arr.Get("2").String()) + assert.Equal(t, "111", arr.Get("3").String()) +} + +func TestBuilder_AsJSONObject(t *testing.T) { + a := NewBuilder(`[]`) + assert.Nil(t, a.AsJSONObject()) + assert.NotNil(t, a.AsJSONArray()) + assert.Equal(t, "[]", a.String()) + assert.Error(t, a.AsJSONArray().Append(http.Client{})) + a = NewBuilder(`{}`) + assert.Nil(t, a.AsJSONArray()) + assert.NotNil(t, a.AsJSONObject()) + assert.Equal(t, "{}", a.String()) +} + +func TestJSONArray_Last(t *testing.T) { + a := NewJSONArray(nil) + assert.False(t, a.First().Exists()) + assert.False(t, a.Last().Exists()) + assert.Empty(t, a.First().String()) + assert.Empty(t, a.Last().String()) + + a.Append("123") + assert.Equal(t, "123", a.First().String()) + assert.Equal(t, "123", a.Last().String()) + + a.Append("456") + assert.Equal(t, "123", a.First().String()) + assert.Equal(t, "456", a.Last().String()) + +} + +func TestNewJSONObjectE(t *testing.T) { + _, err := NewJSONObjectE([]string{}) + assert.Error(t, err) + _, err = NewJSONArrayE(map[string]string{}) + assert.Error(t, err) +} diff --git a/util/value/value.go b/util/value/value.go index 686af1f4f..2f2516635 100644 --- a/util/value/value.go +++ b/util/value/value.go @@ -118,6 +118,7 @@ type Value struct { cacheFloat64 *float64 cacheString *string cacheStringSlice []string + cacheSlice []interface{} cacheBool *bool cacheBytes []byte cacheDuration *time.Duration @@ -319,6 +320,18 @@ func (s *Value) StringSlice() []string { return v } +func (s *Value) Slice() []interface{} { + if s.val == nil { + return nil + } + if s.cacheSlice != nil { + return s.cacheSlice + } + v := s.val.([]interface{}) + s.cacheSlice = v + return v +} + func (s *Value) Duration() time.Duration { if s.val == nil { return 0 @@ -522,6 +535,7 @@ type AnyValue struct { cacheFloat32 *float32 cacheFloat64 *float64 + cacheSlice []interface{} cacheIntSlice []int cacheInt8Slice []int8 cacheInt32Slice []int32 @@ -584,6 +598,19 @@ func (s *AnyValue) Bytes() []byte { return s.cacheBytes } +func (s *AnyValue) Slice() []interface{} { + if s.val == nil { + return nil + } + if s.cacheSlice != nil { + return s.cacheSlice + } + walkSlice(s.val, func(v interface{}) { + s.cacheSlice = append(s.cacheSlice, v) + }) + return s.cacheSlice +} + func (s *AnyValue) Int() int { if s.val == nil { return 0 diff --git a/util/value/value_test.go b/util/value/value_test.go index 909b65a80..0b0a788d1 100644 --- a/util/value/value_test.go +++ b/util/value/value_test.go @@ -127,6 +127,12 @@ func TestValue_xxx(t *testing.T) { assert.NotNil(t, t1.cacheStringSlice) assert.Equal(t, t0, t1.StringSlice()) + t0 = []interface{}{"123"} + t1 = New(t0) + assert.Equal(t, t0, t1.Slice()) + assert.NotNil(t, t1.cacheSlice) + assert.Equal(t, t0, t1.Slice()) + t0 = []byte("123") t1 = New(t0) assert.Equal(t, t0, t1.Bytes()) @@ -253,6 +259,7 @@ func TestValue_xxx2(t *testing.T) { assert.Nil(t, val.MapSlice()) assert.Nil(t, val.MapStringSlice()) assert.Nil(t, val.StringSlice()) + assert.Nil(t, val.Slice()) assert.Nil(t, val.MapUint64()) assert.Nil(t, val.MapUint32()) assert.Nil(t, val.MapUint8()) @@ -300,6 +307,7 @@ func TestAnyValue_xxx(t *testing.T) { assert.Nil(t, val.Map()) assert.Nil(t, val.MapFloat64()) assert.Nil(t, val.MapFloat32()) + assert.Nil(t, val.Slice()) assert.Nil(t, val.IntSlice()) assert.Nil(t, val.Int8Slice()) assert.Nil(t, val.Int32Slice()) @@ -360,6 +368,7 @@ func TestAnyValue(t *testing.T) { assert.Equal(t, val.Bytes(), []byte("10")) val = NewAny([]int{10}) + assert.Equal(t, val.Slice(), []interface{}{10}) assert.Equal(t, val.IntSlice(), []int{10}) assert.Equal(t, val.Int8Slice(), []int8{10}) assert.Equal(t, val.Int32Slice(), []int32{10}) @@ -374,6 +383,7 @@ func TestAnyValue(t *testing.T) { assert.Equal(t, val.StringSlice(), []string{"10"}) assert.Equal(t, val.DurationSlice(), []time.Duration{time.Duration(10)}) + assert.Equal(t, val.Slice(), []interface{}{10}) assert.Equal(t, val.IntSlice(), []int{10}) assert.Equal(t, val.Int8Slice(), []int8{10}) assert.Equal(t, val.Int32Slice(), []int32{10})