reduced dependencies

common/cache.go

@@ -0,0 +1,81 @@
+package common
+
+import (
+	"sync"
+	"time"
+)
+
+type CacheMap[T any] struct {
+	value map[string]T
+	err map[string]error
+	lastUse map[string]time.Time
+	mu sync.Mutex
+	null T
+}
+
+func NewCache[T any]() CacheMap[T] {
+	return CacheMap[T]{
+		value: map[string]T{},
+		err: map[string]error{},
+		lastUse: map[string]time.Time{},
+	}
+}
+
+// get returns a value or an error if it exists
+//
+// if the object key does not exist, it will return both a nil/zero value (of the relevant type) and nil error
+func (cache *CacheMap[T]) Get(key string) (T, error) {
+	cache.mu.Lock()
+	defer cache.mu.Unlock()
+
+	if err, ok := cache.err[key]; ok {
+		cache.lastUse[key] = time.Now()
+		return cache.null, err
+	}else if val, ok := cache.value[key]; ok {
+		cache.lastUse[key] = time.Now()
+		return val, nil
+	}
+
+	return cache.null, nil
+}
+
+// set sets or adds a new key with either a value, or an error
+func (cache *CacheMap[T]) Set(key string, value T, err error) {
+	cache.mu.Lock()
+	defer cache.mu.Unlock()
+
+	if err != nil {
+		cache.err[key] = err
+		delete(cache.value, key)
+		cache.lastUse[key] = time.Now()
+	}else{
+		cache.value[key] = value
+		delete(cache.err, key)
+		cache.lastUse[key] = time.Now()
+	}
+}
+
+// delOld removes old cache items
+func (cache *CacheMap[T]) DelOld(cacheTime time.Duration){
+	cache.mu.Lock()
+	defer cache.mu.Unlock()
+
+	if cacheTime == 0 {
+		for key := range cache.lastUse {
+			delete(cache.value, key)
+			delete(cache.err, key)
+			delete(cache.lastUse, key)
+		}
+		return
+	}
+
+	now := time.Now().UnixNano()
+
+	for key, lastUse := range cache.lastUse {
+		if now - lastUse.UnixNano() > int64(cacheTime) {
+			delete(cache.value, key)
+			delete(cache.err, key)
+			delete(cache.lastUse, key)
+		}
+	}
+}

common/common.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"math"
+	"syscall"
 )
 
 // JoinBytes is an easy way to join multiple values into a single []byte
@@ -13,7 +14,16 @@ func JoinBytes(bytes ...interface{}) []byte {
 	return res
 }
 
-// formatMemoryUsage converts bytes to megabytes
-func FormatMemoryUsage(b uint64) float64 {
-	return math.Round(float64(b) / 1024 / 1024 * 100) / 100
+// SysFreeMemory returns the amount of memory available in megabytes
+func SysFreeMemory() float64 {
+	in := &syscall.Sysinfo_t{}
+	err := syscall.Sysinfo(in)
+	if err != nil {
+		return 0
+	}
+
+	// If this is a 32-bit system, then these fields are
+	// uint32 instead of uint64.
+	// So we always convert to uint64 to match signature.
+	return math.Round(float64(uint64(in.Freeram) * uint64(in.Unit)) / 1024 / 1024 * 100) / 100
 }

go.mod

@@ -2,14 +2,4 @@ module github.com/AspieSoft/go-regex/v8
 
 go 1.18
 
-require (
-	github.com/AspieSoft/go-syncterval v1.0.5
-	github.com/AspieSoft/go-ttlcache v1.2.2
-	github.com/GRbit/go-pcre v1.0.1
-	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
-)
-
-require (
-	github.com/alphadose/haxmap v1.3.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-)
+require github.com/GRbit/go-pcre v1.0.1

go.sum

@@ -1,12 +1,2 @@
-github.com/AspieSoft/go-syncterval v1.0.5 h1:fzSNZofSX/7iBkLrWizMzGLGcBl3d+76cHyr8M9tjGg=
-github.com/AspieSoft/go-syncterval v1.0.5/go.mod h1:r+mTZPOWvfS0Y5YAxKINGNt8eX76i2Lib0VeqAw3SW4=
-github.com/AspieSoft/go-ttlcache v1.2.2 h1:U6MMYY5PKANsB4/vhTiQsQ2U/y1P1yqMCxLw+qFNmCo=
-github.com/AspieSoft/go-ttlcache v1.2.2/go.mod h1:czwXaDat1SKWGJDXkoMZWZFip97MXxuU0KzqJ9zVLDo=
 github.com/GRbit/go-pcre v1.0.1 h1:8F7Wj1rxIq8ejKSXVVW2wE+4I4VnZbuOemrMk8kn3hc=
 github.com/GRbit/go-pcre v1.0.1/go.mod h1:0g7qVGbMpd2Odevd92x1RpaLpR3c3F/Gv2HEnI7CwEA=
-github.com/alphadose/haxmap v1.3.0 h1:C/2LboOnPCZP27GmmSXOcwx360st0P8N0fTJ3voefKc=
-github.com/alphadose/haxmap v1.3.0/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM=
-github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
-github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=

regex.go

@@ -10,10 +10,7 @@ import (
 	"time"
 
 	"github.com/AspieSoft/go-regex/v8/common"
-	"github.com/AspieSoft/go-syncterval"
-	"github.com/AspieSoft/go-ttlcache"
 	"github.com/GRbit/go-pcre"
-	"github.com/pbnjay/memory"
 )
 
 type PCRE pcre.Regexp
@@ -38,27 +35,63 @@ var regCompBGRef *regexp.Regexp = regexp.MustCompile(`%!([0-9]+|o|c)!%`)
 var regComplexSel *Regexp
 var regEscape *Regexp
 
-var cache *ttlcache.Cache[string, *Regexp] = ttlcache.New[string, *Regexp](2 * time.Hour, 1 * time.Hour)
-var compCache *ttlcache.Cache[string, []byte] = ttlcache.New[string, []byte](2 * time.Hour, 1 * time.Hour)
+// var cache *ttlcache.Cache[string, *Regexp] = ttlcache.New[string, *Regexp](2 * time.Hour, 1 * time.Hour)
+var cache common.CacheMap[*Regexp] = common.NewCache[*Regexp]()
+var compCache common.CacheMap[[]byte] = common.NewCache[[]byte]()
 
 func init() {
 	regComplexSel = Comp(`(\\|)\$([0-9]|\{[0-9]+\})`)
 	regEscape = Comp(`[\\\^\$\.\|\?\*\+\(\)\[\]\{\}\%]`)
 
 	go func(){
-		// clear cache items older than 10 minutes if there are only 200MB of free memory
-		syncterval.New(10 * time.Second, func() {
-			if common.FormatMemoryUsage(memory.FreeMemory()) < 200 {
-				cache.ClearEarly(10 * time.Minute)
-				compCache.ClearEarly(5 * time.Minute)
+		for {
+			time.Sleep(10 * time.Minute)
+
+			// default: remove cache items have not been accessed in over 2 hours
+			cacheTime := 2 * time.Hour
+
+			// SysFreeMemory returns the total free system memory in megabytes
+			mb := common.SysFreeMemory()
+			if mb < 200 && mb != 0 {
+				// low memory: remove cache items have not been accessed in over 10 minutes
+				cacheTime = 10 * time.Minute
+			}else if mb < 500 && mb != 0 {
+				// low memory: remove cache items have not been accessed in over 30 minutes
+				cacheTime = 30 * time.Minute
+			}else if mb < 2000 && mb != 0 {
+				// low memory: remove cache items have not been accessed in over 1 hour
+				cacheTime = 1 * time.Hour
+			}else if mb > 64000 {
+				// high memory: remove cache items have not been accessed in over 12 hour
+				cacheTime = 12 * time.Hour
+			}else if mb > 32000 {
+				// high memory: remove cache items have not been accessed in over 6 hour
+				cacheTime = 6 * time.Hour
+			}else if mb > 16000 {
+				// high memory: remove cache items have not been accessed in over 3 hour
+				cacheTime = 3 * time.Hour
 			}
-		})
+
+			cache.DelOld(cacheTime)
+			compCache.DelOld(cacheTime)
+
+			time.Sleep(10 * time.Second)
+
+			// clear cache if were still critically low on available memory
+			if mb := common.SysFreeMemory(); mb < 10 && mb != 0 {
+				cache.DelOld(0)
+				compCache.DelOld(0)
+			}
+		}
 	}()
 }
 
 // this method compiles the RE string to add more functionality to it
 func compRE(re string, params []string) string {
-	if val, ok := compCache.Get(re); ok {
+	if val, err := compCache.Get(re); val != nil || err != nil {
+		if err != nil {
+			return ""
+		}
 		return string(regCompParam.ReplaceAllFunc(val, func(b []byte) []byte {
 			if b[1] == '{' && b[len(b)-1] == '}' {
 				b = b[2:len(b)-1]
@@ -162,7 +195,7 @@ func compRE(re string, params []string) string {
 		return []byte{}
 	})
 
-	compCache.Set(re, reB)
+	compCache.Set(re, reB, nil)
 
 	return string(regCompParam.ReplaceAllFunc(reB, func(b []byte) []byte {
 		if b[1] == '{' && b[len(b)-1] == '}' {
@@ -185,49 +218,56 @@ func compRE(re string, params []string) string {
 func Comp(re string, params ...string) *Regexp {
 	re = compRE(re, params)
 
-	if val, ok := cache.Get(re); ok {
+	if val, err := cache.Get(re); val != nil || err != nil {
+		if err != nil {
+			panic(err)
+		}
 		return val
-	} else {
-		reg := pcre.MustCompile(re, pcre.UTF8)
+	}
 
-		// commented below methods compiled 10000 times in 0.1s (above method being used finished in half of that time)
-		// reg := pcre.MustCompileParse(re)
-		// reg := pcre.MustCompileJIT(re, pcre.UTF8, pcre.STUDY_JIT_COMPILE)
-		// reg := pcre.MustCompileJIT(re, pcre.EXTRA, pcre.STUDY_JIT_COMPILE)
-		// reg := pcre.MustCompileJIT(re, pcre.JAVASCRIPT_COMPAT, pcre.STUDY_JIT_COMPILE)
-		// reg := pcre.MustCompileParseJIT(re, pcre.STUDY_JIT_COMPILE)
+	reg := pcre.MustCompile(re, pcre.UTF8)
 
-		compRe := Regexp{RE: reg, len: int64(len(re))}
+	// commented below methods compiled 10000 times in 0.1s (above method being used finished in half of that time)
+	// reg := pcre.MustCompileParse(re)
+	// reg := pcre.MustCompileJIT(re, pcre.UTF8, pcre.STUDY_JIT_COMPILE)
+	// reg := pcre.MustCompileJIT(re, pcre.EXTRA, pcre.STUDY_JIT_COMPILE)
+	// reg := pcre.MustCompileJIT(re, pcre.JAVASCRIPT_COMPAT, pcre.STUDY_JIT_COMPILE)
+	// reg := pcre.MustCompileParseJIT(re, pcre.STUDY_JIT_COMPILE)
 
-		cache.Set(re, &compRe)
-		return &compRe
-	}
+	compRe := Regexp{RE: reg, len: int64(len(re))}
+
+	cache.Set(re, &compRe, nil)
+	return &compRe
 }
 
 // CompTry tries to compile or returns an error
 func CompTry(re string, params ...string) (*Regexp, error) {
 	re = compRE(re, params)
 
-	if val, ok := cache.Get(re); ok {
-		return val, nil
-	} else {
-		reg, err := pcre.Compile(re, pcre.UTF8)
+	if val, err := cache.Get(re); val != nil || err != nil {
 		if err != nil {
 			return &Regexp{}, err
 		}
+		return val, nil
+	}
 
-		// commented below methods compiled 10000 times in 0.1s (above method being used finished in half of that time)
-		// reg := pcre.MustCompileParse(re)
-		// reg := pcre.MustCompileJIT(re, pcre.UTF8, pcre.STUDY_JIT_COMPILE)
-		// reg := pcre.MustCompileJIT(re, pcre.EXTRA, pcre.STUDY_JIT_COMPILE)
-		// reg := pcre.MustCompileJIT(re, pcre.JAVASCRIPT_COMPAT, pcre.STUDY_JIT_COMPILE)
-		// reg := pcre.MustCompileParseJIT(re, pcre.STUDY_JIT_COMPILE)
+	reg, err := pcre.Compile(re, pcre.UTF8)
+	if err != nil {
+		cache.Set(re, nil, err)
+		return &Regexp{}, err
+	}
 
-		compRe := Regexp{RE: reg, len: int64(len(re))}
+	// commented below methods compiled 10000 times in 0.1s (above method being used finished in half of that time)
+	// reg := pcre.MustCompileParse(re)
+	// reg := pcre.MustCompileJIT(re, pcre.UTF8, pcre.STUDY_JIT_COMPILE)
+	// reg := pcre.MustCompileJIT(re, pcre.EXTRA, pcre.STUDY_JIT_COMPILE)
+	// reg := pcre.MustCompileJIT(re, pcre.JAVASCRIPT_COMPAT, pcre.STUDY_JIT_COMPILE)
+	// reg := pcre.MustCompileParseJIT(re, pcre.STUDY_JIT_COMPILE)
 
-		cache.Set(re, &compRe)
-		return &compRe, nil
-	}
+	compRe := Regexp{RE: reg, len: int64(len(re))}
+
+	cache.Set(re, &compRe, nil)
+	return &compRe, nil
 }