Skip to content

Commit

Permalink
Experimental implementation of a reference counted file.close method
Browse files Browse the repository at this point in the history
The main issue with this implementation is that as k6 does not yet
support opening files in the VU context, the ref count for a given
file will never reach zero as the VU 0 (init context) holds
one reference until the end of the test.

The whole idea behind reference counted closing was that in a context
with multiple scenarios, once all the instances of a file were explicitly
closed by the user, we would be able to release the memory, regardless
of what over workloads might be happening next.
  • Loading branch information
oleiade committed Sep 8, 2023
1 parent dc121fb commit de8ed6e
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 3 deletions.
17 changes: 17 additions & 0 deletions examples/experimental/fs/fs-openclose.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { open } from "k6/experimental/fs";
import { sleep } from "k6";

export const options = {
vus: 10,
iterations: 10,
};

let file;
(async function () {
file = await open("bonjour.txt");
})();

export default async function () {
file.close();
sleep(10);
}
51 changes: 51 additions & 0 deletions js/modules/k6/experimental/fs/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,14 @@ type cache struct {
// Importantly, this also means that if the
// file is modified from outside of k6, the changes will not be reflected in the file's data.
openedFiles sync.Map

// refCounts holds a safe for concurrent use map, holding the number of times
// a file was opened.
openedRefCounts map[string]uint

// mutex is used to ensure that the openedRefCounts map is not accessed concurrently.
// We resort to using a mutex as opposed to a sync.Map because we need to be able to
mutex sync.Mutex
}

// open retrieves the content of a given file from the specified filesystem (fromFs) and
Expand Down Expand Up @@ -56,6 +64,12 @@ func (fr *cache) open(filename string, fromFs afero.Fs) ([]byte, error) {
panic(fmt.Errorf("registry's file %s is not stored as a byte slice", filename))
}

// Increase the ref count.
fr.mutex.Lock()
fr.openedRefCounts[filename]++
fmt.Println("cache.open[file already loaded]: openedRefCount=", fr.openedRefCounts[filename])
fr.mutex.Unlock()

return data, nil
}

Expand All @@ -76,5 +90,42 @@ func (fr *cache) open(filename string, fromFs afero.Fs) ([]byte, error) {

fr.openedFiles.Store(filename, data)

// As this is the first time we open the file, initialize
// the ref count to 1.
fr.mutex.Lock()
fr.openedRefCounts[filename] = 1
fmt.Println("cache.open[initializing open]: openedRefCount=", 1)
fr.mutex.Unlock()

return data, nil
}

func (fr *cache) close(filename string) error {
fr.mutex.Lock()
defer fr.mutex.Unlock()

// If no ref count is found, it means the file was either never
// opened, or already closed.
if _, ok := fr.openedRefCounts[filename]; !ok {
return newFsError(ClosedError, fmt.Sprintf("file %s is not opened", filename))
}

// If the ref count is 0, it means the file was already closed.
if fr.openedRefCounts[filename] == 0 {
return newFsError(ClosedError, fmt.Sprintf("file %s is already closed", filename))
}

// Decrease the ref count.
fr.openedRefCounts[filename]--
fmt.Println("cache.close: openedRefCount=", fr.openedRefCounts[filename])

// If the ref count is 0, it means the file was closed for the last time.
// We can safely remove it from the openedFiles map.
if fr.openedRefCounts[filename] == 0 {
fr.openedFiles.Delete(filename)
delete(fr.openedRefCounts, filename)
fmt.Println("cache.close: last openedRefCount=", fr.openedRefCounts[filename])
}

return nil
}
4 changes: 4 additions & 0 deletions js/modules/k6/experimental/fs/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ const (

// TypeError is emitted when an incorrect type has been used.
TypeError

// ClosedError is emitted when an operation is attempted on a closed
// file.
ClosedError
)

// fsError represents a custom error object emitted by the fs module.
Expand Down
32 changes: 32 additions & 0 deletions js/modules/k6/experimental/fs/file.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package fs

import (
"fmt"
"path/filepath"
)

Expand All @@ -10,6 +11,20 @@ type file struct {

// data holds a pointer to the file's data
data []byte

// isClosed indicates whether the file is closed or not.
isClosed bool

// closerFunc holds a function that can be used to close the file.
//
// This is used to ensure that whatever operation necessary to cleanly
// close the file is performed.
//
// In our case, the [closerFunc] is provided by the [cache] struct, and
// it ensures the file's reference count is decreased on close, and that,
// if relevant (the ref count is zero), the cached file content is released
// from memory.
closerFunc func(string) error
}

// Stat returns a FileInfo describing the named file.
Expand All @@ -26,3 +41,20 @@ type FileInfo struct {
// Size holds the size of the file in bytes.
Size int `json:"size"`
}

func (f *file) close() error {
if f.isClosed {
return nil
}

if f.closerFunc == nil {
return fmt.Errorf("file %s has no closer function", f.path)
}

if err := f.closerFunc(f.path); err != nil {
return err
}
f.isClosed = true

return nil
}
14 changes: 11 additions & 3 deletions js/modules/k6/experimental/fs/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ var (
// New returns a pointer to a new [RootModule] instance.
func New() *RootModule {
return &RootModule{
cache: &cache{},
cache: &cache{openedRefCounts: make(map[string]uint)},
}
}

Expand Down Expand Up @@ -123,8 +123,9 @@ func (mi *ModuleInstance) openImpl(path string) (*File, error) {
return &File{
Path: path,
file: file{
path: path,
data: data,
path: path,
data: data,
closerFunc: mi.cache.close,
},
vu: mi.vu,
registry: mi.cache,
Expand Down Expand Up @@ -165,3 +166,10 @@ func (f *File) Stat() *goja.Promise {

return promise
}

// Close closes the file.
func (f *File) Close() {
if err := f.file.close(); err != nil {
common.Throw(f.vu.Runtime(), err)
}
}

0 comments on commit de8ed6e

Please sign in to comment.