🚀 [Feature]: ObjectBox storage support

[karnadii]

Feature Description

implement driver for ObjectBox from github.com/objectbox/objectbox-go

Additional Context (optional)

No response

Code Snippet (optional)

package main

import "github.com/gofiber/storage/%package%"

func main() {
  // Steps to reproduce
}

Checklist:

• I agree to follow Fiber's Code of Conduct.
• I have checked for existing issues that describe my suggestion prior to opening this one.
• I understand that improperly formatted feature requests may be closed without explanation.

[karnadii]

I have make some small implementation
but I just learned go two days ago so I don't know what the best practice is

package middleware

import (
	"math/rand/v2"
	"time"

	"github.com/objectbox/objectbox-go/objectbox"
)

//go:generate go run github.com/objectbox/objectbox-go/cmd/objectbox-gogen

type CacheEntry struct {
	Id        uint64 `objectbox:"id"`
	Key       string `objectbox:"index"`
	Value     []byte
	ExpiresAt int64
}

type ObjectBoxStorage struct {
	ob  *objectbox.ObjectBox
	box *CacheEntryBox
}

func NewObjectBoxStorage() (*ObjectBoxStorage, error) {
	ob, err := objectbox.NewBuilder().Model(ObjectBoxModel()).Build()
	if err != nil {
		return nil, err
	}

	storage := &ObjectBoxStorage{
		ob:  ob,
		box: BoxForCacheEntry(ob),
	}

	// Run cleanup every hour
	go func() {
		ticker := time.NewTicker(1 * time.Hour)
		for range ticker.C {
			storage.cleanupExpired()
		}
	}()

	return storage, nil
}

func (s *ObjectBoxStorage) Get(key string) ([]byte, error) {
	if rand.Float32() < 0.1 {
		s.cleanupExpired()
	}

	query := s.box.Query(CacheEntry_.Key.Equals(key, true), CacheEntry_.ExpiresAt.GreaterThan(time.Now().Unix()))
	entries, err := query.Find()
	if err != nil {
		return nil, err
	}

	if len(entries) == 0 {
		return nil, nil
	}
	return entries[0].Value, nil

}

func (s *ObjectBoxStorage) Set(key string, val []byte, exp time.Duration) error {
	entry := &CacheEntry{
		Key:       key,
		Value:     val,
		ExpiresAt: time.Now().Add(exp).Unix(),
	}
	_, err := s.box.Put(entry)
	return err
}

func (s *ObjectBoxStorage) Delete(key string) error {
	query := s.box.Query(CacheEntry_.Key.Equals(key, true))
	entries, err := query.Find()
	if err != nil {
		return err
	}

	for _, entry := range entries {
		if err := s.box.Remove(entry); err != nil {
			return err
		}
	}

	return nil
}

func (s *ObjectBoxStorage) Reset() error {
	return s.box.RemoveAll()
}

func (s *ObjectBoxStorage) Close() error {
	s.ob.Close()
	return nil
}

func (s *ObjectBoxStorage) cleanupExpired() {
	query := s.box.Query(CacheEntry_.ExpiresAt.LessThan(time.Now().Unix()))
	entries, err := query.Find()
	if err != nil {
		return
	}
	s.box.ObjectBox.RunInWriteTx(func() error {
		for _, entry := range entries {
			s.box.Remove(entry)
		}
		return nil
	})

}

[gaby]

I will take a look later to see how much effort is this.

[karnadii]

I couldn't wait so after looking for other storage implementation, I write this.

package objectbox

import "time"

// Config defines the configuration options for ObjectBox storage.
type Config struct {
	// Directory is the path where the database is stored.
	// Optional, defaults to "objectbox"
	Directory string

	// MaxSizeInKb sets the maximum size of the database in kilobytes.
	// Optional, defaults to 1GB (1024 * 1024 * 1024)
	MaxSizeInKb uint64

	// MaxReaders defines the maximum number of concurrent readers.
	// Optional, defaults to 126
	MaxReaders uint

	// Reset determines if existing keys should be cleared on startup.
	// Optional, defaults to false
	Reset bool

	// CleanerInterval sets the frequency for deleting expired keys.
	// Optional, defaults to 60 seconds
	CleanerInterval time.Duration
}

var DefaultConfig = Config{
	Directory:       "objectbox_db",
	MaxSizeInKb:     1024 * 1024, // 1GByte
	MaxReaders:      126,
	Reset:           false,
	CleanerInterval: 60 * time.Second,
}

func getConfig(config ...Config) Config {
	if len(config) < 1 {
		return DefaultConfig
	}

	cfg := config[0]

	// Set default values

	if cfg.Directory == "" {
		cfg.Directory = DefaultConfig.Directory
	}

	if cfg.MaxSizeInKb == 0 {
		cfg.MaxSizeInKb = DefaultConfig.MaxSizeInKb
	}

	if cfg.MaxReaders == 0 {
		cfg.MaxReaders = DefaultConfig.MaxReaders
	}

	if int(cfg.CleanerInterval.Seconds()) == 0 {
		cfg.CleanerInterval = DefaultConfig.CleanerInterval
	}

	return cfg

}

package objectbox

import (
	"time"

	"github.com/objectbox/objectbox-go/objectbox"
)

//go:generate go run github.com/objectbox/objectbox-go/cmd/objectbox-gogen

// Cache represents a single cache entry in the storage.
type Cache struct {
	Id        uint64 `objectbox:"id"`
	Key       string `objectbox:"index,unique"`
	Value     []byte
	ExpiresAt int64 `objectbox:"index"`
}

// Storage handles the ObjectBox database operations and cleanup routines.
type Storage struct {
	ob   *objectbox.ObjectBox
	box  *CacheBox
	done chan struct{}
}

// New creates a new Storage instance with the provided configuration.
// It initializes the ObjectBox database and starts the cleanup routine.
func New(config ...Config) *Storage {
	cfg := getConfig(config...)

	ob, err := objectbox.NewBuilder().Model(ObjectBoxModel()).MaxSizeInKb(cfg.MaxSizeInKb).MaxReaders(cfg.MaxReaders).Directory(cfg.Directory).Build()
	if err != nil {
		return nil
	}

	if cfg.Reset {
		box := BoxForCache(ob)
		box.RemoveAll()
	}

	storage := &Storage{
		ob:   ob,
		box:  BoxForCache(ob),
		done: make(chan struct{}),
	}

	go storage.cleanerTicker(cfg.CleanerInterval)

	return storage
}

// Get retrieves a value from cache by its key.
// Returns nil if key doesn't exist or has expired.
func (s *Storage) Get(key string) ([]byte, error) {
	if len(key) < 1 {
		return nil, nil
	}

	query := s.box.Query(Cache_.Key.Equals(key, true),
		objectbox.Any(
			Cache_.ExpiresAt.Equals(0),
			Cache_.ExpiresAt.GreaterThan(time.Now().Unix()),
		))
	caches, err := query.Find()

	if err != nil {
		return nil, err
	}

	if len(caches) < 1 {
		return nil, nil
	}

	return caches[0].Value, nil

}

// Set stores a value in cache with the specified key and expiration.
// If expiration is 0, the entry won't expire.
func (s *Storage) Set(key string, value []byte, exp time.Duration) error {
	if len(key) <= 0 || len(value) <= 0 {
		return nil
	}

	// Since objectbox go doen't support conflict strategy,
	// we need to check if the key already exists
	// and update the value if it does. Thus we need to
	// get the id of the cache first and then update the cache
	// with the new value with the same id.
	query := s.box.Query(Cache_.Key.Equals(key, true))
	cachesIds, err := query.FindIds()
	if err != nil {
		return err
	}

	// if the id is 0 it will create new cache
	// otherwise it will update the existing entry
	var id uint64 = 0
	if len(cachesIds) > 0 {
		id = cachesIds[0]
	}

	var expAt int64

	if exp > 0 { // Changed from exp != 0 to exp > 0
		expAt = time.Now().Add(exp).Unix()
	}

	cache := &Cache{
		Id:        id,
		Key:       key,
		Value:     value,
		ExpiresAt: expAt,
	}

	_, err = s.box.Put(cache)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes an entry from cache by its key.
func (s *Storage) Delete(key string) error {
	if len(key) <= 0 {
		return nil
	}

	query := s.box.Query(Cache_.Key.Equals(key, true))
	cachesIds, err := query.FindIds()
	if err != nil {
		return err
	}

	if len(cachesIds) < 1 {
		return nil
	}

	if err := s.box.RemoveId(cachesIds[0]); err != nil {
		return err
	}

	return nil

}

// Reset removes all entries from the cache.
func (s *Storage) Reset() error {
	return s.box.RemoveAll()
}

// Close shuts down the storage, stopping the cleanup routine
// and closing the database connection.
func (s *Storage) Close() error {
	close(s.done)
	s.ob.Close()
	return nil
}

// cleaneStorage removes all expired cache entries.
func (s *Storage) cleaneStorage() {
	s.box.Query(Cache_.ExpiresAt.LessThan(time.Now().Unix())).Remove()

}

// cleanerTicker runs periodic cleanup of expired entries.
func (s *Storage) cleanerTicker(interval time.Duration) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			s.cleaneStorage()
		case <-s.done:
			return
		}
	}
}

[gaby]

@karnadii Awesome, do you allow us to use this implementation to make an official driver for it on this repo?

[karnadii]

@gaby yes please, the updated code is here https://github.com/karnadii/storage/tree/objectbox/objectbox
I just don't know how to setup the test action. please use the code and made appropriate changes as you please.

[gaby]

Will do this week, thanks! 💪