added filesystem support

Author: alash3al

.gitignore

@@ -1,3 +1,4 @@
 bin
 redixdata*
 .DS_STORE
+data/

README.md

@@ -11,7 +11,7 @@ Features
 - A really simple `key => value` store that speaks `redis` protocol but with our rules!.
 - A real system that you can abuse! it isn't intedented for cache only but a "database system".
 - `Async` (all writes happen in the background), or `Sync` it won't respond to the client before writing to the internal datastore.
-- Pluggable storage engines, currently it supports `postgresql`, and there may be more engines be introduced in the upcomning releases.
+- Pluggable storage engines, currently it supports (`postgresql`, `filesystem`), and there may be more engines be introduced in the upcomning releases.
 - It could be used using `redis` clients easily, i.e: "the famouus redis-cli"
 
 
@@ -23,9 +23,9 @@ Core Commands
 - `FLUSHDB`
 - `SELECT <DB index>`
 - `SET <key> <value> [EX seconds | KEEPTTL] [NX]`
-- `TTL <key>`
+- `TTL <key>` **(not supported while using `filesystem` engine)**
 - `GET <key> [DELETE]`, it has an alias for backward compatibility reasons called `GETDEL <key>`
-- `INCR <key> [<delta>]`, it has an alias for backward compatibility reasons called `INCRBY`
+- `INCR <key> [<delta>]`, it has an alias for backward compatibility reasons called `INCRBY` **(not supported while using `filesystem` engine)**
 - `DEL key [key ...]`
 - `HGETALL <prefix>`
     > Fetches the whole data under the specified prefix as a hashmap result

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/hashicorp/hcl/v2 v2.11.1
+	github.com/jackc/pgx/v4 v4.14.1
 	github.com/tidwall/redcon v1.4.3
 )
 
@@ -18,7 +19,6 @@ require (
 	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jackc/pgtype v1.9.1 // indirect
-	github.com/jackc/pgx/v4 v4.14.1 // indirect
 	github.com/jackc/puddle v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
 	github.com/tidwall/btree v0.7.1 // indirect

go.sum

@@ -7,6 +7,7 @@ github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
 github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -19,6 +20,7 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
 github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -45,6 +47,7 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
 github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
+github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc=
 github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
@@ -91,6 +94,7 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LE
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -99,7 +103,9 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -109,6 +115,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -120,6 +127,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/btree v0.7.1 h1:LPXN3VRIxsdMwyfbtPgOA60jLuj/eEmMpDjOh2szRPw=
 github.com/tidwall/btree v0.7.1/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4=
@@ -189,7 +197,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -213,5 +220,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=

internals/datastore/contract/engine.go

@@ -10,7 +10,6 @@ type Engine interface {
 	Open(string) error
 	Close() error
 	Write(*WriteInput) (*WriteOutput, error)
-	// BatchWrite([]*WriteInput) error
 	Read(*ReadInput) (*ReadOutput, error)
 	Iterate(*IteratorOpts) error
 }

internals/datastore/engines/filesystem/etc.go

@@ -0,0 +1,53 @@
+package filesystem
+
+import (
+	"io/ioutil"
+	"os"
+	"syscall"
+)
+
+// ReadFileWithSharedLock reads the specified file using a shared-lock
+func ReadFileWithSharedLock(filename string) ([]byte, error) {
+	f, err := os.OpenFile(filename, os.O_RDONLY, 0775)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fd := int(f.Fd())
+
+	if err := syscall.Flock(fd, syscall.LOCK_SH); err != nil {
+		return nil, err
+	}
+	defer syscall.Flock(fd, syscall.LOCK_UN)
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+// WriteFileWithExclusiveLock reads the specified file using a shared-lock
+func WriteFileWithExclusiveLock(filename string, data []byte) (int, error) {
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_TRUNC|os.O_SYNC, 07775)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	fd := int(f.Fd())
+
+	if err := syscall.Flock(fd, syscall.LOCK_EX); err != nil {
+		return 0, err
+	}
+	defer syscall.Flock(fd, syscall.LOCK_UN)
+
+	written, err := f.Write(data)
+	if err != nil {
+		return 0, err
+	}
+
+	return written, nil
+}

internals/datastore/engines/filesystem/filesystem.go

@@ -0,0 +1,185 @@
+package filesystem
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/alash3al/redix/internals/datastore/contract"
+)
+
+// Engine represents the contract.Engine implementation
+type Engine struct {
+	storageDir string
+	kvDir      string
+}
+
+// Open opens the database
+func (e *Engine) Open(dir string) (err error) {
+	if err := os.MkdirAll(dir, 0775); err != nil && err != os.ErrExist {
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Join(dir, "/kv"), 0775); err != nil && err != os.ErrExist {
+		return err
+	}
+
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		return err
+	}
+
+	e.storageDir = absDir
+	e.kvDir = filepath.Join(e.storageDir, "/kv")
+
+	return nil
+}
+
+// Write writes into the database
+func (e *Engine) Write(input *contract.WriteInput) (*contract.WriteOutput, error) {
+	if input == nil {
+		return nil, fmt.Errorf("empty input specified")
+	}
+
+	if input.Append {
+		return nil, fmt.Errorf("(filesystem) unsupported feature (append)")
+	}
+
+	if input.Increment {
+		return nil, fmt.Errorf("(filesystem) unsupported feature (increment)")
+	}
+
+	if input.OnlyIfNotExists {
+		return nil, fmt.Errorf("(filesystem) unsupported feature (ifNotExists)")
+	}
+
+	if input.TTL > 0 {
+		return nil, fmt.Errorf("(filesystem) unsupported feature (TTL)")
+	}
+
+	if input.Key == nil {
+		if err := os.RemoveAll(e.kvDir); err != nil {
+			return nil, err
+		}
+
+		return nil, nil
+	}
+
+	key := hex.EncodeToString(input.Key)
+	keyDataPath := filepath.Join(e.kvDir, key)
+
+	if input.Value == nil {
+		if err := os.Remove(keyDataPath); err != nil {
+			return nil, err
+		}
+
+		return nil, nil
+	}
+
+	if err := os.MkdirAll(filepath.Join(e.kvDir, "/kv"), 0775); err != nil && err != os.ErrExist {
+		return nil, err
+	}
+
+	if _, err := WriteFileWithExclusiveLock(keyDataPath, input.Value); err != nil {
+		return nil, err
+	}
+
+	return &contract.WriteOutput{
+		Value: input.Value,
+	}, nil
+}
+
+// Get reads from the database
+func (e *Engine) Read(input *contract.ReadInput) (*contract.ReadOutput, error) {
+	if input == nil {
+		return nil, fmt.Errorf("empty input specified")
+	}
+
+	key := hex.EncodeToString(input.Key)
+	keyDataPath := filepath.Join(e.kvDir, key)
+
+	data, err := ReadFileWithSharedLock(keyDataPath)
+	if err != nil {
+		if err == os.ErrNotExist {
+			return &contract.ReadOutput{}, nil
+		}
+
+		return nil, err
+	}
+
+	if input.Delete {
+		return nil, filepath.WalkDir(e.kvDir, func(path string, d fs.DirEntry, err error) error {
+			if path == e.kvDir {
+				return nil
+			}
+
+			actualKey, err := hex.DecodeString(d.Name())
+			if err != nil {
+				return err
+			}
+
+			if bytes.HasPrefix(actualKey, input.Key) {
+				if err := os.Remove(path); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		})
+	}
+
+	return &contract.ReadOutput{
+		Key:    input.Key,
+		Value:  data,
+		Exists: true,
+		TTL:    -1,
+	}, nil
+}
+
+// Iterate iterates on the whole database stops if the IteratorOpts returns an error
+func (e *Engine) Iterate(opts *contract.IteratorOpts) error {
+	if opts == nil {
+		return fmt.Errorf("empty options specified")
+	}
+
+	if opts.Callback == nil {
+		return fmt.Errorf("you must specify the callback")
+	}
+
+	return filepath.WalkDir(e.kvDir, func(path string, d fs.DirEntry, err error) error {
+		if path == e.kvDir {
+			return nil
+		}
+
+		actualKey, err := hex.DecodeString(d.Name())
+		if err != nil {
+			return err
+		}
+
+		if bytes.HasPrefix(actualKey, opts.Prefix) {
+			data, err := ReadFileWithSharedLock(path)
+			if err != nil {
+				return err
+			}
+
+			if err := opts.Callback(&contract.ReadOutput{
+				Key:    bytes.TrimPrefix(actualKey, opts.Prefix),
+				Value:  data,
+				Exists: true,
+				TTL:    -1,
+			}); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// Close closes the connection
+func (e *Engine) Close() error {
+	return nil
+}

internals/datastore/engines/filesystem/init.go

@@ -0,0 +1,12 @@
+package filesystem
+
+import "github.com/alash3al/redix/internals/datastore/contract"
+
+// Global consts
+const (
+	Name = "filesystem"
+)
+
+func init() {
+	contract.Register(Name, &Engine{})
+}

internals/datastore/engines/postgresql/postgres.go

@@ -72,6 +72,14 @@ func (e *Engine) Write(input *contract.WriteInput) (*contract.WriteOutput, error
 		return nil, fmt.Errorf("empty input specified")
 	}
 
+	if input.Key == nil {
+		if _, err := e.conn.Exec(context.Background(), "DELETE FROM redix_data_v5"); err != nil {
+			return nil, err
+		}
+
+		return nil, nil
+	}
+
 	if input.Value == nil {
 		if _, err := e.conn.Exec(context.Background(), "DELETE FROM redix_data_v5 WHERE _key LIKE $1", append(input.Key, '%')); err != nil {
 			return nil, err