Merge pull request #10 from nandlabs/feature/vfs

Feature/vfs

errutils/errutils.go

@@ -0,0 +1,10 @@
+package errutils
+
+import (
+	"errors"
+	"fmt"
+)
+
+func FmtError(f string, v ...any) error {
+	return errors.New(fmt.Sprintf(f, v...))
+}

fsutils/fileutils_test.go

@@ -15,9 +15,9 @@ func TestFileExists(t *testing.T) {
 		args args
 		want bool
 	}{
-		{name: "Existing File", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: true},
-		{name: "Non Existing File", args: struct{ path string }{path: wd + "/testdata/test-nonexisting.json"}, want: false},
-		{name: "Dir As File", args: struct{ path string }{path: wd + "/testdata"}, want: false},
+		{name: "Existing VFile", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: true},
+		{name: "Non Existing VFile", args: struct{ path string }{path: wd + "/testdata/test-nonexisting.json"}, want: false},
+		{name: "Dir As VFile", args: struct{ path string }{path: wd + "/testdata"}, want: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -40,7 +40,7 @@ func TestDirExists(t *testing.T) {
 	}{
 		{name: "Existing Dir", args: struct{ path string }{path: wd + "/testdata"}, want: true},
 		{name: "Non Existing Dir", args: struct{ path string }{path: wd + "/test"}, want: false},
-		{name: "File AS DIR", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: false},
+		{name: "VFile AS DIR", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -63,8 +63,8 @@ func TestPathExists(t *testing.T) {
 	}{
 		{name: "Existing Dir", args: struct{ path string }{path: wd + "/testdata"}, want: true},
 		{name: "Non Existing Dir", args: struct{ path string }{path: wd + "/test"}, want: false},
-		{name: "Existing File", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: true},
-		{name: "Non Existing File", args: struct{ path string }{path: wd + "/testdata/unknown"}, want: false},
+		{name: "Existing VFile", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: true},
+		{name: "Non Existing VFile", args: struct{ path string }{path: wd + "/testdata/unknown"}, want: false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -86,9 +86,9 @@ func TestLookupContentType(t *testing.T) {
 		args args
 		want string
 	}{
-		{name: "JSON File", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: "application/json"},
-		{name: "YAML File", args: struct{ path string }{path: wd + "/testdata/test.yaml"}, want: "text/yaml"},
-		{name: "Dat File", args: struct{ path string }{path: wd + "/testdata/test.dat"}, want: "application/octet-stream"},
+		{name: "JSON VFile", args: struct{ path string }{path: wd + "/testdata/test.json"}, want: "application/json"},
+		{name: "YAML VFile", args: struct{ path string }{path: wd + "/testdata/test.yaml"}, want: "text/yaml"},
+		{name: "Dat VFile", args: struct{ path string }{path: wd + "/testdata/test.dat"}, want: "application/octet-stream"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

ioutils/io_utils.go

@@ -0,0 +1,10 @@
+package ioutils
+
+import "io"
+
+var CloserFunc = func(closer io.Closer) {
+	err := closer.Close()
+	if err != nil {
+
+	}
+}

secrets/README.md

@@ -0,0 +1,3 @@
+# Secrets Management
+
+TODO

vfs/README.md

@@ -0,0 +1,8 @@
+# Virtual File System (VFS) Package
+The VFS package provides a unified api for accessing multple file system. It is extensible and new implementation can be
+easily plugged in.
+
+
+The default package has methods for local file system.
+
+

vfs/base_file.go

@@ -0,0 +1,53 @@
+package vfs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+)
+
+type BaseFile struct {
+	VFile
+}
+
+func (b *BaseFile) AsString() (s string, err error) {
+	var bytes []byte
+	bytes, err = ioutil.ReadAll(b)
+	if err == nil {
+		s = string(bytes)
+	}
+	return
+}
+
+func (b *BaseFile) AsBytes() ([]byte, error) {
+	return ioutil.ReadAll(b)
+}
+
+func (b *BaseFile) DeleteMatching(filter FileFilter) (err error) {
+	var info VFileInfo
+	info, err = b.Info()
+
+	if err == nil {
+		if info.IsDir() {
+			err = errors.New(fmt.Sprintf("Invalid operation DeleteMatching %s is a file", b.Url().String()))
+		} else {
+			var files []VFile
+			files, err = b.Find(filter)
+			if err == nil {
+				for _, file := range files {
+					err = file.Delete()
+					if err != nil {
+						break
+					}
+				}
+			}
+		}
+	}
+
+	return
+}
+
+func (b *BaseFile) WriteString(s string) (int, error) {
+	return io.WriteString(b, s)
+}

vfs/base_fs.go

@@ -0,0 +1,198 @@
+package vfs
+
+import (
+	"go.nandlabs.io/commons/ioutils"
+	"io"
+	"net/url"
+)
+
+type BaseVFS struct {
+	VFileSystem
+}
+
+func (b *BaseVFS) Copy(src, dst *url.URL) (err error) {
+	var srcFile VFile
+	var srfFileInfo VFileInfo
+	srcFile, err = b.Open(src)
+
+	if err == nil {
+		defer ioutils.CloserFunc(srcFile)
+		srfFileInfo, err = srcFile.Info()
+		if err == nil {
+			if srfFileInfo.IsDir() {
+
+				err = b.Walk(src, func(file VFile) (err error) {
+					var fileInfo VFileInfo
+					fileInfo, err = srcFile.Info()
+					if err == nil {
+						if fileInfo.IsDir() {
+
+						}
+					}
+
+					return
+				})
+
+				// Create directories and copy files
+			} else {
+				var destFile VFile
+				destFile, err = manager.Create(dst)
+				defer ioutils.CloserFunc(destFile)
+				if err == nil {
+					_, err = io.Copy(srcFile, destFile)
+				}
+			}
+		}
+	}
+	return
+}
+
+func (b *BaseVFS) CopyRaw(src, dst string) (err error) {
+	var srcUrl, dstUrl *url.URL
+	srcUrl, err = url.Parse(src)
+	if err == nil {
+		dstUrl, err = url.Parse(dst)
+		if err == nil {
+			err = b.Copy(srcUrl, dstUrl)
+		}
+	}
+	return
+}
+
+func (b *BaseVFS) CreateRaw(u string) (file VFile, err error) {
+	var fileUrl *url.URL
+	fileUrl, err = url.Parse(u)
+	if err == nil {
+		if err == nil {
+			file, err = b.Create(fileUrl)
+		}
+	}
+	return
+}
+
+func (b *BaseVFS) Delete(src *url.URL) (err error) {
+	var srcFile VFile
+	srcFile, err = b.Open(src)
+	if err == nil {
+		defer ioutils.CloserFunc(srcFile)
+		err = srcFile.Delete()
+	}
+	return
+}
+
+func (b *BaseVFS) DeleteRaw(u string) (err error) {
+	var fileUrl *url.URL
+	fileUrl, err = url.Parse(u)
+	if err == nil {
+		err = b.Delete(fileUrl)
+	}
+	return
+}
+
+func (b *BaseVFS) List(src *url.URL) (files []VFile, err error) {
+	var srcFile VFile
+	srcFile, err = b.Open(src)
+	if err == nil {
+		defer ioutils.CloserFunc(srcFile)
+		files, err = srcFile.ListAll()
+	}
+	return
+}
+
+func (b *BaseVFS) ListRaw(src string) (files []VFile, err error) {
+	var fileUrl *url.URL
+	fileUrl, err = url.Parse(src)
+	if err == nil {
+		files, err = b.List(fileUrl)
+	}
+	return
+}
+
+func (b *BaseVFS) MkdirRaw(u string) (vFile VFile, err error) {
+	var fileUrl *url.URL
+	fileUrl, err = url.Parse(u)
+	if err == nil {
+		vFile, err = b.Mkdir(fileUrl)
+	}
+	return
+}
+func (b *BaseVFS) MkdirAllRaw(u string) (vFile VFile, err error) {
+	var fileUrl *url.URL
+	fileUrl, err = url.Parse(u)
+	if err == nil {
+		vFile, err = b.MkdirAll(fileUrl)
+	}
+	return
+}
+
+func (b *BaseVFS) Move(src, dst *url.URL) (err error) {
+	err = b.Copy(src, dst)
+	if err == nil {
+		err = b.Delete(src)
+	}
+	return
+}
+
+func (b *BaseVFS) MoveRaw(src, dst string) (err error) {
+	var srcUrl, dstUrl *url.URL
+	srcUrl, err = url.Parse(src)
+	if err == nil {
+		dstUrl, err = url.Parse(dst)
+		if err == nil {
+			err = b.Move(srcUrl, dstUrl)
+		}
+	}
+	return
+}
+
+func (b *BaseVFS) OpenRaw(l string) (file VFile, err error) {
+	var u *url.URL
+	u, err = url.Parse(l)
+	if err == nil {
+		file, err = b.Open(u)
+	}
+	return
+}
+
+func (b *BaseVFS) Walk(u *url.URL, fn WalkFn) (err error) {
+	var src VFile
+	var srcFi VFileInfo
+	var childInfo VFileInfo
+	var children []VFile
+	src, err = manager.Open(u)
+	if err == nil {
+		srcFi, err = src.Info()
+		if err == nil {
+			if srcFi.IsDir() {
+				children, err = src.ListAll()
+				if err == nil {
+					for _, child := range children {
+						childInfo, err = child.Info()
+						if err == nil {
+
+							if childInfo.IsDir() {
+								err = b.Walk(child.Url(), fn)
+							} else {
+								err = fn(child)
+							}
+							if err != nil {
+								break
+							}
+						}
+					}
+				}
+			}
+
+		}
+	}
+	return
+}
+
+func (b *BaseVFS) WalkRaw(raw string, fn WalkFn) (err error) {
+	var u *url.URL
+	u, err = url.Parse(raw)
+	if err == nil {
+		err = b.Walk(u, fn)
+	}
+	return
+}