solver: add sqlcachestorage as an alternative to the bolt db

cmd/buildkitd/config/config.go

@@ -40,6 +40,8 @@ type Config struct {
 	} `toml:"frontend"`
 
 	System *SystemConfig `toml:"system"`
+
+	Cache CacheConfig `toml:"cache"`
 }
 
 type SystemConfig struct {
@@ -179,3 +181,7 @@ type GatewayFrontendConfig struct {
 	Enabled             *bool    `toml:"enabled"`
 	AllowedRepositories []string `toml:"allowedRepositories"`
 }
+
+type CacheConfig struct {
+	IndexFormat string `toml:"index-format"`
+}

cmd/buildkitd/main.go

@@ -41,6 +41,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/bboltcachestorage"
+	"github.com/moby/buildkit/solver/sqlcachestorage"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/appdefaults"
@@ -540,10 +541,16 @@ func setDefaultConfig(cfg *config.Config) {
 	if cfg.OTEL.SocketPath == "" {
 		cfg.OTEL.SocketPath = appdefaults.TraceSocketPath(isRootlessConfig())
 	}
+
+	if cfg.Cache.IndexFormat == "" {
+		cfg.Cache.IndexFormat = appdefaults.CacheIndexFormat
+	}
 }
 
-var isRootlessConfigOnce sync.Once
-var isRootlessConfigValue bool
+var (
+	isRootlessConfigOnce  sync.Once
+	isRootlessConfigValue bool
+)
 
 // isRootlessConfig is true if we should be using the rootless config
 // defaults instead of the normal defaults.
@@ -799,7 +806,7 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 		frontends["gateway.v0"] = gwfe
 	}
 
-	cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(cfg.Root, "cache.db"))
+	cacheStorage, err := newCacheKeyStorage(cfg.Root, cfg.Cache)
 	if err != nil {
 		return nil, err
 	}
@@ -962,6 +969,17 @@ func parseBoolOrAuto(s string) (*bool, error) {
 	return &b, err
 }
 
+func newCacheKeyStorage(root string, cfg config.CacheConfig) (solver.PersistentCacheKeyStorage, error) {
+	switch cfg.IndexFormat {
+	case "bolt":
+		return bboltcachestorage.NewStore(filepath.Join(root, "cache.db"))
+	case "sqlite":
+		return sqlcachestorage.NewStore(filepath.Join(root, "cache.db"))
+	default:
+		return nil, errors.Errorf("invalid cache index format: %s", cfg.IndexFormat)
+	}
+}
+
 func runTraceController(p string, exp sdktrace.SpanExporter) error {
 	server := grpc.NewServer()
 	tracev1.RegisterTraceServiceServer(server, &traceCollector{exporter: exp})

control/control.go

@@ -30,7 +30,6 @@ import (
 	"github.com/moby/buildkit/session/grpchijack"
 	containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
 	"github.com/moby/buildkit/solver"
-	"github.com/moby/buildkit/solver/bboltcachestorage"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/solver/llbsolver/proc"
 	"github.com/moby/buildkit/solver/pb"
@@ -63,7 +62,7 @@ type Opt struct {
 	Entitlements              []string
 	TraceCollector            sdktrace.SpanExporter
 	HistoryDB                 db.DB
-	CacheStore                *bboltcachestorage.Store
+	CacheStore                solver.PersistentCacheKeyStorage
 	LeaseManager              *leaseutil.Manager
 	ContentStore              *containerdsnapshot.Store
 	HistoryConfig             *config.HistoryConfig
@@ -673,7 +672,7 @@ func cacheOptKey(opt controlapi.CacheOptionsEntry) (string, error) {
 	if opt.Type == "registry" && opt.Attrs["ref"] != "" {
 		return opt.Attrs["ref"], nil
 	}
-	var rawOpt = struct {
+	rawOpt := struct {
 		Type  string
 		Attrs map[string]string
 	}{

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/in-toto/in-toto-golang v0.5.0
 	github.com/klauspost/compress v1.17.9
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/moby/docker-image-spec v1.3.1
 	github.com/moby/locker v1.0.1

go.sum

@@ -276,6 +276,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=

hack/composefiles/buildkitd.toml

@@ -1,4 +1,7 @@
 debug = true
 
 [grpc]
-  debugAddress = "0.0.0.0:6060"
+debugAddress = "0.0.0.0:6060"
+
+[cache]
+index-format = "sqlite"

hack/composefiles/compose.yaml

@@ -5,7 +5,8 @@ services:
     build:
       context: ../..
       args:
-        BUILDKIT_DEBUG: 1
+        #BUILDKIT_DEBUG: 1
+        CGO_ENABLED: 1
     image: moby/buildkit:local
     ports:
       - 127.0.0.1:5000:5000

solver/bboltcachestorage/storage.go

@@ -46,6 +46,7 @@ func NewStore(dbPath string) (*Store, error) {
 	}); err != nil {
 		return nil, err
 	}
+
 	return &Store{db: db}, nil
 }
 

solver/bboltcachestorage/storage_test.go

@@ -3,7 +3,9 @@ package bboltcachestorage
 import (
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/testutil"
 	"github.com/stretchr/testify/require"
@@ -22,3 +24,24 @@ func TestBoltCacheStorage(t *testing.T) {
 		return st
 	})
 }
+
+func BenchmarkAddResult(b *testing.B) {
+	tmpdir := b.TempDir()
+	store, err := NewStore(filepath.Join(tmpdir, "cache.db"))
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer store.Close()
+
+	for i := 0; i < b.N; i++ {
+		id, res := newResult()
+		store.AddResult(id, res)
+	}
+}
+
+func newResult() (string, solver.CacheResult) {
+	return identity.NewID(), solver.CacheResult{
+		CreatedAt: time.Now(),
+		ID:        identity.NewID(),
+	}
+}

solver/cachemanager.go

@@ -306,7 +306,11 @@ func (c *cacheManager) Save(k *CacheKey, r Result, createdAt time.Time) (rck *Ex
 		"stack":         bklog.TraceLevelOnlyStack(),
 	})
 	defer func() {
-		lg.WithError(rerr).WithField("return_cachekey", rck.TraceFields()).Trace("cache manager")
+		l := lg.WithError(rerr)
+		if rck != nil {
+			l = l.WithField("return_cachekey", rck.TraceFields())
+		}
+		l.Trace("cache manager")
 	}()
 
 	c.mu.Lock()

solver/cachestorage.go

@@ -2,6 +2,7 @@ package solver
 
 import (
 	"context"
+	"io"
 	"time"
 
 	"github.com/moby/buildkit/session"
@@ -29,6 +30,11 @@ type CacheKeyStorage interface {
 	WalkBacklinks(id string, fn func(id string, link CacheInfoLink) error) error
 }
 
+type PersistentCacheKeyStorage interface {
+	CacheKeyStorage
+	io.Closer
+}
+
 // CacheResult is a record for a single solve result
 type CacheResult struct {
 	CreatedAt time.Time

solver/sqlcachestorage/sql.go

@@ -0,0 +1,116 @@
+package sqlcachestorage
+
+const (
+	createResultsTableSQL = `
+CREATE TABLE IF NOT EXISTS results (
+	id text NOT NULL,
+	created_at timestamp,
+	worker_ref_id text NOT NULL
+);
+	`
+
+	createResultsIndexSQL = `
+CREATE INDEX IF NOT EXISTS results_index
+ON results (id);
+	`
+
+	createResultsByRefIndexSQL = `
+CREATE INDEX IF NOT EXISTS results_by_ref_index
+ON results (worker_ref_id);
+	`
+
+	createLinksTableSQL = `
+CREATE TABLE IF NOT EXISTS links (
+	source_result_id text NOT NULL,
+	vertex_input integer DEFAULT 0,
+	vertex_output integer DEFAULT 0,
+	vertex_digest text NOT NULL,
+	vertex_selector text DEFAULT '',
+	target_result_id text NOT NULL,
+	UNIQUE (source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector, target_result_id)
+);
+	`
+
+	createUnreferencedLinksTriggerSQL = `
+CREATE TRIGGER IF NOT EXISTS unreferenced_links_sql AFTER DELETE ON results
+BEGIN
+	DELETE FROM links WHERE source_result_id = old.id OR target_result_id = old.id;
+END
+	`
+
+	createLinksIndexSQL = `
+CREATE INDEX IF NOT EXISTS links_index
+ON links (source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector, target_result_id);
+	`
+
+	createBacklinksIndexSQL = `
+CREATE INDEX IF NOT EXISTS backlinks_index
+ON links (target_result_id);
+	`
+
+	existsSQL = `
+SELECT 1 FROM results WHERE id = ? LIMIT 1;
+	`
+
+	walkSQL = `
+SELECT id FROM results;
+	`
+
+	walkResultsSQL = `
+SELECT worker_ref_id, created_at FROM results WHERE id = ?;
+	`
+
+	loadSQL = `
+SELECT worker_ref_id, created_at FROM results
+WHERE id = ? AND worker_ref_id = ?
+LIMIT 1;
+	`
+
+	addResultSQL = `
+INSERT INTO results (id, created_at, worker_ref_id)
+VALUES(?, ?, ?);
+	`
+
+	deleteResultByRefIDSQL = `
+DELETE FROM results WHERE worker_ref_id = ?;
+	`
+
+	walkIDsByResultSQL = `
+SELECT DISTINCT id FROM results WHERE worker_ref_id = ?;
+	`
+
+	addLinkSQL = `
+INSERT INTO links (source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector, target_result_id)
+VALUES (?, ?, ?, ?, ?, ?);
+	`
+
+	walkLinksSQL = `
+SELECT target_result_id FROM links
+WHERE source_result_id = ? AND vertex_input = ? AND vertex_output = ? AND vertex_digest = ? AND vertex_selector = ?;
+	`
+
+	hasLinkSQL = `
+SELECT 1 FROM links
+WHERE source_result_id = ?
+AND vertex_input = ?
+AND vertex_output = ?
+AND vertex_digest = ?
+AND vertex_selector = ?
+AND target_result_id = ?
+LIMIT 1;
+	`
+
+	walkBacklinksSQL = `
+SELECT source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector FROM links WHERE target_result_id = ?;
+	`
+)
+
+var createSQL = []string{
+	createResultsTableSQL,
+	createResultsIndexSQL,
+	createResultsByRefIndexSQL,
+	createLinksTableSQL,
+	createUnreferencedLinksTriggerSQL,
+	createLinksIndexSQL,
+	createBacklinksIndexSQL,
+}