Implement BuildDataSourceRegistry in data sources service

This implements the aforementioned missing function.

Signed-off-by: Juan Antonio Osorio <ozz@stacklok.com>

internal/datasources/factory.go

@@ -12,11 +12,6 @@ import (
 	v1datasources "github.com/mindersec/minder/pkg/datasources/v1"
 )
 
-const (
-	// DataSourceDriverRest is the driver type for a REST data source.
-	DataSourceDriverRest = "rest"
-)
-
 // BuildFromProtobuf is a factory function that builds a new data source based on the given
 // data source type.
 func BuildFromProtobuf(ds *minderv1.DataSource) (v1datasources.DataSource, error) {

internal/datasources/service/convert.go

@@ -9,9 +9,9 @@ import (
 
 	"google.golang.org/protobuf/encoding/protojson"
 
-	"github.com/mindersec/minder/internal/datasources"
 	"github.com/mindersec/minder/internal/db"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	v1datasources "github.com/mindersec/minder/pkg/datasources/v1"
 )
 
 func dataSourceDBToProtobuf(ds db.DataSource, dsfuncs []db.DataSourcesFunction) (*minderv1.DataSource, error) {
@@ -33,7 +33,7 @@ func dataSourceDBToProtobuf(ds db.DataSource, dsfuncs []db.DataSourcesFunction)
 	dsfType := dsfuncs[0].Type
 
 	switch dsfType {
-	case datasources.DataSourceDriverRest:
+	case v1datasources.DataSourceDriverRest:
 		return dataSourceRestDBToProtobuf(outds, dsfuncs)
 	default:
 		return nil, fmt.Errorf("unknown data source type: %s", dsfType)

internal/datasources/service/helpers.go

@@ -5,6 +5,7 @@ package service
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/google/uuid"
@@ -29,9 +30,16 @@ func (d *dataSourceService) getDataSourceSomehow(
 
 	tx := stx.Q()
 
-	projs, err := listRelevantProjects(ctx, tx, project, opts.canSearchHierarchical())
-	if err != nil {
-		return nil, fmt.Errorf("failed to list relevant projects: %w", err)
+	var projs []uuid.UUID
+	if len(opts.hierarchy) > 0 {
+		projs = opts.hierarchy
+	} else {
+		prjs, err := listRelevantProjects(ctx, tx, project, opts.canSearchHierarchical())
+		if err != nil {
+			return nil, fmt.Errorf("failed to list relevant projects: %w", err)
+		}
+
+		projs = prjs
 	}
 
 	ds, err := theSomehow(ctx, tx, projs)
@@ -54,6 +62,31 @@ func (d *dataSourceService) getDataSourceSomehow(
 	return dataSourceDBToProtobuf(ds, dsfuncs)
 }
 
+func (d *dataSourceService) instantiateDataSource(
+	ctx context.Context,
+	ref *minderv1.DataSourceReference,
+	projectHierarchy []uuid.UUID,
+	tx db.ExtendQuerier,
+) (*minderv1.DataSource, error) {
+	if ref == nil {
+		return nil, errors.New("data source reference is nil")
+	}
+
+	// If we end up supporting other ways of referencing a data source, this
+	// would be the place to validate them.
+	if ref.GetName() == "" {
+		return nil, errors.New("data source name is empty")
+	}
+
+	ds, err := d.GetByName(ctx, ref.GetName(), uuid.Nil,
+		ReadBuilder().withHierarchy(projectHierarchy).WithTransaction(tx))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get data source by name: %w", err)
+	}
+
+	return ds, nil
+}
+
 func listRelevantProjects(
 	ctx context.Context, tx db.ExtendQuerier, project uuid.UUID, hierarchical bool,
 ) ([]uuid.UUID, error) {

internal/datasources/service/options.go

@@ -3,7 +3,11 @@
 
 package service
 
-import "github.com/mindersec/minder/internal/db"
+import (
+	"github.com/google/uuid"
+
+	"github.com/mindersec/minder/internal/db"
+)
 
 // Options is a struct that contains the options for a service call
 type Options struct {
@@ -40,6 +44,9 @@ type txGetter interface {
 type ReadOptions struct {
 	Options
 	hierarchical bool
+
+	// Use the actual project hierarchy to search for the data source.
+	hierarchy []uuid.UUID
 }
 
 // ReadBuilder is a function that returns a new ReadOptions struct
@@ -65,6 +72,16 @@ func (o *ReadOptions) WithTransaction(qtx db.ExtendQuerier) *ReadOptions {
 	return o
 }
 
+// withHierarchy allows the service to search in the project hierarchy.
+// This is left internal for now to disallow external use.
+func (o *ReadOptions) withHierarchy(projs []uuid.UUID) *ReadOptions {
+	if o == nil {
+		return nil
+	}
+	o.hierarchy = projs
+	return o
+}
+
 func (o *ReadOptions) canSearchHierarchical() bool {
 	if o == nil {
 		return false

internal/datasources/service/service.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/uuid"
 	"google.golang.org/grpc/codes"
 
+	"github.com/mindersec/minder/internal/datasources"
 	"github.com/mindersec/minder/internal/db"
 	"github.com/mindersec/minder/internal/util"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
@@ -199,9 +200,50 @@ func (d *dataSourceService) ValidateRuleTypeReferences(
 	panic("implement me")
 }
 
-// nolint:revive // there is a TODO
+// BuildDataSourceRegistry bundles up all data sources referenced in the rule type
+// into a registry.
+//
+// Note that this assumes that the rule type has already been validated.
 func (d *dataSourceService) BuildDataSourceRegistry(
 	ctx context.Context, rt *minderv1.RuleType, opts *Options) (*v1datasources.DataSourceRegistry, error) {
-	//TODO implement me
-	panic("implement me")
+	instantiations := rt.GetDef().GetEval().GetDataSources()
+	stx, err := d.txBuilder(d, opts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to start transaction: %w", err)
+	}
+
+	//nolint:gosec // we'll log this error later.
+	defer stx.Rollback()
+
+	tx := stx.Q()
+	reg := v1datasources.NewDataSourceRegistry()
+
+	rawproj := rt.GetContext().GetProject()
+	proj, err := uuid.Parse(rawproj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse project UUID: %w", err)
+	}
+
+	projectHierarchy, err := tx.GetParentProjects(ctx, proj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get project hierarchy: %w", err)
+	}
+
+	for _, ref := range instantiations {
+		inst, err := d.instantiateDataSource(ctx, ref, projectHierarchy, tx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to instantiate data source: %w", err)
+		}
+
+		impl, err := datasources.BuildFromProtobuf(inst)
+		if err != nil {
+			return nil, fmt.Errorf("failed to build data source from protobuf: %w", err)
+		}
+
+		if err := reg.RegisterDataSource(inst.GetName(), impl); err != nil {
+			return nil, fmt.Errorf("failed to register data source: %w", err)
+		}
+	}
+
+	return reg, nil
 }

internal/datasources/service/service_test.go

@@ -17,9 +17,9 @@ import (
 	"google.golang.org/protobuf/types/known/structpb"
 
 	mockdb "github.com/mindersec/minder/database/mock"
-	"github.com/mindersec/minder/internal/datasources"
 	"github.com/mindersec/minder/internal/db"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/datasources/v1"
 )
 
 func TestGetByName(t *testing.T) {
@@ -66,7 +66,7 @@ func TestGetByName(t *testing.T) {
 							ID:           uuid.New(),
 							DataSourceID: dsID,
 							Name:         "test_function",
-							Type:         string(datasources.DataSourceDriverRest),
+							Type:         string(v1.DataSourceDriverRest),
 							Definition: restDriverToJson(t, &minderv1.RestDataSource_Def{
 								Endpoint:    "http://example.com",
 								InputSchema: is,
@@ -183,7 +183,7 @@ func TestGetByID(t *testing.T) {
 							ID:           uuid.New(),
 							DataSourceID: id,
 							Name:         "test_function",
-							Type:         string(datasources.DataSourceDriverRest),
+							Type:         string(v1.DataSourceDriverRest),
 							Definition: restDriverToJson(t, &minderv1.RestDataSource_Def{
 								Endpoint:    "http://example.com",
 								InputSchema: is,
@@ -301,7 +301,7 @@ func TestList(t *testing.T) {
 							ID:           uuid.New(),
 							DataSourceID: dsID,
 							Name:         "test_function",
-							Type:         string(datasources.DataSourceDriverRest),
+							Type:         string(v1.DataSourceDriverRest),
 							Definition: restDriverToJson(t, &minderv1.RestDataSource_Def{
 								Endpoint:    "http://example.com",
 								InputSchema: is,

pkg/datasources/v1/datasources.go

@@ -6,6 +6,11 @@ package v1
 
 //go:generate go run go.uber.org/mock/mockgen -package mock_$GOPACKAGE -destination=./mock/$GOFILE -source=./$GOFILE
 
+const (
+	// DataSourceDriverRest is the driver type for a REST data source.
+	DataSourceDriverRest = "rest"
+)
+
 // DataSourceFuncKey is the key that uniquely identifies a data source function.
 type DataSourceFuncKey string