plumb through build repos (#329)

Picks up chainguard-dev/apko#1169

This lets us define repos to pull packages from at apko-build-time,
which won't be available or visible via `apk update` or `apk add`.

---------

Signed-off-by: Jason Hall <jason@chainguard.dev>

docs/data-sources/config.md

@@ -83,6 +83,7 @@ Read-Only:
 
 Read-Only:
 
+- `build_repositories` (List of String)
 - `keyring` (List of String)
 - `packages` (List of String)
 - `repositories` (List of String)

docs/data-sources/tags.md

@@ -82,6 +82,7 @@ Required:
 
 Required:
 
+- `build_repositories` (List of String)
 - `keyring` (List of String)
 - `packages` (List of String)
 - `repositories` (List of String)

docs/index.md

@@ -21,6 +21,7 @@ provider "apko" {}
 
 ### Optional
 
+- `build_repositories` (List of String) Additional repositories to search for packages, only during apko build
 - `default_annotations` (Map of String) Default annotations to add
 - `default_archs` (List of String) Default architectures to build for
 - `extra_keyring` (List of String) Additional keys to use for package verification

docs/resources/build.md

@@ -124,6 +124,7 @@ Required:
 
 Required:
 
+- `build_repositories` (List of String)
 - `keyring` (List of String)
 - `packages` (List of String)
 - `repositories` (List of String)

go.mod

@@ -3,7 +3,7 @@ module github.com/chainguard-dev/terraform-provider-apko
 go 1.22.3
 
 require (
-	chainguard.dev/apko v0.14.8
+	chainguard.dev/apko v0.14.10-0.20240617143934-ac840f83c1c0
 	github.com/chainguard-dev/clog v1.3.1
 	github.com/chainguard-dev/terraform-provider-oci v0.0.13
 	github.com/google/go-cmp v0.6.0
@@ -16,7 +16,7 @@ require (
 	github.com/sigstore/cosign/v2 v2.2.4
 	golang.org/x/sync v0.7.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/apimachinery v0.30.1
+	k8s.io/apimachinery v0.30.2
 	knative.dev/pkg v0.0.0-20240521083825-99e1685a7997
 )
 
@@ -101,7 +101,7 @@ require (
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/letsencrypt/boulder v0.0.0-20240606225043-de8401e3454f // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -1,5 +1,5 @@
-chainguard.dev/apko v0.14.8 h1:UHnn4qm/erRppygHH8/0OEA+E72fnwAY2px/YaRaI8g=
-chainguard.dev/apko v0.14.8/go.mod h1:aFEwAkFsf7sXvVFQ2ui6KRK3tbG3mIl5PmPd4JqKGVM=
+chainguard.dev/apko v0.14.10-0.20240617143934-ac840f83c1c0 h1:M2W40pecL50Yq87YXfqXgkSRazcYWq4XBE03FAWBsUg=
+chainguard.dev/apko v0.14.10-0.20240617143934-ac840f83c1c0/go.mod h1:Z7lctAs9bQinh3azYJ4+mfLM20A2xPvF9nWYwSCgGdM=
 cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
 cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@@ -233,8 +233,8 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -509,8 +509,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
 gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
-k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
-k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg=
+k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak=

internal/provider/build.go

@@ -53,7 +53,8 @@ func fromImageData(ctx context.Context, ic types.ImageConfiguration, popts Provi
 		build.WithImageConfiguration(ic),
 		build.WithSBOMFormats([]string{"spdx"}),
 		build.WithExtraKeys(popts.keyring),
-		build.WithExtraRepos(popts.repositories),
+		build.WithExtraRuntimeRepos(popts.repositories),
+		build.WithExtraBuildRepos(popts.buildRespositories),
 	}
 
 	o, ic2, err := build.NewOptions(opts...)
@@ -133,7 +134,8 @@ func doBuild(ctx context.Context, data BuildResourceModel) (v1.Hash, coci.Signed
 					build.WithSBOM(tempDir),
 					build.WithArch(arch),
 					build.WithExtraKeys(data.popts.keyring),
-					build.WithExtraRepos(data.popts.repositories))...,
+					build.WithExtraBuildRepos(data.popts.buildRespositories),
+					build.WithExtraRuntimeRepos(data.popts.repositories))...,
 			)
 			if err != nil {
 				return fmt.Errorf("failed to start apko build: %w", err)
@@ -227,7 +229,8 @@ func doBuild(ctx context.Context, data BuildResourceModel) (v1.Hash, coci.Signed
 		build.WithSBOMFormats([]string{"spdx"}),
 		build.WithSBOM(tempDir),
 		build.WithExtraKeys(data.popts.keyring),
-		build.WithExtraRepos(data.popts.repositories),
+		build.WithExtraRuntimeRepos(data.popts.repositories),
+		build.WithExtraBuildRepos(data.popts.buildRespositories),
 	)
 	if err != nil {
 		return v1.Hash{}, nil, nil, fmt.Errorf("failed to create options for index: %w", err)

internal/provider/config_data_source.go

@@ -122,10 +122,12 @@ func (d *ConfigDataSource) Read(ctx context.Context, req datasource.ReadRequest,
 	}
 
 	tflog.Trace(ctx, fmt.Sprintf("got repos: %v", d.popts.repositories))
+	tflog.Trace(ctx, fmt.Sprintf("got build repos: %v", d.popts.buildRespositories))
 	tflog.Trace(ctx, fmt.Sprintf("got keyring: %v", d.popts.keyring))
 
 	// Append any provider-specified repositories, packages, and keys, if specified.
-	ic.Contents.Repositories = sets.List(sets.New(ic.Contents.Repositories...).Insert(d.popts.repositories...))
+	ic.Contents.RuntimeRepositories = sets.List(sets.New(ic.Contents.RuntimeRepositories...).Insert(d.popts.repositories...))
+	ic.Contents.BuildRepositories = sets.List(sets.New(ic.Contents.BuildRepositories...).Insert(d.popts.buildRespositories...))
 	ic.Contents.Packages = sets.List(sets.New(ic.Contents.Packages...).Insert(d.popts.packages...))
 	ic.Contents.Keyring = sets.List(sets.New(ic.Contents.Keyring...).Insert(d.popts.keyring...))
 
@@ -246,7 +248,8 @@ func (d *ConfigDataSource) resolvePackageList(ctx context.Context, ic apkotypes.
 					build.WithSBOMFormats([]string{"spdx"}),
 					build.WithArch(arch),
 					build.WithExtraKeys(d.popts.keyring),
-					build.WithExtraRepos(d.popts.repositories))...,
+					build.WithExtraBuildRepos(d.popts.buildRespositories),
+					build.WithExtraRuntimeRepos(d.popts.repositories))...,
 			)
 			if err != nil {
 				return err

internal/provider/config_data_source_test.go

@@ -59,10 +59,11 @@ func TestAccDataSourceConfig_ExtraPackages(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64", "aarch64"},
-				packages:     []string{"wolfi-baselayout=20230201-r0"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64", "aarch64"},
+				packages:           []string{"wolfi-baselayout=20230201-r0"},
 				anns: map[string]string{
 					"bar": "provider-provided",
 					"baz": "provider-provided",
@@ -106,10 +107,11 @@ func TestAccDataSourceConfig_ProviderOpts_Locked(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64", "aarch64"},
-				packages:     []string{"wolfi-baselayout=20230201-r0"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64", "aarch64"},
+				packages:           []string{"wolfi-baselayout=20230201-r0"},
 			}),
 		},
 		Steps: []resource.TestStep{{
@@ -146,10 +148,11 @@ func TestAccDataSourceConfig_ProviderOpts_Unlocked(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64", "aarch64"},
-				packages:     []string{"wolfi-baselayout"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64", "aarch64"},
+				packages:           []string{"wolfi-baselayout"},
 			}),
 		},
 		Steps: []resource.TestStep{{
@@ -184,10 +187,11 @@ func TestAccDataSourceConfig_ProviderOpts_OverrideArchitecture(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64", "aarch64"},
-				packages:     []string{"wolfi-baselayout"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64", "aarch64"},
+				packages:           []string{"wolfi-baselayout"},
 			}),
 		},
 		Steps: []resource.TestStep{{

internal/provider/provider.go

@@ -18,22 +18,23 @@ var _ provider.Provider = &Provider{}
 type Provider struct {
 	version string
 
-	repositories, packages, keyring, archs []string
-	anns                                   map[string]string
+	repositories, buildRespositories, packages, keyring, archs []string
+	anns                                                       map[string]string
 }
 
 type ProviderModel struct {
 	ExtraRepositories  []string          `tfsdk:"extra_repositories"`
+	BuildRepositories  []string          `tfsdk:"build_repositories"`
 	ExtraPackages      []string          `tfsdk:"extra_packages"`
 	ExtraKeyring       []string          `tfsdk:"extra_keyring"`
 	DefaultAnnotations map[string]string `tfsdk:"default_annotations"`
 	DefaultArchs       []string          `tfsdk:"default_archs"`
 }
 
 type ProviderOpts struct {
-	repositories, packages, keyring, archs []string
-	anns                                   map[string]string
-	ropts                                  []remote.Option
+	repositories, buildRespositories, packages, keyring, archs []string
+	anns                                                       map[string]string
+	ropts                                                      []remote.Option
 }
 
 func (p *Provider) Metadata(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) {
@@ -49,6 +50,11 @@ func (p *Provider) Schema(ctx context.Context, req provider.SchemaRequest, resp
 				Optional:    true,
 				ElementType: basetypes.StringType{},
 			},
+			"build_repositories": schema.ListAttribute{
+				Description: "Additional repositories to search for packages, only during apko build",
+				Optional:    true,
+				ElementType: basetypes.StringType{},
+			},
 			"extra_packages": schema.ListAttribute{
 				Description: "Additional packages to install",
 				Optional:    true,
@@ -112,12 +118,13 @@ func (p *Provider) Configure(ctx context.Context, req provider.ConfigureRequest,
 
 	opts := &ProviderOpts{
 		// This is only for testing, so we can inject provider config
-		repositories: append(p.repositories, data.ExtraRepositories...),
-		packages:     append(p.packages, data.ExtraPackages...),
-		keyring:      append(p.keyring, data.ExtraKeyring...),
-		archs:        append(p.archs, data.DefaultArchs...),
-		anns:         combineMaps(p.anns, data.DefaultAnnotations),
-		ropts:        ropts,
+		repositories:       append(p.repositories, data.ExtraRepositories...),
+		buildRespositories: append(p.buildRespositories, data.BuildRepositories...),
+		packages:           append(p.packages, data.ExtraPackages...),
+		keyring:            append(p.keyring, data.ExtraKeyring...),
+		archs:              append(p.archs, data.DefaultArchs...),
+		anns:               combineMaps(p.anns, data.DefaultAnnotations),
+		ropts:              ropts,
 	}
 
 	// Make provider opts available to resources and data sources.

internal/provider/resource_build_test.go

@@ -129,10 +129,11 @@ func TestAccResourceApkoBuild_ProviderOpts(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64", "aarch64"},
-				packages:     []string{"wolfi-baselayout"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64", "aarch64"},
+				packages:           []string{"wolfi-baselayout"},
 			}),
 		}, Steps: []resource.TestStep{
 			{
@@ -197,10 +198,11 @@ func TestAccResourceApkoBuild_BuildDateEpoch(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64"},
-				packages:     []string{"wolfi-baselayout=20230201-r0"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64"},
+				packages:           []string{"wolfi-baselayout=20230201-r0"},
 			}),
 		},
 		Steps: []resource.TestStep{{
@@ -266,10 +268,11 @@ resource "apko_build" "foo" {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64"},
-				packages:     []string{"wolfi-baselayout=20230201-r3"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64"},
+				packages:           []string{"wolfi-baselayout=20230201-r3"},
 			}),
 		},
 		Steps: []resource.TestStep{{
@@ -341,10 +344,11 @@ func TestAccResourceApkoBuild_OldPackages(t *testing.T) {
 		PreCheck: func() { testAccPreCheck(t) },
 		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 			"apko": providerserver.NewProtocol6WithError(&Provider{
-				repositories: []string{"https://packages.wolfi.dev/os"},
-				keyring:      []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
-				archs:        []string{"x86_64", "aarch64"},
-				packages:     []string{"wolfi-baselayout"},
+				repositories:       []string{"https://packages.wolfi.dev/os"},
+				buildRespositories: []string{"./packages"},
+				keyring:            []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"},
+				archs:              []string{"x86_64", "aarch64"},
+				packages:           []string{"wolfi-baselayout"},
 			}),
 		}, Steps: []resource.TestStep{
 			{