From acbd95c6fba298afcb3e5553560dcc3dc2b67fb9 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Mon, 17 Jun 2024 11:16:21 -0400 Subject: [PATCH] plumb through build repos (#329) Picks up https://github.com/chainguard-dev/apko/pull/1169 This lets us define repos to pull packages from at apko-build-time, which won't be available or visible via `apk update` or `apk add`. --------- Signed-off-by: Jason Hall --- docs/data-sources/config.md | 1 + docs/data-sources/tags.md | 1 + docs/index.md | 1 + docs/resources/build.md | 1 + go.mod | 6 ++-- go.sum | 12 +++---- internal/provider/build.go | 9 +++-- internal/provider/config_data_source.go | 7 ++-- internal/provider/config_data_source_test.go | 36 +++++++++++--------- internal/provider/provider.go | 29 ++++++++++------ internal/provider/resource_build_test.go | 36 +++++++++++--------- 11 files changed, 82 insertions(+), 57 deletions(-) diff --git a/docs/data-sources/config.md b/docs/data-sources/config.md index e933362d..01456439 100644 --- a/docs/data-sources/config.md +++ b/docs/data-sources/config.md @@ -83,6 +83,7 @@ Read-Only: Read-Only: +- `build_repositories` (List of String) - `keyring` (List of String) - `packages` (List of String) - `repositories` (List of String) diff --git a/docs/data-sources/tags.md b/docs/data-sources/tags.md index bd8b240d..cc74d547 100644 --- a/docs/data-sources/tags.md +++ b/docs/data-sources/tags.md @@ -82,6 +82,7 @@ Required: Required: +- `build_repositories` (List of String) - `keyring` (List of String) - `packages` (List of String) - `repositories` (List of String) diff --git a/docs/index.md b/docs/index.md index 1ab52b1e..81cb9c51 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,6 +21,7 @@ provider "apko" {} ### Optional +- `build_repositories` (List of String) Additional repositories to search for packages, only during apko build - `default_annotations` (Map of String) Default annotations to add - `default_archs` (List of String) Default architectures to build for - `extra_keyring` (List of String) Additional keys to use for package verification diff --git a/docs/resources/build.md b/docs/resources/build.md index f4a7ffed..52aa41f7 100644 --- a/docs/resources/build.md +++ b/docs/resources/build.md @@ -124,6 +124,7 @@ Required: Required: +- `build_repositories` (List of String) - `keyring` (List of String) - `packages` (List of String) - `repositories` (List of String) diff --git a/go.mod b/go.mod index 8fd8adcd..25b6cc8b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chainguard-dev/terraform-provider-apko go 1.22.3 require ( - chainguard.dev/apko v0.14.8 + chainguard.dev/apko v0.14.10-0.20240617143934-ac840f83c1c0 github.com/chainguard-dev/clog v1.3.1 github.com/chainguard-dev/terraform-provider-oci v0.0.13 github.com/google/go-cmp v0.6.0 @@ -16,7 +16,7 @@ require ( github.com/sigstore/cosign/v2 v2.2.4 golang.org/x/sync v0.7.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/apimachinery v0.30.1 + k8s.io/apimachinery v0.30.2 knative.dev/pkg v0.0.0-20240521083825-99e1685a7997 ) @@ -101,7 +101,7 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/letsencrypt/boulder v0.0.0-20240606225043-de8401e3454f // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 39883a4f..f5b95c54 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -chainguard.dev/apko v0.14.8 h1:UHnn4qm/erRppygHH8/0OEA+E72fnwAY2px/YaRaI8g= -chainguard.dev/apko v0.14.8/go.mod h1:aFEwAkFsf7sXvVFQ2ui6KRK3tbG3mIl5PmPd4JqKGVM= +chainguard.dev/apko v0.14.10-0.20240617143934-ac840f83c1c0 h1:M2W40pecL50Yq87YXfqXgkSRazcYWq4XBE03FAWBsUg= +chainguard.dev/apko v0.14.10-0.20240617143934-ac840f83c1c0/go.mod h1:Z7lctAs9bQinh3azYJ4+mfLM20A2xPvF9nWYwSCgGdM= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -233,8 +233,8 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -509,8 +509,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= -k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= diff --git a/internal/provider/build.go b/internal/provider/build.go index e459ae7a..fec53417 100644 --- a/internal/provider/build.go +++ b/internal/provider/build.go @@ -53,7 +53,8 @@ func fromImageData(ctx context.Context, ic types.ImageConfiguration, popts Provi build.WithImageConfiguration(ic), build.WithSBOMFormats([]string{"spdx"}), build.WithExtraKeys(popts.keyring), - build.WithExtraRepos(popts.repositories), + build.WithExtraRuntimeRepos(popts.repositories), + build.WithExtraBuildRepos(popts.buildRespositories), } o, ic2, err := build.NewOptions(opts...) @@ -133,7 +134,8 @@ func doBuild(ctx context.Context, data BuildResourceModel) (v1.Hash, coci.Signed build.WithSBOM(tempDir), build.WithArch(arch), build.WithExtraKeys(data.popts.keyring), - build.WithExtraRepos(data.popts.repositories))..., + build.WithExtraBuildRepos(data.popts.buildRespositories), + build.WithExtraRuntimeRepos(data.popts.repositories))..., ) if err != nil { return fmt.Errorf("failed to start apko build: %w", err) @@ -227,7 +229,8 @@ func doBuild(ctx context.Context, data BuildResourceModel) (v1.Hash, coci.Signed build.WithSBOMFormats([]string{"spdx"}), build.WithSBOM(tempDir), build.WithExtraKeys(data.popts.keyring), - build.WithExtraRepos(data.popts.repositories), + build.WithExtraRuntimeRepos(data.popts.repositories), + build.WithExtraBuildRepos(data.popts.buildRespositories), ) if err != nil { return v1.Hash{}, nil, nil, fmt.Errorf("failed to create options for index: %w", err) diff --git a/internal/provider/config_data_source.go b/internal/provider/config_data_source.go index 898027eb..c9e059a6 100644 --- a/internal/provider/config_data_source.go +++ b/internal/provider/config_data_source.go @@ -122,10 +122,12 @@ func (d *ConfigDataSource) Read(ctx context.Context, req datasource.ReadRequest, } tflog.Trace(ctx, fmt.Sprintf("got repos: %v", d.popts.repositories)) + tflog.Trace(ctx, fmt.Sprintf("got build repos: %v", d.popts.buildRespositories)) tflog.Trace(ctx, fmt.Sprintf("got keyring: %v", d.popts.keyring)) // Append any provider-specified repositories, packages, and keys, if specified. - ic.Contents.Repositories = sets.List(sets.New(ic.Contents.Repositories...).Insert(d.popts.repositories...)) + ic.Contents.RuntimeRepositories = sets.List(sets.New(ic.Contents.RuntimeRepositories...).Insert(d.popts.repositories...)) + ic.Contents.BuildRepositories = sets.List(sets.New(ic.Contents.BuildRepositories...).Insert(d.popts.buildRespositories...)) ic.Contents.Packages = sets.List(sets.New(ic.Contents.Packages...).Insert(d.popts.packages...)) ic.Contents.Keyring = sets.List(sets.New(ic.Contents.Keyring...).Insert(d.popts.keyring...)) @@ -246,7 +248,8 @@ func (d *ConfigDataSource) resolvePackageList(ctx context.Context, ic apkotypes. build.WithSBOMFormats([]string{"spdx"}), build.WithArch(arch), build.WithExtraKeys(d.popts.keyring), - build.WithExtraRepos(d.popts.repositories))..., + build.WithExtraBuildRepos(d.popts.buildRespositories), + build.WithExtraRuntimeRepos(d.popts.repositories))..., ) if err != nil { return err diff --git a/internal/provider/config_data_source_test.go b/internal/provider/config_data_source_test.go index 62b07c2f..b30c769f 100644 --- a/internal/provider/config_data_source_test.go +++ b/internal/provider/config_data_source_test.go @@ -59,10 +59,11 @@ func TestAccDataSourceConfig_ExtraPackages(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64", "aarch64"}, - packages: []string{"wolfi-baselayout=20230201-r0"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64", "aarch64"}, + packages: []string{"wolfi-baselayout=20230201-r0"}, anns: map[string]string{ "bar": "provider-provided", "baz": "provider-provided", @@ -106,10 +107,11 @@ func TestAccDataSourceConfig_ProviderOpts_Locked(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64", "aarch64"}, - packages: []string{"wolfi-baselayout=20230201-r0"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64", "aarch64"}, + packages: []string{"wolfi-baselayout=20230201-r0"}, }), }, Steps: []resource.TestStep{{ @@ -146,10 +148,11 @@ func TestAccDataSourceConfig_ProviderOpts_Unlocked(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64", "aarch64"}, - packages: []string{"wolfi-baselayout"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64", "aarch64"}, + packages: []string{"wolfi-baselayout"}, }), }, Steps: []resource.TestStep{{ @@ -184,10 +187,11 @@ func TestAccDataSourceConfig_ProviderOpts_OverrideArchitecture(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64", "aarch64"}, - packages: []string{"wolfi-baselayout"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64", "aarch64"}, + packages: []string{"wolfi-baselayout"}, }), }, Steps: []resource.TestStep{{ diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 6898da55..fad65e88 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -18,12 +18,13 @@ var _ provider.Provider = &Provider{} type Provider struct { version string - repositories, packages, keyring, archs []string - anns map[string]string + repositories, buildRespositories, packages, keyring, archs []string + anns map[string]string } type ProviderModel struct { ExtraRepositories []string `tfsdk:"extra_repositories"` + BuildRepositories []string `tfsdk:"build_repositories"` ExtraPackages []string `tfsdk:"extra_packages"` ExtraKeyring []string `tfsdk:"extra_keyring"` DefaultAnnotations map[string]string `tfsdk:"default_annotations"` @@ -31,9 +32,9 @@ type ProviderModel struct { } type ProviderOpts struct { - repositories, packages, keyring, archs []string - anns map[string]string - ropts []remote.Option + repositories, buildRespositories, packages, keyring, archs []string + anns map[string]string + ropts []remote.Option } func (p *Provider) Metadata(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { @@ -49,6 +50,11 @@ func (p *Provider) Schema(ctx context.Context, req provider.SchemaRequest, resp Optional: true, ElementType: basetypes.StringType{}, }, + "build_repositories": schema.ListAttribute{ + Description: "Additional repositories to search for packages, only during apko build", + Optional: true, + ElementType: basetypes.StringType{}, + }, "extra_packages": schema.ListAttribute{ Description: "Additional packages to install", Optional: true, @@ -112,12 +118,13 @@ func (p *Provider) Configure(ctx context.Context, req provider.ConfigureRequest, opts := &ProviderOpts{ // This is only for testing, so we can inject provider config - repositories: append(p.repositories, data.ExtraRepositories...), - packages: append(p.packages, data.ExtraPackages...), - keyring: append(p.keyring, data.ExtraKeyring...), - archs: append(p.archs, data.DefaultArchs...), - anns: combineMaps(p.anns, data.DefaultAnnotations), - ropts: ropts, + repositories: append(p.repositories, data.ExtraRepositories...), + buildRespositories: append(p.buildRespositories, data.BuildRepositories...), + packages: append(p.packages, data.ExtraPackages...), + keyring: append(p.keyring, data.ExtraKeyring...), + archs: append(p.archs, data.DefaultArchs...), + anns: combineMaps(p.anns, data.DefaultAnnotations), + ropts: ropts, } // Make provider opts available to resources and data sources. diff --git a/internal/provider/resource_build_test.go b/internal/provider/resource_build_test.go index f3e6d011..5656ee5c 100644 --- a/internal/provider/resource_build_test.go +++ b/internal/provider/resource_build_test.go @@ -129,10 +129,11 @@ func TestAccResourceApkoBuild_ProviderOpts(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64", "aarch64"}, - packages: []string{"wolfi-baselayout"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64", "aarch64"}, + packages: []string{"wolfi-baselayout"}, }), }, Steps: []resource.TestStep{ { @@ -197,10 +198,11 @@ func TestAccResourceApkoBuild_BuildDateEpoch(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64"}, - packages: []string{"wolfi-baselayout=20230201-r0"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64"}, + packages: []string{"wolfi-baselayout=20230201-r0"}, }), }, Steps: []resource.TestStep{{ @@ -266,10 +268,11 @@ resource "apko_build" "foo" { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64"}, - packages: []string{"wolfi-baselayout=20230201-r3"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64"}, + packages: []string{"wolfi-baselayout=20230201-r3"}, }), }, Steps: []resource.TestStep{{ @@ -341,10 +344,11 @@ func TestAccResourceApkoBuild_OldPackages(t *testing.T) { PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){ "apko": providerserver.NewProtocol6WithError(&Provider{ - repositories: []string{"https://packages.wolfi.dev/os"}, - keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, - archs: []string{"x86_64", "aarch64"}, - packages: []string{"wolfi-baselayout"}, + repositories: []string{"https://packages.wolfi.dev/os"}, + buildRespositories: []string{"./packages"}, + keyring: []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, + archs: []string{"x86_64", "aarch64"}, + packages: []string{"wolfi-baselayout"}, }), }, Steps: []resource.TestStep{ {