From d9a7416eabc3329f9884522b8f79c6f851cd9de0 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 16:54:31 +0300 Subject: [PATCH 01/16] feat(packmanager): Add options to specify package characteristics Signed-off-by: Cezar Craciunoiu --- packmanager/query.go | 60 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/packmanager/query.go b/packmanager/query.go index ecd34bdf6..daafe3d84 100644 --- a/packmanager/query.go +++ b/packmanager/query.go @@ -31,6 +31,18 @@ type Query struct { // Auth contains required authentication for the query. auths map[string]config.AuthConfig + + // allTargets indicates that the query should package all targets together + allTargets bool + + // Architecture specifies the architecture of the package + architecture string + + // Platform specifies the platform of the package + platform string + + // KConfig specifies the list of config options of the package + kConfig []string } // Source specifies where the origin of the package @@ -53,6 +65,26 @@ func (query *Query) Version() string { return query.version } +// AllTargets indicates that the query should package all targets together +func (query *Query) AllTargets() bool { + return query.allTargets +} + +// Architecture specifies the architecture of the package +func (query *Query) Architecture() string { + return query.architecture +} + +// Platform specifies the platform of the package +func (query *Query) Platform() string { + return query.platform +} + +// KConfig specifies the list of config options of the package +func (query *Query) KConfig() []string { + return query.kConfig +} + // UseCache indicates whether the package manager should use any existing cache. func (query *Query) UseCache() bool { return query.useCache @@ -87,6 +119,27 @@ func NewQuery(qopts ...QueryOption) *Query { return &query } +// WithArchitecture sets the query parameter for the architecture of the package. +func WithArchitecture(arch string) QueryOption { + return func(query *Query) { + query.architecture = arch + } +} + +// WithPlatform sets the query parameter for the platform of the package. +func WithPlatform(platform string) QueryOption { + return func(query *Query) { + query.platform = platform + } +} + +// WithKconfig sets the query parameter for the list of configuration options of the package. +func WithKConfig(kConfig []string) QueryOption { + return func(query *Query) { + query.kConfig = kConfig + } +} + // WithSource sets the query parameter for the origin source of the package. func WithSource(source string) QueryOption { return func(query *Query) { @@ -123,6 +176,13 @@ func WithCache(useCache bool) QueryOption { } } +// WithAllTargets sets whether to package all targets together. +func WithAllTargets(allTargets bool) QueryOption { + return func(query *Query) { + query.allTargets = allTargets + } +} + // WithAuthConfig sets the the required authorization for when making the query. func WithAuthConfig(auths map[string]config.AuthConfig) QueryOption { return func(query *Query) { From a1d697b9d7b0e275cf761cd2017c95de9a2c49f5 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 16:40:27 +0300 Subject: [PATCH 02/16] feat(pkg): Specify platform when pulling Signed-off-by: Cezar Craciunoiu --- cmd/kraft/pkg/pull/pull.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/kraft/pkg/pull/pull.go b/cmd/kraft/pkg/pull/pull.go index 8370db4b6..5a9d45d1e 100644 --- a/cmd/kraft/pkg/pull/pull.go +++ b/cmd/kraft/pkg/pull/pull.go @@ -256,6 +256,8 @@ func (opts *Pull) Run(cmd *cobra.Command, args []string) error { packmanager.WithSource(c.Source()), packmanager.WithTypes(c.Type()), packmanager.WithCache(opts.ForceCache), + packmanager.WithArchitecture(opts.Architecture), + packmanager.WithPlatform(opts.Platform), }, }) } @@ -263,7 +265,10 @@ func (opts *Pull) Run(cmd *cobra.Command, args []string) error { // Is this a list (space delimetered) of packages to pull? } else if len(args) > 0 { for _, arg := range args { - pm, compatible, err := pm.IsCompatible(ctx, arg) + pm, compatible, err := pm.IsCompatible(ctx, arg, + packmanager.WithArchitecture(opts.Architecture), + packmanager.WithPlatform(opts.Platform), + ) if err != nil || !compatible { continue } @@ -273,6 +278,8 @@ func (opts *Pull) Run(cmd *cobra.Command, args []string) error { query: []packmanager.QueryOption{ packmanager.WithCache(opts.ForceCache), packmanager.WithName(arg), + packmanager.WithArchitecture(opts.Architecture), + packmanager.WithPlatform(opts.Platform), }, }) } From 3e673016fa7b4407d68706152dac5b2f38a67add Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 16:41:27 +0300 Subject: [PATCH 03/16] feat(run): Specify platform when pulling packages Signed-off-by: Cezar Craciunoiu --- cmd/kraft/run/runner_package.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/kraft/run/runner_package.go b/cmd/kraft/run/runner_package.go index 9f90afd72..a30ac82b9 100644 --- a/cmd/kraft/run/runner_package.go +++ b/cmd/kraft/run/runner_package.go @@ -75,6 +75,8 @@ func (runner *runnerPackage) Prepare(ctx context.Context, opts *Run, machine *ma packmanager.WithTypes(unikraft.ComponentTypeApp), packmanager.WithName(runner.packName), packmanager.WithCache(true), + packmanager.WithPlatform(string(opts.platform)), + packmanager.WithArchitecture(opts.Architecture), ) if err != nil { return err @@ -88,6 +90,8 @@ func (runner *runnerPackage) Prepare(ctx context.Context, opts *Run, machine *ma packmanager.WithTypes(unikraft.ComponentTypeApp), packmanager.WithName(runner.packName), packmanager.WithCache(false), + packmanager.WithPlatform(string(opts.platform)), + packmanager.WithArchitecture(opts.Architecture), ) if err != nil { return err From c178eef3b77b92a9faaaa168983a77f7dd8c65c6 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:17:57 +0300 Subject: [PATCH 04/16] feat(oci): Add method to package several targets in a single package Signed-off-by: Cezar Craciunoiu --- oci/pack.go | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/oci/pack.go b/oci/pack.go index e18251d7b..748cc7709 100644 --- a/oci/pack.go +++ b/oci/pack.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" "oras.land/oras-go/v2/content" @@ -277,6 +278,164 @@ func NewPackageFromTarget(ctx context.Context, targ target.Target, opts ...packm return &ocipack, nil } +// NewPackageFromTargets generates an OCI implementation of the pack.Package +// construct based on an input Application and options. +func NewPackageFromTargets(ctx context.Context, targ []target.Target, opts ...packmanager.PackOption) (pack.Package, error) { + var packages []pack.Package + var manifests []ocispec.Manifest + var images []ocispec.Image + var handle handler.Handler + var ref name.Reference + var err error + + for _, t := range targ { + pkg, err := NewPackageFromTarget(ctx, t, opts...) + if err != nil { + return nil, err + } + + packages = append(packages, pkg) + manifests = append(manifests, pkg.(*ociPackage).image.manifest) + images = append(images, pkg.(*ociPackage).image.config) + } + + popts := &packmanager.PackOptions{} + for _, opt := range opts { + opt(popts) + } + + if popts.Output() != "" { + ref, err = name.ParseReference(popts.Output(), + name.WithDefaultRegistry(DefaultRegistry), + ) + } else if len(targ) == 1 { + // It's possible to pass an OCI artifact reference in the Kraftfile, e.g.: + // + // ```yaml + // [...] + // targets: + // - name: unikraft.io/library/helloworld:latest + // arch: x86_64 + // plat: kvm + // ``` + n := targ[0].Name() + if popts.Name() != "" { + n = popts.Name() + } + ref, err = name.ParseReference( + n, + name.WithDefaultRegistry(DefaultRegistry), + ) + } else if popts.Name() != "" { + ref, err = name.ParseReference( + popts.Name(), + name.WithDefaultRegistry(DefaultRegistry), + ) + } else { + return nil, fmt.Errorf("cannot generate OCI index without a specific name or path") + } + if err != nil { + return nil, err + } + + auths, err := defaultAuths(ctx) + if err != nil { + return nil, err + } + + if contAddr := config.G[config.KraftKit](ctx).ContainerdAddr; len(contAddr) > 0 { + namespace := DefaultNamespace + if n := os.Getenv("CONTAINERD_NAMESPACE"); n != "" { + namespace = n + } + + log.G(ctx).WithFields(logrus.Fields{ + "addr": contAddr, + "namespace": namespace, + }).Debug("oci: packaging via containerd") + + ctx, handle, err = handler.NewContainerdHandler(ctx, contAddr, namespace, auths) + } else { + if gerr := os.MkdirAll(config.G[config.KraftKit](ctx).RuntimeDir, fs.ModeSetgid|0o775); gerr != nil { + return nil, fmt.Errorf("could not create local oci cache directory: %w", gerr) + } + + ociDir := filepath.Join(config.G[config.KraftKit](ctx).RuntimeDir, "oci") + + log.G(ctx).WithFields(logrus.Fields{ + "path": ociDir, + }).Trace("oci: directory handler") + + handle, err = handler.NewDirectoryHandler(ociDir, auths) + } + if err != nil { + return nil, err + } + + index, err := NewImageIndex(ctx, handle) + if err != nil { + return nil, err + } + + index.manifests = manifests + index.images = images + + index.index.MediaType = ocispec.MediaTypeImageIndex + + // All packages in the index have the same name and version + index.SetAnnotation(ctx, AnnotationName, packages[0].Name()) + index.SetAnnotation(ctx, AnnotationVersion, packages[0].Version()) + index.SetAnnotation(ctx, AnnotationKraftKitVersion, kraftkitversion.Version()) + if version := popts.KernelVersion(); len(version) > 0 { + index.SetAnnotation(ctx, AnnotationKernelVersion, version) + } + + ociindex := ociIndex{ + handle: handle, + index: index, + ref: ref, + } + + _, err = index.Save(ctx, ociindex.imageRef(), nil) + if err != nil { + return nil, err + } + + return ociindex, nil +} + +// NewPackageFromOCIIndexSpec generates a package from a supplied OCI image +// index specification. +func NewPackageFromOCIIndexSpec(ctx context.Context, handle handler.Handler, ref string, index ocispec.Index) (pack.Package, error) { + // Check if the OCI image has a known annotation which identifies if a + // unikernel is contained within + if _, ok := index.Annotations[AnnotationKernelVersion]; !ok { + return nil, fmt.Errorf("OCI image does not contain a Unikraft unikernel") + } + + var err error + + ociindex := ociIndex{ + handle: handle, + } + + ociindex.ref, err = name.ParseReference(ref, + name.WithDefaultRegistry(DefaultRegistry), + ) + if err != nil { + return nil, err + } + + ociindex.index, err = NewImageIndex(ctx, handle) + if err != nil { + return nil, err + } + + ociindex.index.index = index + + return &ociindex, nil +} + // NewPackageFromOCIManifestSpec generates a package from a supplied OCI image // manifest specification. func NewPackageFromOCIManifestSpec(ctx context.Context, handle handler.Handler, ref string, manifest ocispec.Manifest) (pack.Package, error) { From 876a1b77c61b818309c406510dd9f690c22a110c Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 16:59:59 +0300 Subject: [PATCH 05/16] feat!: Update package managers to support packing multiple components Signed-off-by: Cezar Craciunoiu --- manifest/manager.go | 2 +- oci/manager.go | 16 +++++++++++----- packmanager/manager.go | 4 ++-- packmanager/umbrella.go | 12 +++++++----- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/manifest/manager.go b/manifest/manager.go index 4cee5430a..61e1a131e 100644 --- a/manifest/manager.go +++ b/manifest/manager.go @@ -176,7 +176,7 @@ func (m *manifestManager) RemoveSource(ctx context.Context, source string) error return nil } -func (m *manifestManager) Pack(ctx context.Context, c component.Component, opts ...packmanager.PackOption) ([]pack.Package, error) { +func (m *manifestManager) Pack(ctx context.Context, c []component.Component, opts ...packmanager.PackOption) ([]pack.Package, error) { return nil, fmt.Errorf("not implemented manifest.manager.Pack") } diff --git a/oci/manager.go b/oci/manager.go index 2699d79c2..f0275c3ab 100644 --- a/oci/manager.go +++ b/oci/manager.go @@ -65,13 +65,19 @@ func (manager *ociManager) Update(ctx context.Context) error { } // Pack implements packmanager.PackageManager -func (manager *ociManager) Pack(ctx context.Context, entity component.Component, opts ...packmanager.PackOption) ([]pack.Package, error) { - targ, ok := entity.(target.Target) - if !ok { - return nil, fmt.Errorf("entity is not Unikraft target") +func (manager *ociManager) Pack(ctx context.Context, entities []component.Component, opts ...packmanager.PackOption) ([]pack.Package, error) { + var targets []target.Target + + for _, entity := range entities { + targ, ok := entity.(target.Target) + if !ok { + return nil, fmt.Errorf("entity is not Unikraft target") + } + + targets = append(targets, targ) } - pkg, err := NewPackageFromTarget(ctx, targ, opts...) + pkg, err := NewPackageFromTargets(ctx, targets, opts...) if err != nil { return nil, err } diff --git a/packmanager/manager.go b/packmanager/manager.go index 6e2b0a900..6efc0e2fd 100644 --- a/packmanager/manager.go +++ b/packmanager/manager.go @@ -22,11 +22,11 @@ type PackageManager interface { // Update retrieves and stores locally a cache of the upstream registry. Update(context.Context) error - // Pack turns the provided component into the distributable package. Since + // Pack turns the provided components into the distributable package. Since // components can comprise of other components, it is possible to return more // than one package. It is possible to disable this and "flatten" a component // into a single package by setting a relevant `pack.PackOption`. - Pack(context.Context, component.Component, ...PackOption) ([]pack.Package, error) + Pack(context.Context, []component.Component, ...PackOption) ([]pack.Package, error) // Unpack turns a given package into a usable component. Since a package can // compromise of a multiple components, it is possible to return multiple diff --git a/packmanager/umbrella.go b/packmanager/umbrella.go index 3fd1c0a8c..229e5ddca 100644 --- a/packmanager/umbrella.go +++ b/packmanager/umbrella.go @@ -109,14 +109,16 @@ func (u UmbrellaManager) RemoveSource(ctx context.Context, source string) error return nil } -func (u UmbrellaManager) Pack(ctx context.Context, source component.Component, opts ...PackOption) ([]pack.Package, error) { +func (u UmbrellaManager) Pack(ctx context.Context, source []component.Component, opts ...PackOption) ([]pack.Package, error) { var ret []pack.Package for _, manager := range u.packageManagers { - log.G(ctx).WithFields(logrus.Fields{ - "format": manager.Format(), - "source": source.Name(), - }).Tracef("packing") + for _, component := range source { + log.G(ctx).WithFields(logrus.Fields{ + "format": manager.Format(), + "source": component.Name(), + }).Tracef("packing") + } more, err := manager.Pack(ctx, source, opts...) if err != nil { return nil, err From 8469e14e0d3d0ad4b25428f85abca2ce176a0b2a Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 16:43:13 +0300 Subject: [PATCH 06/16] feat(pkg): Package multiple artifacts of a project together Signed-off-by: Cezar Craciunoiu --- cmd/kraft/pkg/pkg.go | 170 ++++++++++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 68 deletions(-) diff --git a/cmd/kraft/pkg/pkg.go b/cmd/kraft/pkg/pkg.go index 6e8e90415..7932e70e9 100644 --- a/cmd/kraft/pkg/pkg.go +++ b/cmd/kraft/pkg/pkg.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "os" + "strings" "github.com/MakeNowJust/heredoc" "github.com/mattn/go-shellwords" @@ -23,6 +24,8 @@ import ( "kraftkit.sh/packmanager" "kraftkit.sh/tui/processtree" "kraftkit.sh/unikraft/app" + "kraftkit.sh/unikraft/component" + "kraftkit.sh/unikraft/target" "kraftkit.sh/cmd/kraft/pkg/list" "kraftkit.sh/cmd/kraft/pkg/pull" @@ -33,19 +36,19 @@ import ( ) type Pkg struct { - Architecture string `local:"true" long:"arch" short:"m" usage:"Filter the creation of the package by architecture of known targets"` - Args string `local:"true" long:"args" short:"a" usage:"Pass arguments that will be part of the running kernel's command line"` - Dbg bool `local:"true" long:"dbg" usage:"Package the debuggable (symbolic) kernel image instead of the stripped image"` - Force bool `local:"true" long:"force-format" usage:"Force the use of a packaging handler format"` - Format string `local:"true" long:"as" short:"M" usage:"Force the packaging despite possible conflicts" default:"auto"` - Initrd string `local:"true" long:"initrd" short:"i" usage:"Path to init ramdisk to bundle within the package (passing a path will automatically generate a CPIO image)"` - Kernel string `local:"true" long:"kernel" short:"k" usage:"Override the path to the unikernel image"` - Kraftfile string `long:"kraftfile" usage:"Set an alternative path of the Kraftfile"` - Name string `local:"true" long:"name" short:"n" usage:"Specify the name of the package"` - Output string `local:"true" long:"output" short:"o" usage:"Save the package at the following output"` - Platform string `local:"true" long:"plat" short:"p" usage:"Filter the creation of the package by platform of known targets"` - Target string `local:"true" long:"target" short:"t" usage:"Package a particular known target"` - WithKConfig bool `local:"true" long:"with-kconfig" usage:"Include the target .config"` + Architecture string `local:"true" long:"arch" short:"m" usage:"Filter the creation of the package by architecture of known targets"` + Args string `local:"true" long:"args" short:"a" usage:"Pass arguments that will be part of the running kernel's command line"` + Dbg bool `local:"true" long:"dbg" usage:"Package the debuggable (symbolic) kernel image instead of the stripped image"` + Force bool `local:"true" long:"force-format" usage:"Force the use of a packaging handler format"` + Format string `local:"true" long:"as" short:"M" usage:"Force the packaging despite possible conflicts" default:"auto"` + Initrd string `local:"true" long:"initrd" short:"i" usage:"Path to init ramdisk to bundle within the package (passing a path will automatically generate a CPIO image)"` + Kernel string `local:"true" long:"kernel" short:"k" usage:"Override the path to the unikernel image"` + Kraftfile string `long:"kraftfile" usage:"Set an alternative path of the Kraftfile"` + Name string `local:"true" long:"name" short:"n" usage:"Specify the name of the package"` + Output string `local:"true" long:"output" short:"o" usage:"Save the package at the following output"` + Platform string `local:"true" long:"plat" short:"p" usage:"Filter the creation of the package by platform of known targets"` + Targets []string `local:"true" long:"target" short:"t" usage:"Package a particular known target"` + WithKConfig bool `local:"true" long:"with-kconfig" usage:"Include the target .config"` } func New() *cobra.Command { @@ -66,7 +69,17 @@ func New() *cobra.Command { `, "`"), Example: heredoc.Doc(` # Package a project as an OCI archive and embed the target's KConfig. - $ kraft pkg --as oci --name unikraft.org/nginx:latest --with-kconfig`), + $ kraft pkg --as oci --name unikraft.org/nginx:latest --with-kconfig + + # Package a project with a given target name + $ kraft pkg --as oci --name unikraft.org/nginx:latest --target my_target + + # Package a project with a given platform and architecture + $ kraft pkg --as oci --name unikraft.org/nginx:latest --plat qemu --arch x86_64 + + # Package a project with multiple targets + $ kraft pkg --as oci --name unikraft.org/nginx:latest --target my_target --target my_other_target + `), Annotations: map[string]string{ cmdfactory.AnnotationHelpGroup: "pkg", }, @@ -86,7 +99,7 @@ func New() *cobra.Command { } func (opts *Pkg) Pre(cmd *cobra.Command, _ []string) error { - if (len(opts.Architecture) > 0 || len(opts.Platform) > 0) && len(opts.Target) > 0 { + if (len(opts.Architecture) > 0 || len(opts.Platform) > 0) && len(opts.Targets) > 0 { return fmt.Errorf("the `--arch` and `--plat` options are not supported in addition to `--target`") } @@ -102,6 +115,22 @@ func (opts *Pkg) Pre(cmd *cobra.Command, _ []string) error { return nil } +func matchOption(targets []string, option string) bool { + for _, target := range targets { + name := target + if strings.Contains(target, "/") { + name = string(platform.PlatformByName(strings.Split(target, "/")[0])) + + "/" + + strings.Split(target, "/")[1] + } + if name == option { + return true + } + } + + return false +} + func (opts *Pkg) Run(cmd *cobra.Command, args []string) error { var err error var workdir string @@ -134,11 +163,13 @@ func (opts *Pkg) Run(cmd *cobra.Command, args []string) error { } var tree []*processtree.ProcessTreeItem + var targets []target.Target + var packageOpts []packmanager.PackOption parallel := !config.G[config.KraftKit](ctx).NoParallel norender := log.LoggerTypeFromString(config.G[config.KraftKit](ctx).Log.Type) != log.FANCY - // Generate a package for every matching requested target + // Generate a package with all matching requested targets for _, targ := range project.Targets() { // See: https://github.com/golang/go/wiki/CommonMistakes#using-reference-to-loop-iterator-variable targ := targ @@ -146,13 +177,17 @@ func (opts *Pkg) Run(cmd *cobra.Command, args []string) error { switch true { case // If no arguments are supplied - len(opts.Target) == 0 && + len(opts.Targets) == 0 && len(opts.Architecture) == 0 && len(opts.Platform) == 0, // If the --target flag is supplied and the target name match - len(opts.Target) > 0 && - targ.Name() == opts.Target, + len(opts.Targets) > 0 && + matchOption(opts.Targets, targ.Name()), + + // If the --target flag is supplied and the target is a plat/arch combo + len(opts.Targets) > 0 && + matchOption(opts.Targets, target.TargetPlatArchName(targ)), // If only the --arch flag is supplied and the target's arch matches len(opts.Architecture) > 0 && @@ -170,68 +205,67 @@ func (opts *Pkg) Run(cmd *cobra.Command, args []string) error { targ.Architecture().Name() == opts.Architecture && targ.Platform().Name() == opts.Platform: - var format pack.PackageFormat - name := "packaging " + targ.Name() - if opts.Format != "auto" { - format = pack.PackageFormat(opts.Format) - } else if targ.Format().String() != "" { - format = targ.Format() - } - if format.String() != "auto" { - name += " (" + format.String() + ")" - } - cmdShellArgs, err := shellwords.Parse(opts.Args) if err != nil { return err } - tree = append(tree, processtree.NewProcessTreeItem( - name, - targ.Architecture().Name()+"/"+targ.Platform().Name(), - func(ctx context.Context) error { - var err error - pm := packmanager.G(ctx) - - // Switch the package manager the desired format for this target - if format != "auto" { - pm, err = pm.From(format) - if err != nil { - return err - } - } - - popts := []packmanager.PackOption{ - packmanager.PackArgs(cmdShellArgs...), - packmanager.PackInitrd(opts.Initrd), - packmanager.PackKConfig(opts.WithKConfig), - packmanager.PackName(opts.Name), - packmanager.PackOutput(opts.Output), - } - - if ukversion, ok := targ.KConfig().Get(unikraft.UK_FULLVERSION); ok { - popts = append(popts, - packmanager.PackWithKernelVersion(ukversion.Value), - ) - } - - if _, err := pm.Pack(ctx, targ, popts...); err != nil { - return err - } - - return nil - }, - )) + packageOpts = append(packageOpts, + packmanager.PackArgs(cmdShellArgs...), + packmanager.PackInitrd(opts.Initrd), + packmanager.PackKConfig(opts.WithKConfig), + packmanager.PackName(opts.Name), + packmanager.PackOutput(opts.Output), + ) + + if ukversion, ok := targ.KConfig().Get(unikraft.UK_FULLVERSION); ok { + packageOpts = append(packageOpts, + packmanager.PackWithKernelVersion(ukversion.Value), + ) + } + + targets = append(targets, targ) default: continue } } + var components []component.Component + var targetNames string + + // Convert targets from []target.Target to []component.Component + for _, targ := range targets { + components = append(components, targ) + targetNames += targ.Name() + " " + } + + tree = append(tree, processtree.NewProcessTreeItem( + opts.Output, + targetNames, + func(ctx context.Context) error { + pm := packmanager.G(ctx) + + // Switch the package manager the desired format for this target + if opts.Format != "auto" { + pm, err = pm.From(pack.PackageFormat(opts.Format)) + if err != nil { + return err + } + } + + if _, err := pm.Pack(ctx, components, packageOpts...); err != nil { + return err + } + + return nil + }, + )) + if len(tree) == 0 { switch true { - case len(opts.Target) > 0: - return fmt.Errorf("no matching targets found for: %s", opts.Target) + case len(opts.Targets) > 0: + return fmt.Errorf("no matching targets found for: %s", opts.Targets) case len(opts.Architecture) > 0 && len(opts.Platform) == 0: return fmt.Errorf("no matching targets found for architecture: %s", opts.Architecture) case len(opts.Architecture) == 0 && len(opts.Platform) > 0: From 75404a3d2db60bc4dea6e17ccad421bf052ad641 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 16:46:34 +0300 Subject: [PATCH 07/16] feat(push): Add option to specify pushing multi-artifact packages Signed-off-by: Cezar Craciunoiu --- cmd/kraft/pkg/push/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kraft/pkg/push/push.go b/cmd/kraft/pkg/push/push.go index 19f85542d..34ded8907 100644 --- a/cmd/kraft/pkg/push/push.go +++ b/cmd/kraft/pkg/push/push.go @@ -125,6 +125,7 @@ func (opts *Push) Run(cmd *cobra.Command, args []string) error { if pm, compatible, err := pmananger.IsCompatible(ctx, ref); err == nil && compatible { packages, err := pm.Catalog(ctx, packmanager.WithCache(true), + packmanager.WithAllTargets(true), packmanager.WithName(ref), ) if err != nil { @@ -138,7 +139,6 @@ func (opts *Push) Run(cmd *cobra.Command, args []string) error { } // Call push if it exists - // TODO push if it doesn't exist too proc := paraprogress.NewProcess( fmt.Sprintf("pushing %s", ref, From a3243bc41d7ecaca5c81a1c07c8bc3b1f918f618 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:09:04 +0300 Subject: [PATCH 08/16] feat(oci): Add image index structure to handle creating indexes Signed-off-by: Cezar Craciunoiu --- oci/image_options.go | 18 ++++++ oci/index.go | 142 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 oci/index.go diff --git a/oci/image_options.go b/oci/image_options.go index 690273f03..cf916f93e 100644 --- a/oci/image_options.go +++ b/oci/image_options.go @@ -22,3 +22,21 @@ func WithAutoSave(autoSave bool) ImageOption { return nil } } + +type ImageIndexOption func(*ImageIndex) error + +// WithIndexAutoSave atomicizes the index as operations occur on its body. +func WithIndexAutoSave(autoSave bool) ImageIndexOption { + return func(index *ImageIndex) error { + index.autoSave = autoSave + return nil + } +} + +// WithIndexWorkdir specifies the working directory of the index. +func WithIndexWorkdir(workdir string) ImageIndexOption { + return func(index *ImageIndex) error { + index.workdir = workdir + return nil + } +} diff --git a/oci/index.go b/oci/index.go new file mode 100644 index 000000000..1ea242638 --- /dev/null +++ b/oci/index.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2022, Unikraft GmbH and The KraftKit Authors. +// Licensed under the BSD-3-Clause License (the "License"). +// You may not use this file except in compliance with the License. +package oci + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/google/go-containerregistry/pkg/name" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + + "kraftkit.sh/oci/handler" +) + +type ImageIndex struct { + workdir string + autoSave bool + + handle handler.Handler + + index ocispec.Index + indexDesc ocispec.Descriptor + + manifests []ocispec.Manifest + images []ocispec.Image + annotations map[string]string +} + +func NewImageIndex(_ context.Context, handle handler.Handler, opts ...ImageIndexOption) (*ImageIndex, error) { + if handle == nil { + return nil, fmt.Errorf("cannot use `NewImageIndex` without handler") + } + + index := ImageIndex{ + handle: handle, + autoSave: true, + } + + for _, opt := range opts { + if err := opt(&index); err != nil { + return nil, err + } + } + + return &index, nil +} + +// AddManifest adds a manifest directly to the index and returns the resulting +// descriptor +func (index *ImageIndex) AddManifest(ctx context.Context, manifest *ocispec.Manifest, image *ocispec.Image) (ocispec.Descriptor, error) { + manifestJson, err := json.Marshal(manifest) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err) + } + + descriptor := content.NewDescriptorFromBytes( + ocispec.MediaTypeImageManifest, + manifestJson, + ) + + descriptor.Platform = &ocispec.Platform{ + Architecture: image.Architecture, + OS: image.OS, + OSFeatures: image.OSFeatures, + } + + index.index.Manifests = append(index.index.Manifests, descriptor) + + return descriptor, nil +} + +// SetAnnotation sets an anootation of the index with the provided key. +func (index *ImageIndex) SetAnnotation(_ context.Context, key, val string) { + if index.annotations == nil { + index.annotations = make(map[string]string) + } + + index.annotations[key] = val +} + +// Save the index. +func (index *ImageIndex) Save(ctx context.Context, source string, onProgress func(float64)) (ocispec.Descriptor, error) { + ref, err := name.ParseReference(source, + name.WithDefaultRegistry(DefaultRegistry), + ) + if err != nil { + return ocispec.Descriptor{}, err + } + + // General annotations + index.annotations[ocispec.AnnotationRefName] = ref.Context().String() + index.annotations[ocispec.AnnotationRevision] = ref.Identifier() + index.annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339) + + // containerd compatibility annotations + index.annotations[images.AnnotationImageName] = ref.String() + + // Add Manifests + for idx, manifest := range index.manifests { + _, err := index.AddManifest(ctx, &manifest, &index.images[idx]) + if err != nil { + return ocispec.Descriptor{}, err + } + } + + index.index.Annotations = index.annotations + index.index.SchemaVersion = 2 + + indexJson, err := json.Marshal(index.index) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err) + } + + index.indexDesc = content.NewDescriptorFromBytes( + ocispec.MediaTypeImageIndex, + indexJson, + ) + index.indexDesc.ArtifactType = index.index.ArtifactType + index.indexDesc.Size = int64(len(indexJson)) + + // save the manifest digest + if err := index.handle.SaveDigest( + ctx, + source, + index.indexDesc, + bytes.NewReader(indexJson), + onProgress, + ); err != nil && !errors.Is(err, errdefs.ErrAlreadyExists) { + return ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err) + } + + return index.indexDesc, nil +} From d0b87a97c12f3a913e27c945f7b05b9b0bd84807 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:12:46 +0300 Subject: [PATCH 09/16] feat(oci): Implement OCI Index structure to wrap over OCI packages Signed-off-by: Cezar Craciunoiu --- oci/pack.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/oci/pack.go b/oci/pack.go index 748cc7709..bd6cb43a9 100644 --- a/oci/pack.go +++ b/oci/pack.go @@ -52,6 +52,12 @@ type ociPackage struct { command []string } +type ociIndex struct { + handle handler.Handler + ref name.Reference + index *ImageIndex +} + var ( _ pack.Package = (*ociPackage)(nil) _ target.Target = (*ociPackage)(nil) @@ -482,8 +488,9 @@ func NewPackageFromOCIManifestSpec(ctx context.Context, handle handler.Handler, // NewPackageFromRemoteOCIRef generates a new package from a given OCI image // reference which is accessed by its remote registry. -func NewPackageFromRemoteOCIRef(ctx context.Context, handle handler.Handler, ref string) (pack.Package, error) { +func NewPackageFromRemoteOCIRef(ctx context.Context, handle handler.Handler, ref string, packPlat *v1.Platform) (pack.Package, error) { var err error + var copts []crane.Option ocipack := ociPackage{ handle: handle, @@ -512,11 +519,17 @@ func NewPackageFromRemoteOCIRef(ctx context.Context, handle handler.Handler, ref authConfig.Username = auth.Username } - raw, err := crane.Manifest(ref, + copts = append(copts, crane.WithAuth(&simpleauth.SimpleAuthenticator{ Auth: authConfig, }), ) + + if packPlat != nil { + copts = append(copts, crane.WithPlatform(packPlat)) + } + + raw, err := crane.Manifest(ref, copts...) if err != nil { return nil, fmt.Errorf("could not get manifest: %v", err) } @@ -574,16 +587,31 @@ func (ocipack *ociPackage) Type() unikraft.ComponentType { return unikraft.ComponentTypeApp } +// Type implements unikraft.Nameable +func (ociidx ociIndex) Type() unikraft.ComponentType { + return unikraft.ComponentTypeApp +} + // Name implements unikraft.Nameable func (ocipack *ociPackage) Name() string { return ocipack.ref.Context().Name() } +// Name implements unikraft.Nameable +func (ociidx ociIndex) Name() string { + return ociidx.ref.Context().Name() +} + // Version implements unikraft.Nameable func (ocipack *ociPackage) Version() string { return ocipack.ref.Identifier() } +// Version implements unikraft.Nameable +func (ociidx ociIndex) Version() string { + return ociidx.ref.Identifier() +} + // imageRef returns the OCI-standard image name in the format `name:tag` func (ocipack *ociPackage) imageRef() string { if strings.HasPrefix(ocipack.Version(), "sha256:") { @@ -592,23 +620,41 @@ func (ocipack *ociPackage) imageRef() string { return fmt.Sprintf("%s:%s", ocipack.Name(), ocipack.Version()) } +// imageRef returns the OCI-standard image name in the format `name:tag` +func (ociidx ociIndex) imageRef() string { + if strings.HasPrefix(ociidx.Version(), "sha256:") { + return fmt.Sprintf("%s@%s", ociidx.Name(), ociidx.Version()) + } + return fmt.Sprintf("%s:%s", ociidx.Name(), ociidx.Version()) +} + // Metadata implements pack.Package func (ocipack *ociPackage) Metadata() any { return ocipack.image.config } +// Metadata implements pack.Package +func (ociidx ociIndex) Metadata() any { + return ociidx.index.index.MediaType +} + // Push implements pack.Package func (ocipack *ociPackage) Push(ctx context.Context, opts ...pack.PushOption) error { - manifestJson, err := json.Marshal(ocipack.image.manifest) + return fmt.Errorf("cannot push target without OCI image index") +} + +// Push implements pack.Package +func (ociidx ociIndex) Push(ctx context.Context, opts ...pack.PushOption) error { + indexJson, err := json.Marshal(ociidx.index.index) if err != nil { return fmt.Errorf("failed to marshal manifest: %w", err) } - ocipack.image.manifestDesc = content.NewDescriptorFromBytes( - ocispec.MediaTypeImageManifest, - manifestJson, + ociidx.index.indexDesc = content.NewDescriptorFromBytes( + ocispec.MediaTypeImageIndex, + indexJson, ) - return ocipack.image.handle.PushImage(ctx, ocipack.imageRef(), &ocipack.image.manifestDesc) + return ociidx.index.handle.PushImage(ctx, ociidx.imageRef(), &ociidx.index.indexDesc) } // Pull implements pack.Package @@ -626,9 +672,11 @@ func (ocipack *ociPackage) Pull(ctx context.Context, opts ...pack.PullOption) er } } + plat := fmt.Sprintf("%s/%s", popts.Platform(), pullArch) + // If it's possible to resolve the image reference, the image has already been // pulled to the local image store - image, err := ocipack.handle.ResolveImage(ctx, ocipack.imageRef()) + image, err := ocipack.handle.ResolveImage(ctx, ocipack.imageRef(), plat) if err == nil { goto unpack } @@ -643,7 +691,7 @@ func (ocipack *ociPackage) Pull(ctx context.Context, opts ...pack.PullOption) er } // Try resolving the image again after pulling it - image, err = ocipack.handle.ResolveImage(ctx, ocipack.imageRef()) + image, err = ocipack.handle.ResolveImage(ctx, ocipack.imageRef(), plat) if err != nil { return err } @@ -654,6 +702,7 @@ unpack: if err := ocipack.image.handle.UnpackImage( ctx, ocipack.imageRef(), + fmt.Sprintf("%s/%s", popts.Platform(), pullArch), popts.Workdir(), ); err != nil { return err @@ -679,11 +728,19 @@ unpack: return nil } -// Pull implements pack.Package +func (ociidx ociIndex) Pull(ctx context.Context, opts ...pack.PullOption) error { + return fmt.Errorf("cannot pull OCI image index") +} + +// Format implements pack.Package func (ocipack *ociPackage) Format() pack.PackageFormat { return OCIFormat } +func (ociidx ociIndex) Format() pack.PackageFormat { + return OCIFormat +} + // Source implements unikraft.component.Component func (ocipack *ociPackage) Source() string { return "" From 12d1c4f72c06eb8b0e56f47fcd6d3b06038ef937 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:25:05 +0300 Subject: [PATCH 10/16] feat(oci): Implement index listing Signed-off-by: Cezar Craciunoiu --- oci/handler/directory.go | 54 ++++++++++++++++++++++++++++++++++++++++ oci/handler/handler.go | 7 +++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/oci/handler/directory.go b/oci/handler/directory.go index add88a912..e13d59363 100644 --- a/oci/handler/directory.go +++ b/oci/handler/directory.go @@ -28,6 +28,7 @@ import ( ) const ( + DirectoryHandlerIndexesDir = "indexes" DirectoryHandlerManifestsDir = "manifests" DirectoryHandlerConfigsDir = "configs" DirectoryHandlerLayersDir = "layers" @@ -65,6 +66,59 @@ func (handle *DirectoryHandler) DigestExists(ctx context.Context, dgst digest.Di return false, nil } +// ListIndexes implements DigestResolver. +func (handle *DirectoryHandler) ListIndexes(ctx context.Context) (indexes []ocispec.Index, err error) { + indexesDir := filepath.Join(handle.path, DirectoryHandlerIndexesDir) + + // Create the manifest directory if it does not exist and return nil, since + // there's nothing to return. + if _, err := os.Stat(indexesDir); err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(indexesDir, 0o775); err != nil { + return nil, fmt.Errorf("could not create local oci cache directory: %w", err) + } + + return nil, nil + } + + // Since the directory structure is nested, recursively walk the manifest + // directory to find all manifest entries. + if err := filepath.WalkDir(indexesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip directories + if d.IsDir() { + return nil + } + + // Skip files that don't end in .json + if !strings.HasSuffix(d.Name(), ".json") { + return nil + } + + // Read the manifest + rawIndex, err := os.ReadFile(path) + if err != nil { + return err + } + + index := ocispec.Index{} + if err = json.Unmarshal(rawIndex, &index); err != nil { + return err + } + + // Append the manifest to the list + indexes = append(indexes, index) + + return nil + }); err != nil { + return nil, err + } + + return indexes, nil +} + // ListManifests implements DigestResolver. func (handle *DirectoryHandler) ListManifests(ctx context.Context) (manifests []ocispec.Manifest, err error) { manifestsDir := filepath.Join(handle.path, DirectoryHandlerManifestsDir) diff --git a/oci/handler/handler.go b/oci/handler/handler.go index afe42c5aa..137c9dd8c 100644 --- a/oci/handler/handler.go +++ b/oci/handler/handler.go @@ -28,6 +28,10 @@ type ManifestLister interface { ListManifests(context.Context) ([]ocispec.Manifest, error) } +type IndexLister interface { + ListIndexes(context.Context) ([]ocispec.Index, error) +} + type ImagePusher interface { PushImage(context.Context, string, *ocispec.Descriptor) error } @@ -41,13 +45,14 @@ type ImageFetcher interface { } type ImageUnpacker interface { - UnpackImage(context.Context, string, string) error + UnpackImage(context.Context, string, string, string) error } type Handler interface { DigestResolver DigestSaver ManifestLister + IndexLister ImagePusher ImageResolver ImageFetcher From 915aeb9501d96a441739b1954394ad67802aa54f Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:22:36 +0300 Subject: [PATCH 11/16] feat(oci): Filter by platform and add index filtering Signed-off-by: Cezar Craciunoiu --- oci/manager.go | 244 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 179 insertions(+), 65 deletions(-) diff --git a/oci/manager.go b/oci/manager.go index f0275c3ab..6439f4370 100644 --- a/oci/manager.go +++ b/oci/manager.go @@ -17,6 +17,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "kraftkit.sh/config" @@ -129,6 +130,19 @@ func (manager *ociManager) Catalog(ctx context.Context, qopts ...packmanager.Que query := packmanager.NewQuery(qopts...) qname := query.Name() qversion := query.Version() + platform := &v1.Platform{} + + if query.Architecture() != "" { + platform.Architecture = query.Architecture() + } + + if query.Platform() != "" { + platform.OS = query.Platform() + } + + if len(query.KConfig()) > 0 { + platform.OSFeatures = query.KConfig() + } // Adjust for the version being suffixed in a prototypical OCI reference // format @@ -154,7 +168,7 @@ func (manager *ociManager) Catalog(ctx context.Context, qopts ...packmanager.Que if !query.UseCache() { // If a direct reference can be made, attempt to generate a package from it if refErr == nil { - pack, err := NewPackageFromRemoteOCIRef(ctx, handle, ref.String()) + pack, err := NewPackageFromRemoteOCIRef(ctx, handle, ref.String(), platform) if err != nil { log.G(ctx).Trace(err) } else { @@ -222,80 +236,177 @@ func (manager *ociManager) Catalog(ctx context.Context, qopts ...packmanager.Que } } - // Access local images that are available on the host - manifests, err := handle.ListManifests(ctx) - if err != nil { - return nil, err - } - - for _, manifest := range manifests { - // Check if the OCI image has a known annotation which identifies if a - // unikernel is contained within - if _, ok := manifest.Annotations[AnnotationKernelVersion]; !ok { - log.G(ctx). - WithField("ref", manifest.Config.Digest.String()). - Trace("skipping non-unikernel digest") - continue + if !query.AllTargets() { + // Access local images that are available on the host + manifests, err := handle.ListManifests(ctx) + if err != nil { + return nil, err } - // Could not determine name from manifest specification - refname, ok := manifest.Annotations[ocispec.AnnotationRefName] - if !ok { - log.G(ctx). - WithField("ref", manifest.Config.Digest.String()). - Trace("skipping non-unikernel digest") - continue - } + for _, manifest := range manifests { + // Check if the OCI image has a known annotation which identifies if a + // unikernel is contained within + if _, ok := manifest.Annotations[AnnotationKernelVersion]; !ok { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + Trace("skipping non-unikernel digest") + continue + } - // Skip if querying for the name and the name does not match - if len(qname) > 0 && refname != qname { - log.G(ctx). - WithField("ref", manifest.Config.Digest.String()). - Trace("skipping non-unikernel digest") - continue - } + // Could not determine name from manifest specification + refname, ok := manifest.Annotations[ocispec.AnnotationRefName] + if !ok { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + Trace("skipping non-unikernel digest") + continue + } - // Could not determine name from manifest specification - revision, ok := manifest.Annotations[ocispec.AnnotationRevision] - if !ok { - log.G(ctx). - WithField("ref", manifest.Config.Digest.String()). - Trace("skipping non-unikernel digest") - continue - } + // Skip if querying for the name and the name does not match + if len(qname) > 0 && refname != qname { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + Trace("skipping non-unikernel digest") + continue + } - fullref := fmt.Sprintf("%s:%s", refname, revision) + // Could not determine name from manifest specification + revision, ok := manifest.Annotations[ocispec.AnnotationRevision] + if !ok { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + Trace("skipping non-unikernel digest") + continue + } - // Skip direct references from the remote registry - if !query.UseCache() && refErr == nil && ref.String() == fullref { - log.G(ctx). - WithField("ref", manifest.Config.Digest.String()). - Trace("skipping non-unikernel digest") - continue - } + fullref := fmt.Sprintf("%s:%s", refname, revision) - // Skip if querying for a version and the version does not match - if len(qversion) > 0 && revision != qversion { - log.G(ctx). - WithField("ref", manifest.Config.Digest.String()). - Trace("skipping non-unikernel digest") - continue - } + // Skip direct references from the remote registry + if !query.UseCache() && refErr == nil && ref.String() == fullref { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + Trace("skipping non-unikernel digest") + continue + } + + // Skip if querying for a version and the version does not match + if len(qversion) > 0 && revision != qversion { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + Trace("skipping non-unikernel digest") + continue + } + + // Skip if querying for a platform and the platform does not match + if platform != nil && platform.Architecture != manifest.Config.Platform.Architecture { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + WithField("arch", manifest.Config.Platform.Architecture). + Trace("skipping not matching digest") + continue + } + + if platform != nil && platform.OS != manifest.Config.Platform.OS { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + WithField("plat", manifest.Config.Platform.OS). + Trace("skipping not matching digest") + continue + } + + if platform != nil && platform.Variant != manifest.Config.Platform.Variant { + log.G(ctx). + WithField("ref", manifest.Config.Digest.String()). + WithField("variant", manifest.Config.Platform.Variant). + Trace("skipping not matching digest") + continue + } - log.G(ctx).WithField("ref", fullref).Debug("found") + log.G(ctx).WithField("ref", fullref).Debug("found") - pack, err := NewPackageFromOCIManifestSpec( - ctx, - handle, - fullref, - manifest, - ) + pack, err := NewPackageFromOCIManifestSpec( + ctx, + handle, + fullref, + manifest, + ) + if err != nil { + // log.G(ctx).Warn(err) + continue + } + + packs = append(packs, pack) + } + } else { + // Access local images that are available on the host + indexes, err := handle.ListIndexes(ctx) if err != nil { - // log.G(ctx).Warn(err) - continue + return nil, err } - packs = append(packs, pack) + for _, index := range indexes { + // Check if the OCI image has a known annotation which identifies if a + // unikernel is contained within + if _, ok := index.Annotations[AnnotationKernelVersion]; !ok { + log.G(ctx). + Trace("skipping non-unikernel digest") + continue + } + + // Could not determine name from manifest specification + refname, ok := index.Annotations[ocispec.AnnotationRefName] + if !ok { + log.G(ctx). + Trace("skipping non-unikernel digest") + continue + } + + // Skip if querying for the name and the name does not match + if len(qname) > 0 && refname != qname { + log.G(ctx). + Trace("skipping non-unikernel digest") + continue + } + + // Could not determine name from manifest specification + revision, ok := index.Annotations[ocispec.AnnotationRevision] + if !ok { + log.G(ctx). + Trace("skipping non-unikernel digest") + continue + } + + fullref := fmt.Sprintf("%s:%s", refname, revision) + + // Skip direct references from the remote registry + if !query.UseCache() && refErr == nil && ref.String() == fullref { + log.G(ctx). + Trace("skipping non-unikernel digest") + continue + } + + // Skip if querying for a version and the version does not match + if len(qversion) > 0 && revision != qversion { + log.G(ctx). + Trace("skipping non-unikernel digest") + continue + } + + log.G(ctx).WithField("ref", fullref).Debug("found") + + pack, err := NewPackageFromOCIIndexSpec( + ctx, + handle, + fullref, + index, + ) + if err != nil { + // log.G(ctx).Warn(err) + continue + } + + packs = append(packs, pack) + } } return packs, nil @@ -334,6 +445,9 @@ func (manager *ociManager) RemoveSource(ctx context.Context, source string) erro // IsCompatible implements packmanager.PackageManager func (manager *ociManager) IsCompatible(ctx context.Context, source string, qopts ...packmanager.QueryOption) (packmanager.PackageManager, bool, error) { + query := packmanager.NewQuery(qopts...) + plat := fmt.Sprintf("%s/%s", query.Platform(), query.Architecture()) + ctx, handle, err := manager.handle(ctx) if err != nil { return nil, false, err @@ -344,7 +458,7 @@ func (manager *ociManager) IsCompatible(ctx context.Context, source string, qopt log.G(ctx).WithField("source", source).Debug("checking if source is OCI image") // First try without known registries - _, err = handle.ResolveImage(ctx, source) + _, err = handle.ResolveImage(ctx, source, plat) if err == nil { return true } @@ -358,7 +472,7 @@ func (manager *ociManager) IsCompatible(ctx context.Context, source string, qopt continue } - _, err = handle.ResolveImage(ctx, ref.Context().String()) + _, err = handle.ResolveImage(ctx, ref.Context().String(), plat) if err == nil { return true } From 2071c1cbdc32a8f71101cc7881a193bdf342bcd1 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:29:16 +0300 Subject: [PATCH 12/16] feat(handler)!: Implement image index interface and adapt existing ones Signed-off-by: Cezar Craciunoiu --- oci/handler/directory_image.go | 101 +++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/oci/handler/directory_image.go b/oci/handler/directory_image.go index feb596171..03baff6bf 100644 --- a/oci/handler/directory_image.go +++ b/oci/handler/directory_image.go @@ -6,6 +6,7 @@ package handler import ( "bytes" + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -22,8 +23,6 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -const algorithm = "sha256" - type DirectoryLayer struct { path string diffID digest.Digest @@ -155,7 +154,7 @@ func (di DirectoryImage) RawConfigFile() ([]byte, error) { configPath := filepath.Join( di.handle.path, DirectoryHandlerConfigsDir, - algorithm, + string(di.manifestDescriptor.Digest.Algorithm()), hex.EncodeToString(h.Sum(nil)), ) @@ -191,17 +190,11 @@ func (di DirectoryImage) Manifest() (*v1.Manifest, error) { // RawManifest returns the manifest of the image in bytes // It reads the manifest from the filesystem func (di DirectoryImage) RawManifest() ([]byte, error) { - var jsonPath string - if strings.ContainsRune(di.ref.Name(), '@') { - jsonPath = strings.ReplaceAll(di.ref.Name(), "@", string(filepath.Separator)) + ".json" - } else { - jsonPath = strings.ReplaceAll(di.ref.Name(), ":", string(filepath.Separator)) + ".json" - } - manifestPath := filepath.Join( di.handle.path, DirectoryHandlerManifestsDir, - jsonPath, + string(di.manifestDescriptor.Digest.Algorithm()), + di.manifestDescriptor.Digest.Encoded(), ) return os.ReadFile(manifestPath) @@ -259,3 +252,89 @@ func (di DirectoryImage) LayerByDiffID(hash v1.Hash) (v1.Layer, error) { } return nil, fmt.Errorf("layer not found") } + +type DirectoryImageIndex struct { + handle *DirectoryHandler + index ocispec.Index + indexDescriptor *ocispec.Descriptor + ref name.Reference +} + +// MediaType of this image's manifest. +func (dix DirectoryImageIndex) MediaType() (types.MediaType, error) { + return types.MediaType(dix.index.MediaType), nil +} + +// Digest returns the sha256 of this index's manifest. +func (dix DirectoryImageIndex) Digest() (v1.Hash, error) { + b, err := dix.RawManifest() + if err != nil { + return v1.Hash{}, err + } + + h, _, err := v1.SHA256(bytes.NewReader(b)) + return h, err +} + +// Size returns the size of the manifest. +func (dix DirectoryImageIndex) Size() (int64, error) { + return dix.indexDescriptor.Size, nil +} + +// IndexManifest returns this image index's manifest object. +func (dix DirectoryImageIndex) IndexManifest() (*v1.IndexManifest, error) { + b, err := dix.RawManifest() + if err != nil { + return nil, err + } + + return v1.ParseIndexManifest(bytes.NewReader(b)) +} + +// RawManifest returns the serialized bytes of IndexManifest(). +func (dix DirectoryImageIndex) RawManifest() ([]byte, error) { + var jsonPath string + if strings.ContainsRune(dix.ref.Name(), '@') { + jsonPath = strings.ReplaceAll(dix.ref.Name(), "@", string(filepath.Separator)) + ".json" + } else { + jsonPath = strings.ReplaceAll(dix.ref.Name(), ":", string(filepath.Separator)) + ".json" + } + + indexPath := filepath.Join( + dix.handle.path, + DirectoryHandlerIndexesDir, + jsonPath, + ) + + return os.ReadFile(indexPath) +} + +// Image returns a v1.Image that this ImageIndex references. +func (dix DirectoryImageIndex) Image(hash v1.Hash) (v1.Image, error) { + var manifest *ocispec.Descriptor + + for _, desc := range dix.index.Manifests { + if desc.Digest.Encoded() == hash.Hex { + desc := desc + manifest = &desc + } + } + + // Fetch manifest and config from the filesystem + image, err := dix.handle.ResolveImage(context.TODO(), manifest.Digest.Encoded()) + if err != nil { + return nil, err + } + + return DirectoryImage{ + image: image, + manifestDescriptor: manifest, + handle: dix.handle, + ref: dix.ref, + }, nil +} + +// ImageIndex returns a v1.ImageIndex that this ImageIndex references. +func (dix DirectoryImageIndex) ImageIndex(hash v1.Hash) (v1.ImageIndex, error) { + return nil, fmt.Errorf("mixed image indexes are not supported, contributions welcome") +} From 06e214a17de7bb4f48d80c93383da8751755cb8e Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:28:09 +0300 Subject: [PATCH 13/16] feat(handler)!: Implement index structuring for the directory handler Signed-off-by: Cezar Craciunoiu --- oci/handler/directory.go | 384 +++++++++++++++++++++++---------- oci/handler/directory_image.go | 2 +- oci/handler/handler.go | 2 +- 3 files changed, 267 insertions(+), 121 deletions(-) diff --git a/oci/handler/directory.go b/oci/handler/directory.go index e13d59363..aab37fe98 100644 --- a/oci/handler/directory.go +++ b/oci/handler/directory.go @@ -16,6 +16,7 @@ import ( "strings" "kraftkit.sh/internal/version" + "kraftkit.sh/log" "kraftkit.sh/oci/simpleauth" regtypes "github.com/docker/docker/api/types/registry" @@ -145,11 +146,6 @@ func (handle *DirectoryHandler) ListManifests(ctx context.Context) (manifests [] return nil } - // Skip files that don't end in .json - if !strings.HasSuffix(d.Name(), ".json") { - return nil - } - // Read the manifest rawManifest, err := os.ReadFile(path) if err != nil { @@ -205,6 +201,13 @@ func (handle *DirectoryHandler) SaveDigest(ctx context.Context, ref string, desc blobPath = filepath.Join( blobPath, DirectoryHandlerManifestsDir, + desc.Digest.Algorithm().String(), + desc.Digest.Encoded(), + ) + case ocispec.MediaTypeImageIndex: + blobPath = filepath.Join( + blobPath, + DirectoryHandlerIndexesDir, strings.ReplaceAll(ref, ":", string(filepath.Separator))+".json", ) case ocispec.MediaTypeImageLayer: @@ -245,12 +248,10 @@ func (handle *DirectoryHandler) SaveDigest(ctx context.Context, ref string, desc return nil } -// ResolveImage implements ImageResolver. -func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref string) (imgspec ocispec.Image, err error) { - // Find the manifest of this image +func (handle *DirectoryHandler) resolveIndex(_ context.Context, fullref string) (ocispec.Index, error) { ref, err := name.ParseReference(fullref) if err != nil { - return ocispec.Image{}, err + return ocispec.Index{}, err } var jsonPath string @@ -260,15 +261,53 @@ func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref string jsonPath = strings.ReplaceAll(ref.Name(), ":", string(filepath.Separator)) + ".json" } + // Fetch the index + indexPath := filepath.Join( + handle.path, + DirectoryHandlerIndexesDir, + jsonPath, + ) + + // Check whether the index exists + if _, err := os.Stat(indexPath); err != nil { + return ocispec.Index{}, fmt.Errorf("index for %s does not exist: %s", ref.Name(), indexPath) + } + + // Read the index + reader, err := os.Open(indexPath) + if err != nil { + return ocispec.Index{}, err + } + defer reader.Close() + + indexRaw, err := io.ReadAll(reader) + if err != nil { + return ocispec.Index{}, err + } + + // Unmarshal the index + index := ocispec.Index{} + if err = json.Unmarshal(indexRaw, &index); err != nil { + return ocispec.Index{}, err + } + + return index, nil +} + +// ResolveImage fetches the image config from a given manifest reference. +// the reference is the manifest sha256 digest. +// ResolveImage implements ImageResolver. +func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref, platform string) (imgspec ocispec.Image, err error) { manifestPath := filepath.Join( handle.path, DirectoryHandlerManifestsDir, - jsonPath, + "sha256", + fullref, ) // Check whether the manifest exists if _, err := os.Stat(manifestPath); err != nil { - return ocispec.Image{}, fmt.Errorf("manifest for %s does not exist: %s", ref.Name(), manifestPath) + return ocispec.Image{}, fmt.Errorf("manifest for %s does not exist: %s", fullref, manifestPath) } // Read the manifest @@ -276,6 +315,7 @@ func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref string if err != nil { return ocispec.Image{}, err } + defer reader.Close() manifestRaw, err := io.ReadAll(reader) if err != nil { @@ -304,7 +344,7 @@ func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref string // Check whether the config exists if _, err := os.Stat(configDir); err != nil { - return ocispec.Image{}, fmt.Errorf("could not access config file for %s: %w", ref.Name(), err) + return ocispec.Image{}, fmt.Errorf("could not access config file for %s: %w", fullref, err) } // Read the config @@ -330,6 +370,9 @@ func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref string // FetchImage implements ImageFetcher. func (handle *DirectoryHandler) FetchImage(ctx context.Context, fullref, platform string, onProgress func(float64)) (err error) { + plat := strings.Split(platform, "/")[0] + arch := strings.Split(platform, "/")[1] + ref, err := name.ParseReference(fullref) if err != nil { return err @@ -346,11 +389,11 @@ func (handle *DirectoryHandler) FetchImage(ctx context.Context, fullref, platfor authConfig.Username = auth.Username } - img, err := remote.Image(ref, + idx, err := remote.Index(ref, remote.WithContext(ctx), remote.WithPlatform(v1.Platform{ - OS: strings.Split(platform, "/")[0], - Architecture: strings.Split(platform, "/")[1], + OS: plat, + Architecture: arch, }), remote.WithUserAgent(version.UserAgent()), remote.WithAuth(&simpleauth.SimpleAuthenticator{ @@ -361,8 +404,13 @@ func (handle *DirectoryHandler) FetchImage(ctx context.Context, fullref, platfor return err } - // Write the manifest - manifest, err := img.RawManifest() + // Write the index manifest + manifest, err := idx.RawManifest() + if err != nil { + return err + } + + digest, err := idx.Digest() if err != nil { return err } @@ -376,7 +424,7 @@ func (handle *DirectoryHandler) FetchImage(ctx context.Context, fullref, platfor manifestPath := filepath.Join( handle.path, - DirectoryHandlerManifestsDir, + DirectoryHandlerIndexesDir, jsonPath, ) @@ -396,88 +444,154 @@ func (handle *DirectoryHandler) FetchImage(ctx context.Context, fullref, platfor return err } - config, err := img.RawConfigFile() - if err != nil { - return err - } - - configName, err := img.ConfigName() + parsableManifest, err := idx.IndexManifest() if err != nil { return err } - configPath := filepath.Join( - handle.path, - DirectoryHandlerConfigsDir, - configName.Algorithm, - configName.Hex, - ) - - // If the config already exists, skip it - if _, err := os.Stat(configPath); err == nil { - return nil - } - - // Recursively create the directory - if err = os.MkdirAll(configPath[:strings.LastIndex(configPath, "/")], 0o775); err != nil { - return err + var manifests []v1.Descriptor + for _, descriptor := range parsableManifest.Manifests { + if descriptor.Platform.OS == plat && descriptor.Platform.Architecture == arch { + log.G(ctx). + WithField("platform", fmt.Sprintf("%s/%s", descriptor.Platform.OS, descriptor.Platform.Architecture)). + WithField("digest", descriptor.Digest). + Trace("fetching") + manifests = append(manifests, descriptor) + } else { + log.G(ctx). + WithField("platform", fmt.Sprintf("%s/%s", descriptor.Platform.OS, descriptor.Platform.Architecture)). + WithField("digest", descriptor.Digest). + Trace("skip fetching") + } } - writer, err = os.Create(configPath) - if err != nil { - return err + if len(manifests) == 0 { + return fmt.Errorf("no manifest found for %s/%s", arch, plat) } - defer writer.Close() - // Write the config - if _, err = writer.Write(config); err != nil { - return err - } + for _, descriptor := range manifests { + img, err := idx.Image(descriptor.Digest) + if err != nil { + return err + } - // Write the layers - layers, err := img.Layers() - if err != nil { - return err - } + // Write the manifest + manifest, err = img.RawManifest() + if err != nil { + return err + } - for _, layer := range layers { - digest, err := layer.Digest() + digest, err = img.Digest() if err != nil { return err } - layerPath := filepath.Join( + manifestPath = filepath.Join( handle.path, - DirectoryHandlerLayersDir, + DirectoryHandlerManifestsDir, digest.Algorithm, digest.Hex, ) // Recursively create the directory - if err = os.MkdirAll(layerPath[:strings.LastIndex(layerPath, "/")], 0o775); err != nil { + if err = os.MkdirAll(manifestPath[:strings.LastIndex(manifestPath, "/")], 0o775); err != nil { return err } - // If the layer already exists, skip it - if _, err := os.Stat(layerPath); err == nil { - continue + // Open a writer to the specified path + writer, err := os.Create(manifestPath) + if err != nil { + return err } + defer writer.Close() - writer, err = os.Create(layerPath) + if _, err := writer.Write(manifest); err != nil { + return err + } + + config, err := img.RawConfigFile() if err != nil { return err } - defer writer.Close() - reader, err := layer.Compressed() + configName, err := img.ConfigName() + if err != nil { + return err + } + + configPath := filepath.Join( + handle.path, + DirectoryHandlerConfigsDir, + configName.Algorithm, + configName.Hex, + ) + + // If the config already exists, skip it + if _, err := os.Stat(configPath); err == nil { + return nil + } + + // Recursively create the directory + if err = os.MkdirAll(configPath[:strings.LastIndex(configPath, "/")], 0o775); err != nil { + return err + } + + writer, err = os.Create(configPath) if err != nil { return err } - defer reader.Close() + defer writer.Close() + + // Write the config + if _, err = writer.Write(config); err != nil { + return err + } - if _, err = io.Copy(writer, reader); err != nil { + // Write the layers + layers, err := img.Layers() + if err != nil { return err } + + for _, layer := range layers { + digest, err := layer.Digest() + if err != nil { + return err + } + + layerPath := filepath.Join( + handle.path, + DirectoryHandlerLayersDir, + digest.Algorithm, + digest.Hex, + ) + + // Recursively create the directory + if err = os.MkdirAll(layerPath[:strings.LastIndex(layerPath, "/")], 0o775); err != nil { + return err + } + + // If the layer already exists, skip it + if _, err := os.Stat(layerPath); err == nil { + continue + } + + writer, err = os.Create(layerPath) + if err != nil { + return err + } + defer writer.Close() + + reader, err := layer.Compressed() + if err != nil { + return err + } + defer reader.Close() + + if _, err = io.Copy(writer, reader); err != nil { + return err + } + } } return nil @@ -490,11 +604,6 @@ func (handle *DirectoryHandler) PushImage(ctx context.Context, fullref string, t return err } - image, err := handle.ResolveImage(ctx, fullref) - if err != nil { - return err - } - authConfig := &authn.AuthConfig{} // Annoyingly convert between regtypes and authn. @@ -506,12 +615,17 @@ func (handle *DirectoryHandler) PushImage(ctx context.Context, fullref string, t authConfig.Username = auth.Username } - return remote.Write(ref, - DirectoryImage{ - image: image, - manifestDescriptor: target, - handle: handle, - ref: ref, + index, err := handle.resolveIndex(ctx, fullref) + if err != nil { + return err + } + + return remote.WriteIndex(ref, + DirectoryImageIndex{ + index: index, + indexDescriptor: target, + handle: handle, + ref: ref, }, remote.WithContext(ctx), remote.WithUserAgent(version.UserAgent()), @@ -522,72 +636,104 @@ func (handle *DirectoryHandler) PushImage(ctx context.Context, fullref string, t } // UnpackImage implements ImageUnpacker. -func (handle *DirectoryHandler) UnpackImage(ctx context.Context, ref string, dest string) (err error) { - img, err := handle.ResolveImage(ctx, ref) +func (handle *DirectoryHandler) UnpackImage(ctx context.Context, ref string, platform string, dest string) (err error) { + plat := strings.Split(platform, "/")[0] + arch := strings.Split(platform, "/")[1] + + idx, err := handle.resolveIndex(ctx, ref) if err != nil { return err } - // Iterate over the layers - for _, layer := range img.RootFS.DiffIDs { - // Get the digest - digest, err := v1.NewHash(layer.String()) - if err != nil { - return err + var manifests []digest.Digest + for _, descriptor := range idx.Manifests { + if descriptor.Platform.Architecture == arch && descriptor.Platform.OS == plat { + log.G(ctx).Tracef("Pick for unpacking %s/%s manifest %s", + descriptor.Platform.Architecture, + descriptor.Platform.OS, + descriptor.Digest) + manifests = append(manifests, descriptor.Digest) + } else { + log.G(ctx).Tracef("Skip unpacking %s/%s manifest %s", + descriptor.Platform.Architecture, + descriptor.Platform.OS, + descriptor.Digest) } + } - // Get the layer path - layerPath := filepath.Join( - handle.path, - DirectoryHandlerLayersDir, - digest.Algorithm, - digest.Hex, - ) + if len(manifests) == 0 { + return fmt.Errorf("no manifest found for platform %s", platform) + } else if len(manifests) > 1 { + return fmt.Errorf("multiple manifests found for platform %s, unpacking all would overwrite results", platform) + } - // Layer path is a tarball, so we need to extract it - reader, err := os.Open(layerPath) + for _, manifest := range manifests { + img, err := handle.ResolveImage(ctx, manifest.Encoded(), platform) if err != nil { return err } - defer reader.Close() + // Iterate over the layers + for _, layer := range img.RootFS.DiffIDs { + // Get the digest + digest, err := v1.NewHash(layer.String()) + if err != nil { + return err + } - tr := tar.NewReader(reader) + // Get the layer path + layerPath := filepath.Join( + handle.path, + DirectoryHandlerLayersDir, + digest.Algorithm, + digest.Hex, + ) - for { - hdr, err := tr.Next() + // Layer path is a tarball, so we need to extract it + reader, err := os.Open(layerPath) if err != nil { - break + return err } - // Write the file to the destination - path := filepath.Join(dest, hdr.Name) + defer reader.Close() - // If the file is a directory, create it - if hdr.Typeflag == tar.TypeDir { - if err := os.MkdirAll(path, 0o775); err != nil { - return err + tr := tar.NewReader(reader) + + for { + hdr, err := tr.Next() + if err != nil { + break } - continue - } - // If the directory in the path doesn't exist, create it - if _, err := os.Stat(path[:strings.LastIndex(path, "/")]); os.IsNotExist(err) { - if err := os.MkdirAll(path[:strings.LastIndex(path, "/")], 0o775); err != nil { - return err + // Write the file to the destination + path := filepath.Join(dest, hdr.Name) + + // If the file is a directory, create it + if hdr.Typeflag == tar.TypeDir { + if err := os.MkdirAll(path, 0o775); err != nil { + return err + } + continue } - } - // Otherwise, create the file - writer, err := os.Create(path) - if err != nil { - return err - } + // If the directory in the path doesn't exist, create it + if _, err := os.Stat(path[:strings.LastIndex(path, "/")]); os.IsNotExist(err) { + if err := os.MkdirAll(path[:strings.LastIndex(path, "/")], 0o775); err != nil { + return err + } + } - defer writer.Close() + // Otherwise, create the file + writer, err := os.Create(path) + if err != nil { + return err + } - if _, err = io.Copy(writer, tr); err != nil { - return err + defer writer.Close() + + if _, err = io.Copy(writer, tr); err != nil { + return err + } } } } diff --git a/oci/handler/directory_image.go b/oci/handler/directory_image.go index 03baff6bf..b8285b9c9 100644 --- a/oci/handler/directory_image.go +++ b/oci/handler/directory_image.go @@ -321,7 +321,7 @@ func (dix DirectoryImageIndex) Image(hash v1.Hash) (v1.Image, error) { } // Fetch manifest and config from the filesystem - image, err := dix.handle.ResolveImage(context.TODO(), manifest.Digest.Encoded()) + image, err := dix.handle.ResolveImage(context.TODO(), manifest.Digest.Encoded(), "") if err != nil { return nil, err } diff --git a/oci/handler/handler.go b/oci/handler/handler.go index 137c9dd8c..08b9ab10c 100644 --- a/oci/handler/handler.go +++ b/oci/handler/handler.go @@ -37,7 +37,7 @@ type ImagePusher interface { } type ImageResolver interface { - ResolveImage(context.Context, string) (ocispec.Image, error) + ResolveImage(context.Context, string, string) (ocispec.Image, error) } type ImageFetcher interface { From 8105b3746caca44167861548ad8553566aec91a6 Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Wed, 9 Aug 2023 17:26:16 +0300 Subject: [PATCH 14/16] feat(handler): Add partially-working implementation for containerd Currently pulling works. Packaging is not working and pushing can only push manifests. Signed-off-by: Cezar Craciunoiu --- oci/handler/containerd.go | 268 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 255 insertions(+), 13 deletions(-) diff --git a/oci/handler/containerd.go b/oci/handler/containerd.go index 2b2fe93eb..121773f2e 100644 --- a/oci/handler/containerd.go +++ b/oci/handler/containerd.go @@ -108,6 +108,67 @@ func (handle *ContainerdHandler) DigestExists(ctx context.Context, dgst digest.D return true, nil } +// ListIndexes implements DigestResolver. +func (handle *ContainerdHandler) ListIndexes(ctx context.Context) (indx []ocispec.Index, err error) { + ctx, done, err := handle.lease(ctx) + if err != nil { + return nil, err + } + + defer func() { + err = combineErrors(err, done(ctx)) + }() + + all, err := handle.client.ImageService().List( + namespaces.WithNamespace(ctx, handle.namespace), + ) + if err != nil { + return nil, err + } + + for _, image := range all { + found, err := handle.client.GetImage( + namespaces.WithNamespace(ctx, handle.namespace), + image.Name, + ) + if err != nil { + return nil, err + } + + if found.Target().MediaType != ocispec.MediaTypeImageIndex { + continue + } + + manifest, err := images.Manifest( + namespaces.WithNamespace(ctx, handle.namespace), + handle.client.ContentStore(), found.Target(), nil, + ) + if err != nil { + return nil, err + } + manifests, err := images.Children( + namespaces.WithNamespace(ctx, handle.namespace), + handle.client.ContentStore(), + found.Target(), + ) + if err != nil { + return nil, err + } + + index := ocispec.Index{ + MediaType: manifest.MediaType, + ArtifactType: manifest.ArtifactType, + Manifests: manifests, + Subject: manifest.Subject, + Annotations: manifest.Annotations, + } + + indx = append(indx, index) + } + + return indx, nil +} + // ListManifests implements DigestResolver. func (handle *ContainerdHandler) ListManifests(ctx context.Context) (manifests []ocispec.Manifest, err error) { ctx, done, err := handle.lease(ctx) @@ -141,8 +202,11 @@ func (handle *ContainerdHandler) ListManifests(ctx context.Context) (manifests [ return manifests, nil } +// TODO(craciunoiuc): Saving only saves the index and not the manifests themselves // SaveDigest implements DigestSaver. func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, desc ocispec.Descriptor, reader io.Reader, onProgress func(float64)) (err error) { + log.G(ctx).Errorf("oci: Packaging not supported for containerd with index manifests. Saving will fail.") + ctx, done, err := handle.lease(ctx) if err != nil { return err @@ -169,7 +233,8 @@ func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, des var tee io.Reader var cache bytes.Buffer - if desc.MediaType == ocispec.MediaTypeImageManifest { + if desc.MediaType == ocispec.MediaTypeImageManifest || + desc.MediaType == ocispec.MediaTypeImageIndex { tee = io.TeeReader(reader, &cache) } else { tee = reader @@ -185,9 +250,9 @@ func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, des switch desc.MediaType { // case ociimages.MediaTypeDockerSchema2Manifest, // ocispec.MediaTypeImageManifest, - // ociimages.MediaTypeDockerSchema2ManifestList, - // ocispec.MediaTypeImageIndex: - case ocispec.MediaTypeImageManifest: + // ociimages.MediaTypeDockerSchema2ManifestList: + case ocispec.MediaTypeImageIndex, + ocispec.MediaTypeImageManifest: // ref, ok := desc.Annotations[ociimages.AnnotationImageName] // if !ok { // return fmt.Errorf("cannot push image layer without image annotation") @@ -211,7 +276,7 @@ func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, des labels[fmt.Sprintf("%s.%d", ContainerdGCLayerPrefix, len(manifest.Layers))] = manifest.Config.Digest.String() var image images.Image - existingImage, err := is.Get(ctx, ref) + existingImage, err := is.Get(namespaces.WithNamespace(ctx, handle.namespace), ref) if err != nil || existingImage.Target.Digest.String() == "" { log.G(ctx).Trace("oci: creating new image") @@ -222,7 +287,7 @@ func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, des CreatedAt: time.Now(), UpdatedAt: time.Time{}, } - _, err = is.Create(ctx, image) + _, err = is.Create(namespaces.WithNamespace(ctx, handle.namespace), image) } else { log.G(ctx).Trace("oci: updating existing image") image = images.Image{ @@ -231,7 +296,7 @@ func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, des Target: desc, UpdatedAt: time.Time{}, } - _, err = is.Update(ctx, image) + _, err = is.Update(namespaces.WithNamespace(ctx, handle.namespace), image) } if err != nil { return err @@ -260,7 +325,7 @@ func (handle *ContainerdHandler) SaveDigest(ctx context.Context, ref string, des } // ResolveImage implements ImageResolver. -func (handle *ContainerdHandler) ResolveImage(ctx context.Context, fullref string) (imgspec ocispec.Image, err error) { +func (handle *ContainerdHandler) ResolveImage(ctx context.Context, fullref, platform string) (imgspec ocispec.Image, err error) { ctx, done, err := handle.lease(ctx) if err != nil { return ocispec.Image{}, err @@ -270,12 +335,56 @@ func (handle *ContainerdHandler) ResolveImage(ctx context.Context, fullref strin err = combineErrors(err, done(ctx)) }() - image, err := handle.client.GetImage(ctx, fullref) + image, err := handle.client.GetImage( + namespaces.WithNamespace(ctx, handle.namespace), + fullref, + ) if err != nil { return ocispec.Image{}, err } - return image.Spec(ctx) + if image.Target().MediaType == ocispec.MediaTypeImageIndex { + manifests, err := images.Children( + namespaces.WithNamespace(ctx, handle.namespace), + handle.client.ContentStore(), + image.Target(), + ) + if err != nil { + return ocispec.Image{}, err + } + + // Split on ':' + parsed := strings.SplitN(fullref, ":", 2) + var base string + if len(parsed) == 2 { + base = parsed[0] + } else if len(parsed) == 1 { + base = fullref + } else { + return ocispec.Image{}, fmt.Errorf("invalid image reference: %s", fullref) + } + + for _, manifest := range manifests { + if manifest.MediaType == ocispec.MediaTypeImageManifest && + (fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture) == platform || + len(platform) == 0 && len(manifests) == 1) { + + image, err := handle.client.GetImage( + namespaces.WithNamespace(ctx, handle.namespace), + base+":"+manifest.Digest.String(), + ) + if err != nil { + return ocispec.Image{}, err + } + + return image.Spec(ctx) + } + } + + return ocispec.Image{}, fmt.Errorf("no matching platform found") + } else { + return image.Spec(ctx) + } } // FetchImage implements ImageFetcher. @@ -337,6 +446,50 @@ func (handle *ContainerdHandler) FetchImage(ctx context.Context, ref, plat strin <-progress + image, err := handle.client.GetImage( + namespaces.WithNamespace(ctx, handle.namespace), + ref, + ) + if err != nil { + return err + } + manifests, err := images.Children( + namespaces.WithNamespace(ctx, handle.namespace), + handle.client.ContentStore(), + image.Target(), + ) + if err != nil { + return err + } + + // Split on ':' + parsed := strings.SplitN(ref, ":", 2) + var base string + if len(parsed) == 2 { + base = parsed[0] + } else if len(parsed) == 1 { + base = ref + } else { + return fmt.Errorf("invalid image reference: %s", ref) + } + + for _, manifest := range manifests { + if manifest.MediaType == ocispec.MediaTypeImageManifest && + (fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture) == plat || + len(plat) == 0 && len(manifests) == 1) { + + // Fetch the image + _, err = handle.client.Fetch( + namespaces.WithNamespace(ctx, handle.namespace), + base+":"+manifest.Digest.String(), + ropts..., + ) + if err != nil { + return err + } + } + } + return nil } @@ -497,6 +650,22 @@ outer: // PushImage implements ImagePusher. func (handle *ContainerdHandler) PushImage(ctx context.Context, ref string, target *ocispec.Descriptor) error { + img, err := handle.client.ImageService().Get( + namespaces.WithNamespace(ctx, handle.namespace), + ref, + ) + if err != nil { + return err + } + manifests, err := images.Children( + namespaces.WithNamespace(ctx, handle.namespace), + handle.client.ContentStore(), + img.Target, + ) + if err != nil { + return err + } + resolver, err := dockerconfigresolver.New( ctx, strings.Split(ref, "/")[0], @@ -514,6 +683,36 @@ func (handle *ContainerdHandler) PushImage(ctx context.Context, ref string, targ return err } + // Split on ':' + parsed := strings.SplitN(ref, ":", 2) + var base string + if len(parsed) == 2 { + base = parsed[0] + } else if len(parsed) == 1 { + base = ref + } else { + return fmt.Errorf("invalid image reference: %s", ref) + } + + for _, manifest := range manifests { + image, err := handle.client.GetImage( + namespaces.WithNamespace(ctx, handle.namespace), + base+":"+manifest.Digest.String(), + ) + if err != nil { + return err + } + + if err := handle.client.Push( + namespaces.WithNamespace(ctx, handle.namespace), + base+":"+manifest.Digest.String(), + image.Target(), + containerd.WithResolver(resolver), + ); err != nil { + return err + } + } + return handle.client.Push( namespaces.WithNamespace(ctx, handle.namespace), ref, @@ -523,7 +722,7 @@ func (handle *ContainerdHandler) PushImage(ctx context.Context, ref string, targ } // UnpackImage implements ImageUnpacker. -func (handle *ContainerdHandler) UnpackImage(ctx context.Context, ref string, dest string) (err error) { +func (handle *ContainerdHandler) UnpackImage(ctx context.Context, ref string, platform string, dest string) (err error) { ctx, done, err := handle.lease(ctx) if err != nil { return err @@ -533,12 +732,55 @@ func (handle *ContainerdHandler) UnpackImage(ctx context.Context, ref string, de err = combineErrors(err, done(ctx)) }() - img, err := handle.client.ImageService().Get(ctx, ref) + img, err := handle.client.ImageService().Get( + namespaces.WithNamespace(ctx, handle.namespace), + ref, + ) + if err != nil { + return err + } + manifests, err := images.Children( + namespaces.WithNamespace(ctx, handle.namespace), + handle.client.ContentStore(), + img.Target, + ) if err != nil { return err } - i := containerd.NewImage(handle.client, img) + // Split on ':' + parsed := strings.SplitN(ref, ":", 2) + var base string + if len(parsed) == 2 { + base = parsed[0] + } else if len(parsed) == 1 { + base = ref + } else { + return fmt.Errorf("invalid image reference: %s", ref) + } + + var i containerd.Image + for _, manifest := range manifests { + if manifest.MediaType == ocispec.MediaTypeImageManifest && + (fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture) == platform || + len(platform) == 0 && len(manifests) == 1) { + + imgBase, err := handle.client.ImageService().Get( + namespaces.WithNamespace(ctx, handle.namespace), + base+":"+manifest.Digest.String(), + ) + if err != nil { + return err + } + i = containerd.NewImage(handle.client, imgBase) + + break + } + } + + if i == nil { + return fmt.Errorf("no matching platform found") + } // TODO: We need to pass the architecture, platform and any desired KConfig // values via the platform specifier: From 08f1f477f075c7ee472e12710e74bc4b7bacfc0a Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Fri, 11 Aug 2023 13:38:30 +0300 Subject: [PATCH 15/16] fix(oci): Trim OCI prefix when using output argument Signed-off-by: Cezar Craciunoiu --- oci/pack.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/oci/pack.go b/oci/pack.go index bd6cb43a9..963b0a391 100644 --- a/oci/pack.go +++ b/oci/pack.go @@ -84,7 +84,13 @@ func NewPackageFromTarget(ctx context.Context, targ target.Target, opts ...packm } if popts.Output() != "" { - ocipack.ref, err = name.ParseReference(popts.Output(), + output := popts.Output() + if strings.HasPrefix(popts.Output(), "oci://") { + output = strings.TrimPrefix(popts.Output(), "oci://") + } + + ocipack.ref, err = name.ParseReference( + output, name.WithDefaultRegistry(DefaultRegistry), ) } else { @@ -311,7 +317,13 @@ func NewPackageFromTargets(ctx context.Context, targ []target.Target, opts ...pa } if popts.Output() != "" { - ref, err = name.ParseReference(popts.Output(), + output := popts.Output() + if strings.HasPrefix(popts.Output(), "oci://") { + output = strings.TrimPrefix(popts.Output(), "oci://") + } + + ref, err = name.ParseReference( + output, name.WithDefaultRegistry(DefaultRegistry), ) } else if len(targ) == 1 { From 73570cc15786ae1d12f80101cd3fd0360a70800e Mon Sep 17 00:00:00 2001 From: Cezar Craciunoiu Date: Fri, 8 Sep 2023 19:00:57 +0300 Subject: [PATCH 16/16] =?UTF-8?q?REMOVE-ME:=20=C2=B0=C2=BA=C2=A4=C3=B8,?= =?UTF-8?q?=C2=B8=C2=B8,=C3=B8=C2=A4=C2=BA=C2=B0=C2=B0=C2=BA=C2=A4=C3=B8,?= =?UTF-8?q?=C2=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cezar Craciunoiu --- oci/handler/directory.go | 60 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/oci/handler/directory.go b/oci/handler/directory.go index aab37fe98..c43ae33b7 100644 --- a/oci/handler/directory.go +++ b/oci/handler/directory.go @@ -249,6 +249,11 @@ func (handle *DirectoryHandler) SaveDigest(ctx context.Context, ref string, desc } func (handle *DirectoryHandler) resolveIndex(_ context.Context, fullref string) (ocispec.Index, error) { + // Check whether the reference is a digest + if !strings.ContainsRune(fullref, '/') { + return ocispec.Index{}, fmt.Errorf("invalid reference: %s", fullref) + } + ref, err := name.ParseReference(fullref) if err != nil { return ocispec.Index{}, err @@ -298,12 +303,55 @@ func (handle *DirectoryHandler) resolveIndex(_ context.Context, fullref string) // the reference is the manifest sha256 digest. // ResolveImage implements ImageResolver. func (handle *DirectoryHandler) ResolveImage(ctx context.Context, fullref, platform string) (imgspec ocispec.Image, err error) { - manifestPath := filepath.Join( - handle.path, - DirectoryHandlerManifestsDir, - "sha256", - fullref, - ) + var manifestPath string + + idx, err := handle.resolveIndex(ctx, fullref) + if err == nil { + var manifestDesc ocispec.Descriptor + + arch := strings.Split(platform, "/")[1] + plat := strings.Split(platform, "/")[0] + + if arch == "" || plat == "" { + return ocispec.Image{}, fmt.Errorf("incomplete platform: %s", platform) + } + + for _, descriptor := range idx.Manifests { + if descriptor.Platform.OS == plat && descriptor.Platform.Architecture == arch { + manifestDesc = descriptor + break + } + } + + if manifestDesc.Digest == "" { + return ocispec.Image{}, fmt.Errorf("no manifest found for %s/%s", strings.Split(platform, "/")[0], strings.Split(platform, "/")[1]) + } + + // Split the digest into algorithm and hex + manifestHash := v1.Hash{ + Algorithm: manifestDesc.Digest.Algorithm().String(), + Hex: manifestDesc.Digest.Encoded(), + } + + manifestPath = filepath.Join( + handle.path, + DirectoryHandlerManifestsDir, + manifestHash.Algorithm, + manifestHash.Hex, + ) + } else { + // Check whether the reference is a digest + if strings.ContainsRune(fullref, '/') { + return ocispec.Image{}, fmt.Errorf("invalid reference: %s", fullref) + } + + manifestPath = filepath.Join( + handle.path, + DirectoryHandlerManifestsDir, + "sha256", + fullref, + ) + } // Check whether the manifest exists if _, err := os.Stat(manifestPath); err != nil {