feat(sdk): add collections for nanotdf (#1695)

Adds in Datasets for NanoTDF. If dataset store is enabled in SDK,
nanoTDF headers will be saved with their unwrapped key, so future
nanoTDFs can be decrypted without extra rewrap calls. NanoTDFs will be
able to be written in a dataset in the NanoTDFConfig, allowing for
flexible usage of datasets.


#### Example Local Run
To run a quick example collection locally. This will write two
collections with 50 nTDFs with the plaintext being changed to `<i>:
<plaintext>`. Decrypt will decrypt the two collections and will only do
two unwrap calls rather than 100 for each nTDF.

```
mkdir collection
go run ./examples encrypt "Collection A" --nano -c 50 -o collection/collection_a.ntdf
go run ./examples encrypt "Collection B" --nano -c 50 -o collection/collection_b.ntdf
go run ./examples decrypt collection
```

examples/cmd/decrypt.go

@@ -3,10 +3,11 @@ package cmd
 import (
 	"bytes"
 	"errors"
+	"fmt"
+	"github.com/spf13/cobra"
 	"io"
 	"os"
-
-	"github.com/spf13/cobra"
+	"path/filepath"
 )
 
 func init() {
@@ -31,6 +32,29 @@ func decrypt(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	// Collection
+	if stat, err := os.Stat(tdfFile); err == nil && stat.IsDir() {
+		entries, err := os.ReadDir(tdfFile)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			if !entry.IsDir() {
+				f, err := os.Open(filepath.Join(tdfFile, entry.Name()))
+				if err != nil {
+					return err
+				}
+				_, err = client.ReadNanoTDF(os.Stdout, f)
+				fmt.Println()
+				if err != nil {
+					return err
+				}
+			}
+		}
+		client.Close()
+		return nil
+	}
+
 	file, err := os.Open(tdfFile)
 	if err != nil {
 		return err

examples/cmd/encrypt.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/opentdf/platform/lib/ocrypto"
@@ -20,6 +22,7 @@ var (
 	noKIDInNano    bool
 	outputName     string
 	dataAttributes []string
+	collection     int
 )
 
 func init() {
@@ -35,6 +38,7 @@ func init() {
 	encryptCmd.Flags().BoolVar(&noKIDInKAO, "no-kid-in-kao", false, "[deprecated] Disable storing key identifiers in TDF KAOs")
 	encryptCmd.Flags().BoolVar(&noKIDInNano, "no-kid-in-nano", true, "Disable storing key identifiers in nanoTDF KAS ResourceLocator")
 	encryptCmd.Flags().StringVarP(&outputName, "output", "o", "sensitive.txt.tdf", "name or path of output file; - for stdout")
+	encryptCmd.Flags().IntVarP(&collection, "collection", "c", 0, "number of nano's to create for collection. If collection >0 (default) then output will be <iteration>_<output>")
 
 	ExamplesCmd.AddCommand(&encryptCmd)
 }
@@ -50,7 +54,6 @@ func encrypt(cmd *cobra.Command, args []string) error {
 	opts := []sdk.Option{
 		sdk.WithInsecurePlaintextConn(),
 		sdk.WithClientCredentials("opentdf-sdk", "secret", nil),
-		sdk.WithTokenEndpoint("http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token"),
 	}
 
 	if noKIDInKAO {
@@ -68,17 +71,32 @@ func encrypt(cmd *cobra.Command, args []string) error {
 	}
 
 	out := os.Stdout
-	if outputName != "-" {
-		out, err = os.Create(outputName)
-		if err != nil {
-			return err
-		}
+	if outputName == "-" && collection > 0 {
+		return fmt.Errorf("cannot use stdout for collection")
 	}
-	defer func() {
-		if outputName != "-" {
-			out.Close()
+
+	var writer []io.Writer
+	if outputName == "-" {
+		writer = append(writer, out)
+	} else {
+		dir, file := filepath.Split(outputName)
+		for i := 0; i < collection; i++ {
+			out, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d_%s", i, file)))
+			if err != nil {
+				return err
+			}
+			writer = append(writer, out)
+			defer out.Close()
+		}
+		if collection == 0 {
+			out, err = os.Create(outputName)
+			writer = append(writer, out)
+			defer out.Close()
+			if err != nil {
+				return err
+			}
 		}
-	}()
+	}
 
 	if !nanoFormat {
 		opts := []sdk.TDFOption{sdk.WithDataAttributes(dataAttributes...)}
@@ -108,17 +126,27 @@ func encrypt(cmd *cobra.Command, args []string) error {
 		}
 		nanoTDFConfig.SetAttributes(dataAttributes)
 		nanoTDFConfig.EnableECDSAPolicyBinding()
+		if collection > 0 {
+			nanoTDFConfig.EnableCollection()
+		}
 		err = nanoTDFConfig.SetKasURL(fmt.Sprintf("http://%s/kas", platformEndpoint))
 		if err != nil {
 			return err
 		}
+		for i, writer := range writer {
+			input := plainText
+			if collection > 0 {
+				input = fmt.Sprintf("%d: %s", i, plainText)
+			}
+			in = strings.NewReader(input)
+			_, err = client.CreateNanoTDF(writer, in, *nanoTDFConfig)
+			if err != nil {
+				return err
 
-		_, err = client.CreateNanoTDF(out, in, *nanoTDFConfig)
-		if err != nil {
-			return err
+			}
 		}
 
-		if outputName != "-" {
+		if outputName != "-" && collection == 0 {
 			err = cat(cmd, outputName)
 			if err != nil {
 				return err

examples/cmd/examples.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"google.golang.org/grpc/resolver"
 	"log"
 	"os"
 	"strings"
@@ -28,7 +29,8 @@ func init() {
 }
 
 func newSDK() (*sdk.SDK, error) {
-	opts := []sdk.Option{sdk.WithInsecurePlaintextConn()}
+	resolver.SetDefaultScheme("passthrough")
+	opts := []sdk.Option{sdk.WithStoreCollectionHeaders(), sdk.WithInsecurePlaintextConn()}
 	if clientCredentials != "" {
 		i := strings.Index(clientCredentials, ":")
 		if i < 0 {

examples/go.mod

@@ -6,21 +6,25 @@ require (
 	github.com/opentdf/platform/lib/ocrypto v0.1.6
 	github.com/opentdf/platform/protocol/go v0.2.20
 	github.com/opentdf/platform/sdk v0.3.19
-	github.com/spf13/cobra v1.8.0
+	github.com/spf13/cobra v1.8.1
+	google.golang.org/grpc v1.66.0
 	google.golang.org/protobuf v1.34.2
 )
 
 require (
-	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240221180331-f05a6f4403ce.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/containerd/containerd v1.7.21 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/docker/docker v27.1.1+incompatible // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gowebpki/jcs v1.0.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/httprc v1.0.5 // indirect
@@ -31,17 +35,18 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/testcontainers/testcontainers-go v0.32.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
-	google.golang.org/grpc v1.62.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/crypto v0.26.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )