Merge pull request #132 from nyaruka/minio

Tweak s3x.Service to make it easier to support minio

s3x/s3.go

@@ -7,45 +7,57 @@ import (
 	"io"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
 // Service is simple abstraction layer to work with a S3-compatible storage service
 type Service struct {
-	client *s3.S3
+	Client *s3.S3
 	urler  ObjectURLer
 }
 
-func NewService(client *s3.S3, urler ObjectURLer) *Service {
-	return &Service{client: client, urler: urler}
-}
-
-func (s *Service) HeadBucket(ctx context.Context, bucket string) error {
-	_, err := s.client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)})
+// NewService creates a new S3 service with the given credentials and configuration
+func NewService(accessKey, secretKey, region, endpoint string, minio bool) (*Service, error) {
+	cfg := &aws.Config{
+		Region:           aws.String(region),
+		Endpoint:         aws.String(endpoint),
+		S3ForcePathStyle: aws.Bool(minio), // urls as endpoint/bucket/key instead of bucket.endpoint/key
+		MaxRetries:       aws.Int(3),
+	}
+	if accessKey != "" || secretKey != "" {
+		cfg.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+	}
+	s, err := session.NewSession(cfg)
 	if err != nil {
-		return fmt.Errorf("error heading bucket: %w", err)
+		return nil, err
 	}
-	return nil
-}
 
-func (s *Service) CreateBucket(ctx context.Context, bucket string) error {
-	_, err := s.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
-	if err != nil {
-		return fmt.Errorf("error creating bucket: %w", err)
+	var urler ObjectURLer
+	if minio {
+		urler = MinioURLer(endpoint)
+	} else {
+		urler = AWSURLer(region)
 	}
-	return nil
+
+	return &Service{Client: s3.New(s), urler: urler}, nil
 }
 
-func (s *Service) DeleteBucket(ctx context.Context, bucket string) error {
-	_, err := s.client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
-	if err != nil {
-		return fmt.Errorf("error deleting bucket: %w", err)
-	}
-	return nil
+// ObjectURL returns the publicly accessible URL for the given object
+func (s *Service) ObjectURL(bucket, key string) string {
+	return s.urler(bucket, key)
 }
 
+// Test is a convenience method to HEAD a bucket to test if it exists and we can access it
+func (s *Service) Test(ctx context.Context, bucket string) error {
+	_, err := s.Client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)})
+	return err
+}
+
+// GetObject is a convenience method to get an object from S3 and read its contents into a byte slice
 func (s *Service) GetObject(ctx context.Context, bucket, key string) (string, []byte, error) {
-	out, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
+	out, err := s.Client.GetObjectWithContext(ctx, &s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
@@ -61,9 +73,9 @@ func (s *Service) GetObject(ctx context.Context, bucket, key string) (string, []
 	return aws.StringValue(out.ContentType), body, nil
 }
 
-// PutObject writes the passed in file to the given bucket with the passed in content type and ACL
+// PutObject is a convenience method to put the given object and return its publicly accessible URL
 func (s *Service) PutObject(ctx context.Context, bucket, key string, contentType string, body []byte, acl string) (string, error) {
-	_, err := s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{
+	_, err := s.Client.PutObjectWithContext(ctx, &s3.PutObjectInput{
 		Bucket:      aws.String(bucket),
 		Body:        bytes.NewReader(body),
 		Key:         aws.String(key),
@@ -73,6 +85,5 @@ func (s *Service) PutObject(ctx context.Context, bucket, key string, contentType
 	if err != nil {
 		return "", fmt.Errorf("error putting S3 object: %w", err)
 	}
-
-	return s.urler(key), nil
+	return s.urler(bucket, key), nil
 }

s3x/s3_test.go

@@ -5,55 +5,45 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/nyaruka/gocommon/s3x"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-func TestGetAndPutObject(t *testing.T) {
+func TestService(t *testing.T) {
 	ctx := context.Background()
 
-	config := &aws.Config{
-		Endpoint:         aws.String("http://localhost:9000"),
-		Region:           aws.String("us-east-1"),
-		Credentials:      credentials.NewStaticCredentials("root", "tembatemba", ""),
-		S3ForcePathStyle: aws.Bool(true),
-	}
-	s, err := session.NewSession(config)
-	require.NoError(t, err)
-
-	client := s3.New(s)
-	require.NotNil(t, client)
-
-	svc := s3x.NewService(client, s3x.MinioURLer("http://localhost:9000", "mybucket"))
+	svc, err := s3x.NewService("root", "tembatemba", "us-east-1", "http://localhost:9000", true)
+	assert.NoError(t, err)
 
-	err = svc.HeadBucket(ctx, "gocommon-tests")
-	assert.ErrorContains(t, err, "error heading bucket: NotFound: Not Found\n\tstatus code: 404")
+	err = svc.Test(ctx, "gocommon-tests")
+	assert.ErrorContains(t, err, "NotFound: Not Found\n\tstatus code: 404")
 
-	err = svc.CreateBucket(ctx, "gocommon-tests")
+	_, err = svc.Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("gocommon-tests")})
 	assert.NoError(t, err)
 
-	err = svc.HeadBucket(ctx, "gocommon-tests")
+	err = svc.Test(ctx, "gocommon-tests")
 	assert.NoError(t, err)
 
-	url, err := svc.PutObject(ctx, "gocommon-tests", "test.txt", "text/plain", []byte("hello world"), s3.BucketCannedACLPublicRead)
+	url, err := svc.PutObject(ctx, "gocommon-tests", "hello world.txt", "text/plain", []byte("hello world"), s3.BucketCannedACLPublicRead)
 	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:9000/mybucket/test.txt", url)
+	assert.Equal(t, "http://localhost:9000/gocommon-tests/hello%20world.txt", url)
 
-	contentType, body, err := svc.GetObject(ctx, "gocommon-tests", "test.txt")
+	contentType, body, err := svc.GetObject(ctx, "gocommon-tests", "hello world.txt")
 	assert.NoError(t, err)
 	assert.Equal(t, "text/plain", contentType)
 	assert.Equal(t, []byte("hello world"), body)
 
-	_, err = client.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String("gocommon-tests"), Key: aws.String("test.txt")})
+	_, err = svc.Client.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String("gocommon-tests"), Key: aws.String("hello world.txt")})
 	assert.NoError(t, err)
 
-	err = svc.DeleteBucket(ctx, "gocommon-tests")
+	_, err = svc.Client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String("gocommon-tests")})
 	assert.NoError(t, err)
 
-	err = svc.HeadBucket(ctx, "gocommon-tests")
+	err = svc.Test(ctx, "gocommon-tests")
 	assert.Error(t, err)
+
+	aws, err := s3x.NewService("AA1234", "2345263", "us-east-1", "https://s3.amazonaws.com", false)
+	assert.NoError(t, err)
+	assert.Equal(t, "https://gocommon-tests.s3.us-east-1.amazonaws.com/hello%20world.txt", aws.ObjectURL("gocommon-tests", "hello world.txt"))
 }

s3x/urls.go

@@ -6,16 +6,16 @@ import (
 )
 
 // ObjectURLer is a function that takes a key and returns the publicly accessible URL for that object
-type ObjectURLer func(string) string
+type ObjectURLer func(string, string) string
 
-func AWSURLer(region, bucket string) ObjectURLer {
-	return func(key string) string {
+func AWSURLer(region string) ObjectURLer {
+	return func(bucket, key string) string {
 		return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, url.PathEscape(key))
 	}
 }
 
-func MinioURLer(endpoint, bucket string) ObjectURLer {
-	return func(key string) string {
+func MinioURLer(endpoint string) ObjectURLer {
+	return func(bucket, key string) string {
 		return fmt.Sprintf("%s/%s/%s", endpoint, bucket, url.PathEscape(key))
 	}
 }

s3x/urls_test.go

@@ -8,9 +8,9 @@ import (
 )
 
 func TestURLers(t *testing.T) {
-	urler := s3x.AWSURLer("us-east-1", "mybucket")
-	assert.Equal(t, "https://mybucket.s3.us-east-1.amazonaws.com/hello%20world.txt", urler("hello world.txt"))
+	urler := s3x.AWSURLer("us-east-1")
+	assert.Equal(t, "https://mybucket.s3.us-east-1.amazonaws.com/hello%20world.txt", urler("mybucket", "hello world.txt"))
 
-	urler = s3x.MinioURLer("http://localhost:9000", "mybucket")
-	assert.Equal(t, "http://localhost:9000/mybucket/hello%20world.txt", urler("hello world.txt"))
+	urler = s3x.MinioURLer("http://localhost:9000")
+	assert.Equal(t, "http://localhost:9000/mybucket/hello%20world.txt", urler("mybucket", "hello world.txt"))
 }