diff --git a/s3x/s3.go b/s3x/s3.go index 9e0a374..cb1447b 100644 --- a/s3x/s3.go +++ b/s3x/s3.go @@ -7,45 +7,57 @@ import ( "io" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) // Service is simple abstraction layer to work with a S3-compatible storage service type Service struct { - client *s3.S3 + Client *s3.S3 urler ObjectURLer } -func NewService(client *s3.S3, urler ObjectURLer) *Service { - return &Service{client: client, urler: urler} -} - -func (s *Service) HeadBucket(ctx context.Context, bucket string) error { - _, err := s.client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)}) +// NewService creates a new S3 service with the given credentials and configuration +func NewService(accessKey, secretKey, region, endpoint string, minio bool) (*Service, error) { + cfg := &aws.Config{ + Region: aws.String(region), + Endpoint: aws.String(endpoint), + S3ForcePathStyle: aws.Bool(minio), // urls as endpoint/bucket/key instead of bucket.endpoint/key + MaxRetries: aws.Int(3), + } + if accessKey != "" || secretKey != "" { + cfg.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") + } + s, err := session.NewSession(cfg) if err != nil { - return fmt.Errorf("error heading bucket: %w", err) + return nil, err } - return nil -} -func (s *Service) CreateBucket(ctx context.Context, bucket string) error { - _, err := s.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)}) - if err != nil { - return fmt.Errorf("error creating bucket: %w", err) + var urler ObjectURLer + if minio { + urler = MinioURLer(endpoint) + } else { + urler = AWSURLer(region) } - return nil + + return &Service{Client: s3.New(s), urler: urler}, nil } -func (s *Service) DeleteBucket(ctx context.Context, bucket string) error { - _, err := s.client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)}) - if err != nil { - return fmt.Errorf("error deleting bucket: %w", err) - } - return nil +// ObjectURL returns the publicly accessible URL for the given object +func (s *Service) ObjectURL(bucket, key string) string { + return s.urler(bucket, key) } +// Test is a convenience method to HEAD a bucket to test if it exists and we can access it +func (s *Service) Test(ctx context.Context, bucket string) error { + _, err := s.Client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)}) + return err +} + +// GetObject is a convenience method to get an object from S3 and read its contents into a byte slice func (s *Service) GetObject(ctx context.Context, bucket, key string) (string, []byte, error) { - out, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{ + out, err := s.Client.GetObjectWithContext(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) @@ -61,9 +73,9 @@ func (s *Service) GetObject(ctx context.Context, bucket, key string) (string, [] return aws.StringValue(out.ContentType), body, nil } -// PutObject writes the passed in file to the given bucket with the passed in content type and ACL +// PutObject is a convenience method to put the given object and return its publicly accessible URL func (s *Service) PutObject(ctx context.Context, bucket, key string, contentType string, body []byte, acl string) (string, error) { - _, err := s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{ + _, err := s.Client.PutObjectWithContext(ctx, &s3.PutObjectInput{ Bucket: aws.String(bucket), Body: bytes.NewReader(body), Key: aws.String(key), @@ -73,6 +85,5 @@ func (s *Service) PutObject(ctx context.Context, bucket, key string, contentType if err != nil { return "", fmt.Errorf("error putting S3 object: %w", err) } - - return s.urler(key), nil + return s.urler(bucket, key), nil } diff --git a/s3x/s3_test.go b/s3x/s3_test.go index d92ffa7..d9e339e 100644 --- a/s3x/s3_test.go +++ b/s3x/s3_test.go @@ -5,55 +5,45 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/nyaruka/gocommon/s3x" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func TestGetAndPutObject(t *testing.T) { +func TestService(t *testing.T) { ctx := context.Background() - config := &aws.Config{ - Endpoint: aws.String("http://localhost:9000"), - Region: aws.String("us-east-1"), - Credentials: credentials.NewStaticCredentials("root", "tembatemba", ""), - S3ForcePathStyle: aws.Bool(true), - } - s, err := session.NewSession(config) - require.NoError(t, err) - - client := s3.New(s) - require.NotNil(t, client) - - svc := s3x.NewService(client, s3x.MinioURLer("http://localhost:9000", "mybucket")) + svc, err := s3x.NewService("root", "tembatemba", "us-east-1", "http://localhost:9000", true) + assert.NoError(t, err) - err = svc.HeadBucket(ctx, "gocommon-tests") - assert.ErrorContains(t, err, "error heading bucket: NotFound: Not Found\n\tstatus code: 404") + err = svc.Test(ctx, "gocommon-tests") + assert.ErrorContains(t, err, "NotFound: Not Found\n\tstatus code: 404") - err = svc.CreateBucket(ctx, "gocommon-tests") + _, err = svc.Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("gocommon-tests")}) assert.NoError(t, err) - err = svc.HeadBucket(ctx, "gocommon-tests") + err = svc.Test(ctx, "gocommon-tests") assert.NoError(t, err) - url, err := svc.PutObject(ctx, "gocommon-tests", "test.txt", "text/plain", []byte("hello world"), s3.BucketCannedACLPublicRead) + url, err := svc.PutObject(ctx, "gocommon-tests", "hello world.txt", "text/plain", []byte("hello world"), s3.BucketCannedACLPublicRead) assert.NoError(t, err) - assert.Equal(t, "http://localhost:9000/mybucket/test.txt", url) + assert.Equal(t, "http://localhost:9000/gocommon-tests/hello%20world.txt", url) - contentType, body, err := svc.GetObject(ctx, "gocommon-tests", "test.txt") + contentType, body, err := svc.GetObject(ctx, "gocommon-tests", "hello world.txt") assert.NoError(t, err) assert.Equal(t, "text/plain", contentType) assert.Equal(t, []byte("hello world"), body) - _, err = client.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String("gocommon-tests"), Key: aws.String("test.txt")}) + _, err = svc.Client.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String("gocommon-tests"), Key: aws.String("hello world.txt")}) assert.NoError(t, err) - err = svc.DeleteBucket(ctx, "gocommon-tests") + _, err = svc.Client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String("gocommon-tests")}) assert.NoError(t, err) - err = svc.HeadBucket(ctx, "gocommon-tests") + err = svc.Test(ctx, "gocommon-tests") assert.Error(t, err) + + aws, err := s3x.NewService("AA1234", "2345263", "us-east-1", "https://s3.amazonaws.com", false) + assert.NoError(t, err) + assert.Equal(t, "https://gocommon-tests.s3.us-east-1.amazonaws.com/hello%20world.txt", aws.ObjectURL("gocommon-tests", "hello world.txt")) } diff --git a/s3x/urls.go b/s3x/urls.go index 4e7f908..033b316 100644 --- a/s3x/urls.go +++ b/s3x/urls.go @@ -6,16 +6,16 @@ import ( ) // ObjectURLer is a function that takes a key and returns the publicly accessible URL for that object -type ObjectURLer func(string) string +type ObjectURLer func(string, string) string -func AWSURLer(region, bucket string) ObjectURLer { - return func(key string) string { +func AWSURLer(region string) ObjectURLer { + return func(bucket, key string) string { return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, url.PathEscape(key)) } } -func MinioURLer(endpoint, bucket string) ObjectURLer { - return func(key string) string { +func MinioURLer(endpoint string) ObjectURLer { + return func(bucket, key string) string { return fmt.Sprintf("%s/%s/%s", endpoint, bucket, url.PathEscape(key)) } } diff --git a/s3x/urls_test.go b/s3x/urls_test.go index 1c49417..19e95b1 100644 --- a/s3x/urls_test.go +++ b/s3x/urls_test.go @@ -8,9 +8,9 @@ import ( ) func TestURLers(t *testing.T) { - urler := s3x.AWSURLer("us-east-1", "mybucket") - assert.Equal(t, "https://mybucket.s3.us-east-1.amazonaws.com/hello%20world.txt", urler("hello world.txt")) + urler := s3x.AWSURLer("us-east-1") + assert.Equal(t, "https://mybucket.s3.us-east-1.amazonaws.com/hello%20world.txt", urler("mybucket", "hello world.txt")) - urler = s3x.MinioURLer("http://localhost:9000", "mybucket") - assert.Equal(t, "http://localhost:9000/mybucket/hello%20world.txt", urler("hello world.txt")) + urler = s3x.MinioURLer("http://localhost:9000") + assert.Equal(t, "http://localhost:9000/mybucket/hello%20world.txt", urler("mybucket", "hello world.txt")) }