diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3949e9d..64de089 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,14 @@ jobs: ports: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + minio: + image: bitnami/minio:latest + env: + MINIO_ROOT_USER: root + MINIO_ROOT_PASSWORD: tembatemba + ports: + - 9000:9000 + options: --health-cmd "mc ready local" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - name: Checkout code diff --git a/s3x/s3.go b/s3x/s3.go new file mode 100644 index 0000000..9e0a374 --- /dev/null +++ b/s3x/s3.go @@ -0,0 +1,78 @@ +package s3x + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Service is simple abstraction layer to work with a S3-compatible storage service +type Service struct { + client *s3.S3 + urler ObjectURLer +} + +func NewService(client *s3.S3, urler ObjectURLer) *Service { + return &Service{client: client, urler: urler} +} + +func (s *Service) HeadBucket(ctx context.Context, bucket string) error { + _, err := s.client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)}) + if err != nil { + return fmt.Errorf("error heading bucket: %w", err) + } + return nil +} + +func (s *Service) CreateBucket(ctx context.Context, bucket string) error { + _, err := s.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)}) + if err != nil { + return fmt.Errorf("error creating bucket: %w", err) + } + return nil +} + +func (s *Service) DeleteBucket(ctx context.Context, bucket string) error { + _, err := s.client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)}) + if err != nil { + return fmt.Errorf("error deleting bucket: %w", err) + } + return nil +} + +func (s *Service) GetObject(ctx context.Context, bucket, key string) (string, []byte, error) { + out, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return "", nil, fmt.Errorf("error getting S3 object: %w", err) + } + + body, err := io.ReadAll(out.Body) + if err != nil { + return "", nil, fmt.Errorf("error reading S3 object: %w", err) + } + + return aws.StringValue(out.ContentType), body, nil +} + +// PutObject writes the passed in file to the given bucket with the passed in content type and ACL +func (s *Service) PutObject(ctx context.Context, bucket, key string, contentType string, body []byte, acl string) (string, error) { + _, err := s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Body: bytes.NewReader(body), + Key: aws.String(key), + ContentType: aws.String(contentType), + ACL: aws.String(acl), + }) + if err != nil { + return "", fmt.Errorf("error putting S3 object: %w", err) + } + + return s.urler(key), nil +} diff --git a/s3x/s3_test.go b/s3x/s3_test.go new file mode 100644 index 0000000..d92ffa7 --- /dev/null +++ b/s3x/s3_test.go @@ -0,0 +1,59 @@ +package s3x_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/nyaruka/gocommon/s3x" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetAndPutObject(t *testing.T) { + ctx := context.Background() + + config := &aws.Config{ + Endpoint: aws.String("http://localhost:9000"), + Region: aws.String("us-east-1"), + Credentials: credentials.NewStaticCredentials("root", "tembatemba", ""), + S3ForcePathStyle: aws.Bool(true), + } + s, err := session.NewSession(config) + require.NoError(t, err) + + client := s3.New(s) + require.NotNil(t, client) + + svc := s3x.NewService(client, s3x.MinioURLer("http://localhost:9000", "mybucket")) + + err = svc.HeadBucket(ctx, "gocommon-tests") + assert.ErrorContains(t, err, "error heading bucket: NotFound: Not Found\n\tstatus code: 404") + + err = svc.CreateBucket(ctx, "gocommon-tests") + assert.NoError(t, err) + + err = svc.HeadBucket(ctx, "gocommon-tests") + assert.NoError(t, err) + + url, err := svc.PutObject(ctx, "gocommon-tests", "test.txt", "text/plain", []byte("hello world"), s3.BucketCannedACLPublicRead) + assert.NoError(t, err) + assert.Equal(t, "http://localhost:9000/mybucket/test.txt", url) + + contentType, body, err := svc.GetObject(ctx, "gocommon-tests", "test.txt") + assert.NoError(t, err) + assert.Equal(t, "text/plain", contentType) + assert.Equal(t, []byte("hello world"), body) + + _, err = client.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String("gocommon-tests"), Key: aws.String("test.txt")}) + assert.NoError(t, err) + + err = svc.DeleteBucket(ctx, "gocommon-tests") + assert.NoError(t, err) + + err = svc.HeadBucket(ctx, "gocommon-tests") + assert.Error(t, err) +} diff --git a/s3x/urls.go b/s3x/urls.go new file mode 100644 index 0000000..4e7f908 --- /dev/null +++ b/s3x/urls.go @@ -0,0 +1,21 @@ +package s3x + +import ( + "fmt" + "net/url" +) + +// ObjectURLer is a function that takes a key and returns the publicly accessible URL for that object +type ObjectURLer func(string) string + +func AWSURLer(region, bucket string) ObjectURLer { + return func(key string) string { + return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, url.PathEscape(key)) + } +} + +func MinioURLer(endpoint, bucket string) ObjectURLer { + return func(key string) string { + return fmt.Sprintf("%s/%s/%s", endpoint, bucket, url.PathEscape(key)) + } +} diff --git a/s3x/urls_test.go b/s3x/urls_test.go new file mode 100644 index 0000000..1c49417 --- /dev/null +++ b/s3x/urls_test.go @@ -0,0 +1,16 @@ +package s3x_test + +import ( + "testing" + + "github.com/nyaruka/gocommon/s3x" + "github.com/stretchr/testify/assert" +) + +func TestURLers(t *testing.T) { + urler := s3x.AWSURLer("us-east-1", "mybucket") + assert.Equal(t, "https://mybucket.s3.us-east-1.amazonaws.com/hello%20world.txt", urler("hello world.txt")) + + urler = s3x.MinioURLer("http://localhost:9000", "mybucket") + assert.Equal(t, "http://localhost:9000/mybucket/hello%20world.txt", urler("hello world.txt")) +}