Skip to content

Commit

Permalink
wip - translating RetrieveLogs
Browse files Browse the repository at this point in the history
  • Loading branch information
therealvio committed Nov 26, 2024
1 parent c2be95a commit 9e0d7ac
Show file tree
Hide file tree
Showing 6 changed files with 444 additions and 17 deletions.
10 changes: 6 additions & 4 deletions src/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,28 @@ module ecs-task-runner
go 1.22.3

require (
github.com/aws/aws-sdk-go-v2 v1.32.3
github.com/aws/aws-sdk-go-v2 v1.32.5
github.com/aws/aws-sdk-go-v2/config v1.28.1
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.44.0
github.com/aws/aws-sdk-go-v2/service/ecs v1.49.0
github.com/aws/aws-sdk-go-v2/service/ssm v1.55.3
github.com/kelseyhightower/envconfig v1.4.0
github.com/stretchr/testify v1.9.0
)

require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.42 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
github.com/aws/smithy-go v1.22.0 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand Down
20 changes: 12 additions & 8 deletions src/go.sum
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo=
github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.44.0 h1:OREVd94+oXW5a+3SSUAo4K0L5ci8cucCLu+PSiek8OU=
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.44.0/go.mod h1:Qbr4yfpNqVNl69l/GEDK+8wxLf/vHi0ChoiSDzD7thU=
github.com/aws/aws-sdk-go-v2/service/ecs v1.49.0 h1:xhCV6zY5ZFzfyAUOiBXK6wh0HVQTBkvNwA/eiz89ZWY=
github.com/aws/aws-sdk-go-v2/service/ecs v1.49.0/go.mod h1:RXYd/Ts+sFnjDrVdAZsAfHVkYxQUxhC+l2zrSpSgCGc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
Expand All @@ -26,8 +30,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gE
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE=
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down
30 changes: 30 additions & 0 deletions src/internal/aws/cloudwatch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package aws

import (
"context"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types"
)

type cloudwatchLogsClientAPI interface {
GetLogEvents(ctx context.Context, params *cloudwatchlogs.GetLogEventsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.GetLogEventsOutput, error)
}

type LogDetails struct {
logGroupName string
logStreamName string
}

func RetrieveLogs(ctx context.Context, cloudwatchLogsClientAPI cloudwatchLogsClientAPI, loggingDetails LogDetails) ([]types.OutputLogEvent, error) {
response, err := cloudwatchLogsClientAPI.GetLogEvents(ctx, &cloudwatchlogs.GetLogEventsInput{
LogStreamName: &loggingDetails.logStreamName,
LogGroupName: &loggingDetails.logGroupName,
StartFromHead: aws.Bool(true),
})
if err != nil {
return []types.OutputLogEvent{}, err
}
return response.Events, nil
}
99 changes: 99 additions & 0 deletions src/internal/aws/cloudwatch_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package aws

import (
"context"
"errors"
"testing"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types"
"github.com/stretchr/testify/assert"
)

type mockCloudwatchLogsClient struct {
mockGetLogEvents func(ctx context.Context, params *cloudwatchlogs.GetLogEventsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.GetLogEventsOutput, error)
}

func (m mockCloudwatchLogsClient) GetLogEvents(ctx context.Context, params *cloudwatchlogs.GetLogEventsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.GetLogEventsOutput, error) {
return m.mockGetLogEvents(ctx, params, optFns...)
}

func TestRetrieveLogs(t *testing.T) {
input := LogDetails{
logGroupName: "test-group",
logStreamName: "test-stream/test-container/07cc583696bd44e0be450bff7314ddaf",
}

events := []types.OutputLogEvent{
{
IngestionTime: aws.Int64(0),
Message: aws.String("beans have been detected in the system"),
Timestamp: aws.Int64(0),
},
{
IngestionTime: aws.Int64(1),
Message: aws.String("beans have been removed from the system"),
Timestamp: aws.Int64(1),
},
}

positiveTests := []struct {
name string
input LogDetails
client mockCloudwatchLogsClient
expected []types.OutputLogEvent
}{
{
// This is a regression test to ensure the function signature remains the same
name: "given a valid LogDetails input, return the events of the log stream",
input: input,
client: mockCloudwatchLogsClient{
mockGetLogEvents: func(ctx context.Context, params *cloudwatchlogs.GetLogEventsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.GetLogEventsOutput, error) {
return &cloudwatchlogs.GetLogEventsOutput{Events: events}, nil
},
},
expected: events,
},
}

for _, tc := range positiveTests {
t.Run(tc.name, func(t *testing.T) {
result, err := RetrieveLogs(context.TODO(), tc.client, tc.input)

t.Logf("result: %v", result)
t.Logf("expected: %v", tc.expected)
assert.NoError(t, err)
assert.Equal(t, tc.expected, result)
})
}

negativeTests := []struct {
name string
input LogDetails
client mockCloudwatchLogsClient
expected []types.OutputLogEvent
}{
{
name: "when the underlying cloudwatch client experiences an error, return it in the function ",
input: input,
client: mockCloudwatchLogsClient{
mockGetLogEvents: func(ctx context.Context, params *cloudwatchlogs.GetLogEventsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.GetLogEventsOutput, error) {
return &cloudwatchlogs.GetLogEventsOutput{}, errors.New("generic cloudwatch error")
},
},
expected: []types.OutputLogEvent{},
},
}

for _, tc := range negativeTests {
t.Run(tc.name, func(t *testing.T) {
result, err := RetrieveLogs(context.TODO(), tc.client, tc.input)

t.Logf("result: %v", result)
t.Logf("expected: %v", tc.expected)
assert.Error(t, err)
assert.Equal(t, tc.expected, result)
})
}
}
57 changes: 55 additions & 2 deletions src/internal/aws/ecs.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,17 @@ import (
)

// internal interface for ecs
type ecsClientAPI interface {
type EcsClientAPI interface {
RunTask(ctx context.Context, params *ecs.RunTaskInput, optFns ...func(*ecs.Options)) (*ecs.RunTaskOutput, error)
DescribeTasks(ctx context.Context, params *ecs.DescribeTasksInput, optFns ...func(*ecs.Options)) (*ecs.DescribeTasksOutput, error)
DescribeTaskDefinition(ctx context.Context, params *ecs.DescribeTaskDefinitionInput, optFns ...func(*ecs.Options)) (*ecs.DescribeTaskDefinitionOutput, error)
}

type ecsWaiterAPI interface {
WaitForOutput(ctx context.Context, params *ecs.DescribeTasksInput, maxWaitDur time.Duration, optFns ...func(*ecs.TasksStoppedWaiterOptions)) (*ecs.DescribeTasksOutput, error)
}

func SubmitTask(ctx context.Context, ecsAPI ecsClientAPI, input *TaskRunnerConfiguration) (string, error) {
func SubmitTask(ctx context.Context, ecsAPI EcsClientAPI, input *TaskRunnerConfiguration) (string, error) {
response, err := ecsAPI.RunTask(ctx, &ecs.RunTaskInput{
Cluster: &input.Cluster,
LaunchType: "FARGATE",
Expand Down Expand Up @@ -77,3 +78,55 @@ func ClusterFromTaskArn(arn string) string {
parts := strings.Split(arn, "/")
return parts[len(parts)-2]
}

func TaskIdFromArn(taskArn string) string {
parts := strings.Split(taskArn, "/")
return parts[len(parts)-1]
}

// Describes a single task from a cluster using the native `ecs:DescribeTasks` API.
// The taskArn is the task of interest to be described
func DescribeTask(ctx context.Context, ecsClientAPI EcsClientAPI, taskArn string) (types.Task, error) {
cluster := ClusterFromTaskArn(taskArn)
response, err := ecsClientAPI.DescribeTasks(ctx, &ecs.DescribeTasksInput{
Tasks: []string{taskArn},
Cluster: &cluster,
})

if err != nil {
return types.Task{}, err
}

// Given the input of `DescribeTasks` we should only be getting one Task back
return response.Tasks[0], nil
}

// Acquires LogStream details for given ECS Task
func FindLogStreamFromTask(ctx context.Context, ecsClientApi EcsClientAPI, task types.Task) (LogDetails, error) {
response, err := ecsClientApi.DescribeTaskDefinition(ctx, &ecs.DescribeTaskDefinitionInput{
TaskDefinition: task.TaskDefinitionArn,
})
if err != nil {
return LogDetails{}, err
}

// TODO: This was originally part of the if statement below, but it was moved out to avoid a nil pointer dereference when getting the `logGroupName` and `streamPrefix` values
if len(response.TaskDefinition.ContainerDefinitions) == 0 {
return LogDetails{}, fmt.Errorf("ecs:DescribeTaskDefinition response is missing ContainerDefinitions data: %v", response)
}

container := response.TaskDefinition.ContainerDefinitions[0] // assume first container is the application container
logGroupName := container.LogConfiguration.Options["awslogs-group"]
//NOTE: Takes the format: prefix-name/container-name/ecs-task-id
streamPrefix := container.LogConfiguration.Options["awslogs-stream-prefix"]

// We need the logGroupName, streamPrefix, and a container name to be able to produce a FindLogStreamOutput in full
if logGroupName == "" || streamPrefix == "" {
return LogDetails{}, fmt.Errorf("ecs:DescribeTaskDefinition response does not conftain required logging configuration: %v", response)
}

return LogDetails{
logGroupName: logGroupName,
logStreamName: fmt.Sprintf("%s/%s/%s", streamPrefix, *container.Name, TaskIdFromArn(*task.TaskArn)),
}, nil
}
Loading

0 comments on commit 9e0d7ac

Please sign in to comment.