From d6414702dce600df71fb5e38c1fab6abc4cc738b Mon Sep 17 00:00:00 2001 From: Katie McKew <5915468+ktmq@users.noreply.github.com> Date: Mon, 11 Nov 2024 12:57:10 -0500 Subject: [PATCH] [AWSMC-1069] Update templates to use v2 API and support UPDATE (#129) Co-authored-by: jvanbrie --- aws_organizations/main_organizations.yaml | 302 ++++++++++------ aws_organizations/taskcat/.taskcat.yml | 19 + aws_organizations/taskcat/README.md | 19 + .../taskcat/run-taskcat-tests.sh | 41 +++ .../datadog_integration_api_call.yaml | 221 ------------ .../datadog_integration_api_call_v2.yaml | 324 +++++++++++------- aws_quickstart/dd_api_call.py | 218 ++++++++++++ aws_quickstart/main.yaml | 163 --------- aws_quickstart/main_v2.yaml | 22 +- aws_quickstart/taskcat/README.md | 19 + aws_quickstart/taskcat/run-taskcat-tests.sh | 5 + aws_quickstart/version.txt | 2 +- 12 files changed, 732 insertions(+), 623 deletions(-) create mode 100644 aws_organizations/taskcat/.taskcat.yml create mode 100644 aws_organizations/taskcat/README.md create mode 100755 aws_organizations/taskcat/run-taskcat-tests.sh delete mode 100644 aws_quickstart/datadog_integration_api_call.yaml create mode 100644 aws_quickstart/dd_api_call.py delete mode 100644 aws_quickstart/main.yaml create mode 100644 aws_quickstart/taskcat/README.md diff --git a/aws_organizations/main_organizations.yaml b/aws_organizations/main_organizations.yaml index cb19c2a..d828e9d 100644 --- a/aws_organizations/main_organizations.yaml +++ b/aws_organizations/main_organizations.yaml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Integration +Description: Configures the Datadog AWS Integration Parameters: DatadogApiKey: Description: >- @@ -84,7 +84,7 @@ Conditions: - ap1.datadoghq.com Resources: - LambdaExecutionRoleDatadogAPICall: + DatadogAPIHandlerLambdaExecutionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: @@ -99,8 +99,8 @@ Resources: Path: "/" ManagedPolicyArns: - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - DatadogAPICall: - Type: "Custom::DatadogAPICall" + DatadogAWSAccountIntegration: + Type: "Custom::DatadogAWSAccountIntegration" Properties: ServiceToken: !GetAtt DatadogAPICallFunction.Arn APIKey: !Ref DatadogApiKey @@ -108,7 +108,8 @@ Resources: ApiURL: !Ref DatadogSite AccountId: !Ref AWS::AccountId RoleName: !Ref IAMRoleName - HostTags: [ !Sub "aws_account:${AWS::AccountId}" ] + AWSPartition: !Ref AWS::Partition + AccountTags: [!Sub "aws_account:${AWS::AccountId}"] CloudSecurityPostureManagement: !Ref CloudSecurityPostureManagement DisableMetricCollection: !Ref DisableMetricCollection DisableResourceCollection: !Ref DisableResourceCollection @@ -116,145 +117,230 @@ Resources: Type: "AWS::Lambda::Function" Properties: Description: "A function to call the Datadog API." - Role: !GetAtt LambdaExecutionRoleDatadogAPICall.Arn + Role: !GetAtt DatadogAPIHandlerLambdaExecutionRole.Arn Handler: "index.handler" Runtime: "python3.11" Timeout: 30 Code: ZipFile: | - import boto3 - import json import logging import signal - from urllib.request import build_opener, HTTPHandler, Request + from urllib.request import Request import urllib.parse + import cfnresponse LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) API_CALL_SOURCE_HEADER_VALUE = "cfn-organizations" - def call_datadog_api(event, method): - api_key = event['ResourceProperties']['APIKey'] - app_key = event['ResourceProperties']['APPKey'] - api_url = event['ResourceProperties']['ApiURL'] - account_id = event['ResourceProperties']['AccountId'] - role_name = event['ResourceProperties']['RoleName'] - host_tags = event['ResourceProperties']['HostTags'] - cspm = event['ResourceProperties']['CloudSecurityPostureManagement'] - metrics_disabled = event['ResourceProperties']['DisableMetricCollection'] + class TimeoutError(Exception): + """Exception for timeouts""" + pass + + def call_datadog_api(uuid, event, method): + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] + role_name = event["ResourceProperties"]["RoleName"] + aws_partition = event["ResourceProperties"]["AWSPartition"] + account_tags = event["ResourceProperties"]["AccountTags"] + cspm = event["ResourceProperties"]["CloudSecurityPostureManagement"] + metrics_disabled = event["ResourceProperties"]["DisableMetricCollection"] resource_collection_disabled = event['ResourceProperties']['DisableResourceCollection'] # Make the url Request - url = 'https://api.' + api_url + '/api/v1/integration/aws' - values = { - 'account_id': account_id, - 'role_name': role_name, - } - if method != "DELETE": - values["host_tags"] = host_tags - values["cspm_resource_collection_enabled"] = cspm == "true" - values["metrics_collection_enabled"] = metrics_disabled == "false" - values["extended_resource_collection_enabled"] = resource_collection_disabled == "false" - + url = f"https://api.{api_url}/api/v2/integration/aws/accounts" headers = { - 'DD-API-KEY': api_key, - 'DD-APPLICATION-KEY': app_key, - 'Dd-Aws-Api-Call-Source': API_CALL_SOURCE_HEADER_VALUE, + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, } - data = json.dumps(values) - data = data.encode('utf-8') # data should be bytes - request = Request(url, data=data, headers=headers) - request.add_header('Content-Type', 'application/json; charset=utf-8') - request.add_header('Content-Length', len(data)) + + if method == "PATCH" or method == "DELETE": + url = url + "/" + uuid + + if method == "DELETE": + # DELETE request has no body + request = Request(url, headers=headers) + else: + # Create the request body for POST and PATCH + values = { + "data": { + "type": "account", + "attributes": { + "aws_account_id": account_id, + "account_tags": account_tags, + "aws_partition": aws_partition, + "auth_config": {"role_name": role_name}, + "metrics_config": { + "enabled": (metrics_disabled == "false"), + }, + "resources_config": { + "cloud_security_posture_management_collection": ( + cspm == "true" + ), + "extended_resource_collection": ( + resource_collection_disabled == "false" + ) + } + } + } + } + + data = json.dumps(values) + data = data.encode("utf-8") # data should be bytes + request = Request(url, data=data, headers=headers) + request.add_header("Content-Type", "application/json; charset=utf-8") + request.add_header("Content-Length", len(data)) + + # Send the request request.get_method = lambda: method + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + # Return error response from API + response = e + return response + + def get_datadog_account(event): + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] - # Send the url Request, store external_id - response = urllib.request.urlopen(request) + # Make the url Request + url = f"https://api.{api_url}/api/v2/integration/aws/accounts?aws_account_id={account_id}" + headers = { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + request = Request(url, headers=headers) + request.get_method = lambda: "GET" + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + # Return error response from API + response = e return response + def handler(event, context): - '''Handle Lambda event from AWS''' - try: - LOGGER.info('REQUEST RECEIVED:\n %s', event) - LOGGER.info('REQUEST RECEIVED:\n %s', context) - if event['RequestType'] == 'Create': - LOGGER.info('Received Create request.') - response = call_datadog_api(event, 'POST') - if response.getcode() == 200: - json_response = json.loads(response.read().decode("utf-8")) - send_response(event, context, "SUCCESS", - { - "Message": "Datadog AWS Integration created successfully.", - "ExternalId": json_response["external_id"], - }) - else: - LOGGER.info('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Http response: {}".format(response.msg)}) + """Handle Lambda event from AWS""" + if event["RequestType"] == "Create": + LOGGER.info("Received Create request.") + method = "POST" - elif event['RequestType'] == 'Update': - LOGGER.info('Received Update request.') - send_response(event, context, "SUCCESS", - {"Message": "Update not supported, no operation performed."}) - elif event['RequestType'] == 'Delete': - LOGGER.info('Received Delete request.') - response = call_datadog_api(event, 'DELETE') + elif event["RequestType"] == "Update": + LOGGER.info("Received Update request.") + method = "PATCH" - if response.getcode() == 200: - send_response(event, context, "SUCCESS", - { - "Message": "Datadog AWS Integration deleted successfully.", - }) - else: - LOGGER.info('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Http response: {}".format(response.msg)}) + elif event["RequestType"] == "Delete": + LOGGER.info("Received Delete request.") + method = "DELETE" + else: + LOGGER.info("Failed - received unexpected request.") + cfResponse = {"Message": "Received unexpected request type: {}".format(event["RequestType"])} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + responseStatus="FAILED", + responseData=cfResponse, + reason=reason, + ) + return - else: - LOGGER.info('Failed - received unexpected request.') - send_response(event, context, "FAILED", - {"Message": "Unexpected event received from CloudFormation"}) - except Exception as e: # pylint: disable=W0702 - LOGGER.info('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Exception during processing: {}".format(e)}) + try: + # Call Datadog API and report response back to CloudFormation + uuid = "" + if event["RequestType"] != "Create": + datadog_account_response = get_datadog_account(event) + uuid = extract_uuid_from_account_response(event, context, datadog_account_response) + if uuid is None: + return + response = call_datadog_api(uuid, event, method) + cfn_response_send_api_result(event, context, method, response) + except Exception as e: + LOGGER.info("Failed - exception thrown during processing.") + cfResponse = {"Message": "Exception during processing: {}".format(e)} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + "FAILED", + responseData=cfResponse, + reason=reason, + ) - def send_response(event, context, response_status, response_data): - '''Send a resource manipulation status response to CloudFormation''' - response_body = json.dumps({ - "Status": response_status, - "Reason": "See the details in CloudWatch Log Stream: " + context.log_stream_name, - "PhysicalResourceId": context.log_stream_name, - "StackId": event['StackId'], - "RequestId": event['RequestId'], - "LogicalResourceId": event['LogicalResourceId'], - "Data": response_data - }) - formatted_response = response_body.encode("utf-8") + def extract_uuid_from_account_response(event, context, account_response): + json_response = "" + code = account_response.getcode() + data = account_response.read() + if data: + json_response = json.loads(data) + if code == 200 or code == 204: + if len(json_response["data"]) == 0: + cfn_response_send_failure(event, context, "Datadog account not found.") + return None + if len(json_response["data"]) > 1: + cfn_response_send_failure(event, context, "Datadog account not unique.") + return None + return json_response["data"][0]["id"] + cfn_response_send_failure(event, context, "Datadog API returned error: {}".format(json_response)) + return None - LOGGER.info('ResponseURL: %s', event['ResponseURL']) - LOGGER.info('ResponseBody: %s', response_body) - opener = build_opener(HTTPHandler) - request = Request(event['ResponseURL'], data=formatted_response) - request.add_header('Content-Type', 'application/json; charset=utf-8') - request.add_header('Content-Length', len(formatted_response)) - request.get_method = lambda: 'PUT' - response = opener.open(request) - LOGGER.info("Status code: %s", response.getcode()) - LOGGER.info("Status message: %s", response.msg) + def cfn_response_send_api_result(event, context, method, response): + reason = None + json_response = "" + code = response.getcode() + data = response.read() + if data: + json_response = json.loads(data) + if code == 200 or code == 204: + LOGGER.info("Success - Datadog API call was successful.") + response_status = "SUCCESS" + cfResponse = {"Message": "Datadog AWS Integration {} API request was successful.".format(method)} + # return external ID for create and update + if method == "POST" or method == "PATCH": + external_id = json_response["data"]["attributes"]["auth_config"]["external_id"] + cfResponse["ExternalId"] = external_id + cfnresponse.send( + event, + context, + responseStatus=response_status, + responseData=cfResponse, + reason=reason, + ) + return + cfn_response_send_failure(event, context, "Datadog API returned error: {}".format(json_response)) - def timeout_handler(_signal, _frame): - '''Handle SIGALRM''' - raise Exception('Time exceeded') + def cfn_response_send_failure(event, context, message): + LOGGER.info("Failed - Datadog API call failed.") + reason = None + response_status = "FAILED" + cfResponse = {"Message": message} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + signalcontext, + responseStatus=response_status, + responseData=cfResponse, + reason=reason, + ) - signal.signal(signal.SIGALRM, timeout_handler) + def timeout_handler(_signal, _frame): + """Handle SIGALRM""" + raise TimeoutError("Lambda function timeout exceeded - increase the timeout set in the api_call Cloudformation template.") + signal.signal(signal.SIGALRM, timeout_handler) DatadogIntegrationRole: Type: 'AWS::IAM::Role' Metadata: @@ -281,7 +367,7 @@ Resources: - 'sts:AssumeRole' Condition: StringEquals: - 'sts:ExternalId': !GetAtt DatadogAPICall.ExternalId + 'sts:ExternalId': !GetAtt DatadogAWSAccountIntegration.ExternalId Path: / RoleName: !Ref IAMRoleName ManagedPolicyArns: !If [ ResourceCollectionPermissions, [ !Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit" ], !Ref AWS::NoValue ] diff --git a/aws_organizations/taskcat/.taskcat.yml b/aws_organizations/taskcat/.taskcat.yml new file mode 100644 index 0000000..001221c --- /dev/null +++ b/aws_organizations/taskcat/.taskcat.yml @@ -0,0 +1,19 @@ +general: + auth: + default: "" + s3_bucket: datadog-cloudformation-templates-aws-taskcat-test + +project: + name: aws-organizations + regions: + - us-east-2 +tests: + default: + template: ./main_organizations.yaml + parameters: + DatadogApiKey: "" + DatadogAppKey: "" + DatadogSite: "datad0g.com" + IAMRoleName: "DatadogIntegrationRole-taskcat-$[taskcat_random-string]" + DisableMetricCollection: "false" + CloudSecurityPostureManagement: "false" diff --git a/aws_organizations/taskcat/README.md b/aws_organizations/taskcat/README.md new file mode 100644 index 0000000..1b5679e --- /dev/null +++ b/aws_organizations/taskcat/README.md @@ -0,0 +1,19 @@ +# How to run taskcat tests + +## Setup +``` +export DD_API_KEY= +export DD_APP_KEY= +export AWS_SSO_PROFILE_NAME= +``` + +### Run +``` +./run-taskcat-test.sh +``` + +### Cleanup +``` +# To delete test stacks, run: +taskcat test clean aws-organizations -a ${AWS_SSO_PROFILE_NAME} +``` diff --git a/aws_organizations/taskcat/run-taskcat-tests.sh b/aws_organizations/taskcat/run-taskcat-tests.sh new file mode 100755 index 0000000..01096b3 --- /dev/null +++ b/aws_organizations/taskcat/run-taskcat-tests.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +if [ -z "$AWS_SSO_PROFILE_NAME" ]; then + echo "Missing AWS_SSO_PROFILE_NAME - Must specify an AWS profile name" + exit 1 +fi + +aws sso login --profile ${AWS_SSO_PROFILE_NAME} + +TASKCAT_S3_BUCKET="datadog-cloudformation-templates-aws-taskcat-test" +TASKCAT_PROJECT="aws-organizations" + +if [ -z "$DD_API_KEY" ]; then + echo "Missing DD_API_KEY - Must specify a Datadog API key" + exit 1 +fi + +if [ -z "$DD_APP_KEY" ]; then + echo "Missing DD_APP_KEY - Must specify a Datadog APP key" + exit 1 +fi + +if ! docker info > /dev/null 2>&1; then + echo "This script uses docker, and it isn't running - please start docker and try again!" + exit 1 +fi + +mkdir -p ./tmp + +for f in ../*.yaml; do + sed "s|.s3.amazonaws.com/aws/|${TASKCAT_S3_BUCKET}.s3.amazonaws.com/${TASKCAT_PROJECT}|g" $f > ./tmp/$(basename $f) +done + +sed "s||${DD_API_KEY}|g ; s||${DD_APP_KEY}|g ; s||${AWS_SSO_PROFILE_NAME}|g" ./.taskcat.yml > ./tmp/.taskcat.yml + +taskcat upload -b ${TASKCAT_S3_BUCKET} -k ${TASKCAT_PROJECT} -p tmp + +taskcat test run --skip-upload --project-root tmp --no-delete + +echo "To delete test stacks, run:" +echo " taskcat test clean ${TASKCAT_PROJECT} -a ${AWS_SSO_PROFILE_NAME}" diff --git a/aws_quickstart/datadog_integration_api_call.yaml b/aws_quickstart/datadog_integration_api_call.yaml deleted file mode 100644 index b689864..0000000 --- a/aws_quickstart/datadog_integration_api_call.yaml +++ /dev/null @@ -1,221 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Integration API Call (qs-1sfrgof99) -Parameters: - DatadogApiKey: - Description: >- - API key for the Datadog account - Type: String - NoEcho: true - Default: "" - DatadogAppKey: - Description: >- - APP key for the Datadog account - Type: String - NoEcho: true - Default: "" - DatadogSite: - Type: String - Default: datadoghq.com - Description: Define your Datadog Site to send data to. - AllowedValues: - - datadoghq.com - - datadoghq.eu - - us3.datadoghq.com - - us5.datadoghq.com - - ddog-gov.com - RoleName: - Description: >- - The name of the IAM role created for Datadog's use. - Type: String - Products: - Type: CommaDelimitedList - Description: >- - A comma separated list of relevant Datadog products you want to use with this account. - Chose from: Infrastructure Monitoring, Serverless, Log Management, Cloud Security Posture Management - Default: "Infrastructure Monitoring,Serverless,Log Management" -Resources: - LambdaExecutionRoleDatadogAPICall: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - ManagedPolicyArns: - - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - # Retrieving secrets passed in via SecretsManager Arn - DatadogAPICall: - Type: "Custom::DatadogAPICall" - Properties: - ServiceToken: !GetAtt DatadogAPICallFunction.Arn - APIKey: !Ref DatadogApiKey - APPKey: !Ref DatadogAppKey - ApiURL: !Ref DatadogSite - AccountId: !Ref AWS::AccountId - RoleName: !Ref RoleName - HostTags: [!Sub "aws_account:${AWS::AccountId}"] - Products: !Ref Products - DatadogAPICallFunction: - Type: "AWS::Lambda::Function" - Properties: - Description: "A function to call the Datadog API." - Role: !GetAtt LambdaExecutionRoleDatadogAPICall.Arn - Handler: "index.handler" - Runtime: "python3.11" - Timeout: 30 - Code: - ZipFile: | - import boto3 - - import json - import logging - import signal - from urllib.request import build_opener, HTTPHandler, Request - import urllib.parse - - LOGGER = logging.getLogger() - LOGGER.setLevel(logging.INFO) - - def call_datadog_api(event, method): - api_key = event['ResourceProperties']['APIKey'] - app_key = event['ResourceProperties']['APPKey'] - api_url = event['ResourceProperties']['ApiURL'] - account_id = event['ResourceProperties']['AccountId'] - role_name = event['ResourceProperties']['RoleName'] - host_tags = event['ResourceProperties']['HostTags'] - products = event['ResourceProperties']['Products'] - - # Make the url Request - url = 'https://api.' + api_url + '/api/v1/integration/aws' - values = { - 'account_id': account_id, - 'role_name': role_name, - } - if method != "DELETE": - products = [p.strip() for p in products] - values["host_tags"] = host_tags - values["cspm_resource_collection_enabled"] = "Cloud Security Posture Management" in products - infra_monitoring = "Infrastructure Monitoring" in products - serverless = "Serverless Monitoring" in products - if infra_monitoring: - values["metrics_collection_enabled"] = True - if not serverless: - values["account_specific_namespace_rules"] = {"lambda": False, "xray": False} - else: - if serverless: - values["metrics_collection_enabled"] = True - values["account_specific_namespace_rules"] = { - "application_elb": False, - "ebs": False, - "ec2": False, - "elb": False, - "network_elb": False, - "ec2api": False, - "ec2spot": False, - } - else: - values["metrics_collection_enabled"] = False - - headers = { - 'DD-API-KEY': api_key, - 'DD-APPLICATION-KEY': app_key, - } - data = json.dumps(values) - data = data.encode('utf-8') # data should be bytes - request = Request(url, data=data, headers=headers) - request.add_header('Content-Type', 'application/json; charset=utf-8') - request.add_header('Content-Length', len(data)) - request.get_method = lambda: method - - # Send the url Request, store external_id - response = urllib.request.urlopen(request) - return response - - def handler(event, context): - '''Handle Lambda event from AWS''' - try: - if event['RequestType'] == 'Create': - LOGGER.info('Received Create request.') - response = call_datadog_api(event, 'POST') - if response.getcode() == 200: - json_response = json.loads(response.read().decode("utf-8")) - send_response(event, context, "SUCCESS", - { - "Message": "Datadog AWS Integration created successfully.", - "ExternalId": json_response["external_id"], - }) - else: - LOGGER.info('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Http response: {}".format(response.msg)}) - - elif event['RequestType'] == 'Update': - LOGGER.info('Received Update request.') - send_response(event, context, "SUCCESS", - {"Message": "Update not supported, no operation performed."}) - elif event['RequestType'] == 'Delete': - LOGGER.info('Received Delete request.') - response = call_datadog_api(event, 'DELETE') - - if response.getcode() == 200: - send_response(event, context, "SUCCESS", - { - "Message": "Datadog AWS Integration deleted successfully.", - }) - else: - LOGGER.info('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Http response: {}".format(response.msg)}) - - else: - LOGGER.info('Failed - received unexpected request.') - send_response(event, context, "FAILED", - {"Message": "Unexpected event received from CloudFormation"}) - except Exception as e: # pylint: disable=W0702 - LOGGER.info('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Exception during processing: {}".format(e)}) - - - def send_response(event, context, response_status, response_data): - '''Send a resource manipulation status response to CloudFormation''' - response_body = json.dumps({ - "Status": response_status, - "Reason": "See the details in CloudWatch Log Stream: " + context.log_stream_name, - "PhysicalResourceId": context.log_stream_name, - "StackId": event['StackId'], - "RequestId": event['RequestId'], - "LogicalResourceId": event['LogicalResourceId'], - "Data": response_data - }) - formatted_response = response_body.encode("utf-8") - - LOGGER.info('ResponseURL: %s', event['ResponseURL']) - LOGGER.info('ResponseBody: %s', response_body) - - opener = build_opener(HTTPHandler) - request = Request(event['ResponseURL'], data=formatted_response) - request.add_header('Content-Type', 'application/json; charset=utf-8') - request.add_header('Content-Length', len(formatted_response)) - request.get_method = lambda: 'PUT' - response = opener.open(request) - LOGGER.info("Status code: %s", response.getcode()) - LOGGER.info("Status message: %s", response.msg) - - - def timeout_handler(_signal, _frame): - '''Handle SIGALRM''' - raise Exception('Time exceeded') - - - signal.signal(signal.SIGALRM, timeout_handler) -Outputs: - ExternalId: - Description: Datadog AWS Integration ExternalId - Value: !GetAtt DatadogAPICall.ExternalId diff --git a/aws_quickstart/datadog_integration_api_call_v2.yaml b/aws_quickstart/datadog_integration_api_call_v2.yaml index 6f9229f..dbede29 100644 --- a/aws_quickstart/datadog_integration_api_call_v2.yaml +++ b/aws_quickstart/datadog_integration_api_call_v2.yaml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Integration API Call +Description: Configures the Datadog AWS Integration Parameters: DatadogApiKey: Description: >- @@ -24,7 +24,7 @@ Parameters: - us5.datadoghq.com - ap1.datadoghq.com - ddog-gov.com - RoleName: + IAMRoleName: Description: >- The name of the IAM role created for Datadog's use. Type: String @@ -59,7 +59,7 @@ Parameters: industry benchmarks. More info: https://www.datadoghq.com/product/security-platform/cloud-security-posture-management/ Default: false Resources: - LambdaExecutionRoleDatadogAPICall: + DatadogAPIHandlerLambdaExecutionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: @@ -74,18 +74,17 @@ Resources: Path: "/" ManagedPolicyArns: - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - # Retrieving secrets passed in via SecretsManager Arn - DatadogAPICall: - Type: "Custom::DatadogAPICall" + DatadogAWSAccountIntegration: + Type: "Custom::DatadogAWSAccountIntegration" Properties: ServiceToken: !GetAtt DatadogAPICallFunction.Arn APIKey: !Ref DatadogApiKey APPKey: !Ref DatadogAppKey ApiURL: !Ref DatadogSite AccountId: !Ref AWS::AccountId - RoleName: !Ref RoleName + RoleName: !Ref IAMRoleName AWSPartition: !Ref AWS::Partition - HostTags: [!Sub "aws_account:${AWS::AccountId}"] + AccountTags: [!Sub "aws_account:${AWS::AccountId}"] CloudSecurityPostureManagement: !Ref CloudSecurityPostureManagement DisableMetricCollection: !Ref DisableMetricCollection DisableResourceCollection: !Ref DisableResourceCollection @@ -93,147 +92,234 @@ Resources: Type: "AWS::Lambda::Function" Properties: Description: "A function to call the Datadog API." - Role: !GetAtt LambdaExecutionRoleDatadogAPICall.Arn + Role: !GetAtt DatadogAPIHandlerLambdaExecutionRole.Arn Handler: "index.handler" LoggingConfig: ApplicationLogLevel: "INFO" LogFormat: "JSON" Runtime: "python3.11" - Timeout: 30 + Timeout: 5 Code: ZipFile: | - import boto3 - import json import logging import signal - from urllib.request import build_opener, HTTPHandler, Request + from urllib.request import Request import urllib.parse + import cfnresponse LOGGER = logging.getLogger() + LOGGER.setLevel(logging.INFO) + + API_CALL_SOURCE_HEADER_VALUE = "cfn-organizations" - API_CALL_SOURCE_HEADER_VALUE = "cfn-quick-start" - - def call_datadog_api(event, method): - api_key = event['ResourceProperties']['APIKey'] - app_key = event['ResourceProperties']['APPKey'] - api_url = event['ResourceProperties']['ApiURL'] - account_id = event['ResourceProperties']['AccountId'] - role_name = event['ResourceProperties']['RoleName'] - aws_partition = event['ResourceProperties']['AWSPartition'] - host_tags = event['ResourceProperties']['HostTags'] - cspm = event['ResourceProperties']['CloudSecurityPostureManagement'] - metrics_disabled = event['ResourceProperties']['DisableMetricCollection'] + class TimeoutError(Exception): + """Exception for timeouts""" + pass + + def call_datadog_api(uuid, event, method): + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] + role_name = event["ResourceProperties"]["RoleName"] + aws_partition = event["ResourceProperties"]["AWSPartition"] + account_tags = event["ResourceProperties"]["AccountTags"] + cspm = event["ResourceProperties"]["CloudSecurityPostureManagement"] + metrics_disabled = event["ResourceProperties"]["DisableMetricCollection"] resource_collection_disabled = event['ResourceProperties']['DisableResourceCollection'] # Make the url Request - url = 'https://api.' + api_url + '/api/v1/integration/aws' - values = { - 'account_id': account_id, - 'role_name': role_name - } - if method == "POST": - values["host_tags"] = host_tags - values["aws_partition"] = aws_partition - values["cspm_resource_collection_enabled"] = cspm == "true" - values["metrics_collection_enabled"] = metrics_disabled == "false" - values["extended_resource_collection_enabled"] = resource_collection_disabled == "false" - + url = f"https://api.{api_url}/api/v2/integration/aws/accounts" headers = { - 'DD-API-KEY': api_key, - 'DD-APPLICATION-KEY': app_key, - 'Dd-Aws-Api-Call-Source': API_CALL_SOURCE_HEADER_VALUE, + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, } - data = json.dumps(values) - data = data.encode('utf-8') # data should be bytes - request = Request(url, data=data, headers=headers) - request.add_header('Content-Type', 'application/json; charset=utf-8') - request.add_header('Content-Length', len(data)) + + if method == "PATCH" or method == "DELETE": + url = url + "/" + uuid + + if method == "DELETE": + # DELETE request has no body + request = Request(url, headers=headers) + else: + # Create the request body for POST and PATCH + values = { + "data": { + "type": "account", + "attributes": { + "aws_account_id": account_id, + "account_tags": account_tags, + "aws_partition": aws_partition, + "auth_config": {"role_name": role_name}, + "metrics_config": { + "enabled": (metrics_disabled == "false"), + }, + "resources_config": { + "cloud_security_posture_management_collection": ( + cspm == "true" + ), + "extended_resource_collection": ( + resource_collection_disabled == "false" + ) + } + } + } + } + + data = json.dumps(values) + data = data.encode("utf-8") # data should be bytes + request = Request(url, data=data, headers=headers) + request.add_header("Content-Type", "application/json; charset=utf-8") + request.add_header("Content-Length", len(data)) + + # Send the request request.get_method = lambda: method + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + # Return error response from API + response = e + return response - # Send the url Request, store external_id - response = urllib.request.urlopen(request) + def get_datadog_account(event): + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] + + # Make the url Request + url = f"https://api.{api_url}/api/v2/integration/aws/accounts?aws_account_id={account_id}" + headers = { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + request = Request(url, headers=headers) + request.get_method = lambda: "GET" + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + # Return error response from API + response = e return response + def handler(event, context): - '''Handle Lambda event from AWS''' + """Handle Lambda event from AWS""" + if event["RequestType"] == "Create": + LOGGER.info("Received Create request.") + method = "POST" + + elif event["RequestType"] == "Update": + LOGGER.info("Received Update request.") + method = "PATCH" + + elif event["RequestType"] == "Delete": + LOGGER.info("Received Delete request.") + method = "DELETE" + else: + LOGGER.info("Failed - received unexpected request.") + cfResponse = {"Message": "Received unexpected request type: {}".format(event["RequestType"])} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + responseStatus="FAILED", + responseData=cfResponse, + reason=reason, + ) + return + try: - if event['RequestType'] == 'Create': - LOGGER.info('Received Create request.') - response = call_datadog_api(event, 'POST') - if response.getcode() == 200: - json_response = json.loads(response.read().decode("utf-8")) - send_response(event, context, "SUCCESS", - { - "Message": "Datadog AWS Integration created successfully.", - "ExternalId": json_response["external_id"], - }) - else: - LOGGER.error('Failed - unexpected status code: %d', response.getcode()) - send_response(event, context, "FAILED", { - "Message": "Http response: {}".format(response.msg)}) - - elif event['RequestType'] == 'Update': - LOGGER.info('Received Update request.') - send_response(event, context, "FAILED", - {"Message": "Update not supported, no operation performed."}) - elif event['RequestType'] == 'Delete': - LOGGER.info('Received Delete request.') - response = call_datadog_api(event, 'DELETE') - - if response.getcode() == 200: - send_response(event, context, "SUCCESS", - { - "Message": "Datadog AWS Integration deleted successfully.", - }) - else: - LOGGER.error('Failed - unexpected status code: %d', response.getcode()) - send_response(event, context, "FAILED", { - "Message": "Http response: {}".format(response.msg)}) - - else: - LOGGER.error('Failed - received unexpected request: %s', event['RequestType']) - send_response(event, context, "FAILED", - {"Message": "Unexpected event received from CloudFormation"}) - except Exception as e: # pylint: disable=W0702 - LOGGER.exception('Failed - exception thrown during processing.') - send_response(event, context, "FAILED", { - "Message": "Exception during processing: {}".format(e)}) - - - def send_response(event, context, response_status, response_data): - '''Send a resource manipulation status response to CloudFormation''' - response_body = json.dumps({ - "Status": response_status, - "Reason": "See the details in CloudWatch Log Stream: " + context.log_stream_name, - "PhysicalResourceId": context.log_stream_name, - "StackId": event['StackId'], - "RequestId": event['RequestId'], - "LogicalResourceId": event['LogicalResourceId'], - "Data": response_data - }) - formatted_response = response_body.encode("utf-8") - - LOGGER.info('ResponseURL: %s', event['ResponseURL']) - LOGGER.info('ResponseBody: %s', response_body) - - opener = build_opener(HTTPHandler) - request = Request(event['ResponseURL'], data=formatted_response) - request.add_header('Content-Type', 'application/json; charset=utf-8') - request.add_header('Content-Length', len(formatted_response)) - request.get_method = lambda: 'PUT' - response = opener.open(request) - LOGGER.info("Status code: %s", response.getcode()) - LOGGER.info("Status message: %s", response.msg) + # Call Datadog API and report response back to CloudFormation + uuid = "" + if event["RequestType"] != "Create": + datadog_account_response = get_datadog_account(event) + uuid = extract_uuid_from_account_response(event, context, datadog_account_response) + if uuid is None: + return + response = call_datadog_api(uuid, event, method) + cfn_response_send_api_result(event, context, method, response) + except Exception as e: + LOGGER.info("Failed - exception thrown during processing.") + cfResponse = {"Message": "Exception during processing: {}".format(e)} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + "FAILED", + responseData=cfResponse, + reason=reason, + ) - def timeout_handler(_signal, _frame): - '''Handle SIGALRM''' - raise Exception('Time exceeded') + def extract_uuid_from_account_response(event, context, account_response): + json_response = "" + code = account_response.getcode() + data = account_response.read() + if data: + json_response = json.loads(data) + if code == 200 or code == 204: + if len(json_response["data"]) == 0: + cfn_response_send_failure(event, context, "Datadog account not found.") + return None + if len(json_response["data"]) > 1: + cfn_response_send_failure(event, context, "Datadog account not unique.") + return None + return json_response["data"][0]["id"] + cfn_response_send_failure(event, context, "Datadog API returned error: {}".format(json_response)) + return None + def cfn_response_send_api_result(event, context, method, response): + reason = None + json_response = "" + code = response.getcode() + data = response.read() + if data: + json_response = json.loads(data) + if code == 200 or code == 204: + LOGGER.info("Success - Datadog API call was successful.") + response_status = "SUCCESS" + cfResponse = {"Message": "Datadog AWS Integration {} API request was successful.".format(method)} + + # return external ID for create and update + if method == "POST" or method == "PATCH": + external_id = json_response["data"]["attributes"]["auth_config"]["external_id"] + cfResponse["ExternalId"] = external_id + cfnresponse.send( + event, + context, + responseStatus=response_status, + responseData=cfResponse, + reason=reason, + ) + return + cfn_response_send_failure(event, context, "Datadog API returned error: {}".format(json_response)) + + + def cfn_response_send_failure(event, context, message): + LOGGER.info("Failed - Datadog API call failed.") + reason = None + response_status = "FAILED" + cfResponse = {"Message": message} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + signalcontext, + responseStatus=response_status, + responseData=cfResponse, + reason=reason, + ) + + def timeout_handler(_signal, _frame): + """Handle SIGALRM""" + raise TimeoutError("Lambda function timeout exceeded - increase the timeout set in the api_call Cloudformation template.") + signal.signal(signal.SIGALRM, timeout_handler) Outputs: ExternalId: Description: Datadog AWS Integration ExternalId - Value: !GetAtt DatadogAPICall.ExternalId + Value: !GetAtt DatadogAWSAccountIntegration.ExternalId diff --git a/aws_quickstart/dd_api_call.py b/aws_quickstart/dd_api_call.py new file mode 100644 index 0000000..1953758 --- /dev/null +++ b/aws_quickstart/dd_api_call.py @@ -0,0 +1,218 @@ +import json +import logging +import signal +from urllib.request import Request +import urllib.parse +import cfnresponse + +LOGGER = logging.getLogger() +LOGGER.setLevel(logging.INFO) + +API_CALL_SOURCE_HEADER_VALUE = "cfn-organizations" + +class TimeoutError(Exception): + """Exception for timeouts""" + pass + +def call_datadog_api(uuid, event, method): + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] + role_name = event["ResourceProperties"]["RoleName"] + aws_partition = event["ResourceProperties"]["AWSPartition"] + account_tags = event["ResourceProperties"]["AccountTags"] + cspm = event["ResourceProperties"]["CloudSecurityPostureManagement"] + metrics_disabled = event["ResourceProperties"]["DisableMetricCollection"] + resource_collection_disabled = event['ResourceProperties']['DisableResourceCollection'] + + # Make the url Request + url = f"https://api.{api_url}/api/v2/integration/aws/accounts" + headers = { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + + if method == "PATCH" or method == "DELETE": + url = url + "/" + uuid + + if method == "DELETE": + # DELETE request has no body + request = Request(url, headers=headers) + else: + # Create the request body for POST and PATCH + values = { + "data": { + "type": "account", + "attributes": { + "aws_account_id": account_id, + "account_tags": account_tags, + "aws_partition": aws_partition, + "auth_config": {"role_name": role_name}, + "metrics_config": { + "enabled": (metrics_disabled == "false"), + }, + "resources_config": { + "cloud_security_posture_management_collection": ( + cspm == "true" + ), + "extended_resource_collection": ( + resource_collection_disabled == "false" + ) + } + } + } + } + + data = json.dumps(values) + data = data.encode("utf-8") # data should be bytes + request = Request(url, data=data, headers=headers) + request.add_header("Content-Type", "application/json; charset=utf-8") + request.add_header("Content-Length", len(data)) + + # Send the request + request.get_method = lambda: method + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + # Return error response from API + response = e + return response + +def get_datadog_account(event): + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] + + # Make the url Request + url = f"https://api.{api_url}/api/v2/integration/aws/accounts?aws_account_id={account_id}" + headers = { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + request = Request(url, headers=headers) + request.get_method = lambda: "GET" + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + # Return error response from API + response = e + return response + + +def handler(event, context): + """Handle Lambda event from AWS""" + if event["RequestType"] == "Create": + LOGGER.info("Received Create request.") + method = "POST" + + elif event["RequestType"] == "Update": + LOGGER.info("Received Update request.") + method = "PATCH" + + elif event["RequestType"] == "Delete": + LOGGER.info("Received Delete request.") + method = "DELETE" + else: + LOGGER.info("Failed - received unexpected request.") + cfResponse = {"Message": "Received unexpected request type: {}".format(event["RequestType"])} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + responseStatus="FAILED", + responseData=cfResponse, + reason=reason, + ) + return + + try: + # Call Datadog API and report response back to CloudFormation + uuid = "" + if event["RequestType"] != "Create": + datadog_account_response = get_datadog_account(event) + uuid = extract_uuid_from_account_response(event, context, datadog_account_response) + if uuid is None: + return + response = call_datadog_api(uuid, event, method) + cfn_response_send_api_result(event, context, method, response) + + except Exception as e: + LOGGER.info("Failed - exception thrown during processing.") + cfResponse = {"Message": "Exception during processing: {}".format(e)} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + "FAILED", + responseData=cfResponse, + reason=reason, + ) + +def extract_uuid_from_account_response(event, context, account_response): + json_response = "" + code = account_response.getcode() + data = account_response.read() + if data: + json_response = json.loads(data) + if code == 200 or code == 204: + if len(json_response["data"]) == 0: + cfn_response_send_failure(event, context, "Datadog account not found.") + return None + if len(json_response["data"]) > 1: + cfn_response_send_failure(event, context, "Datadog account not unique.") + return None + return json_response["data"][0]["id"] + cfn_response_send_failure(event, context, "Datadog API returned error: {}".format(json_response)) + return None + + +def cfn_response_send_api_result(event, context, method, response): + reason = None + json_response = "" + code = response.getcode() + data = response.read() + if data: + json_response = json.loads(data) + if code == 200 or code == 204: + LOGGER.info("Success - Datadog API call was successful.") + response_status = "SUCCESS" + cfResponse = {"Message": "Datadog AWS Integration {} API request was successful.".format(method)} + + # return external ID for create and update + if method == "POST" or method == "PATCH": + external_id = json_response["data"]["attributes"]["auth_config"]["external_id"] + cfResponse["ExternalId"] = external_id + cfnresponse.send( + event, + context, + responseStatus=response_status, + responseData=cfResponse, + reason=reason, + ) + return + cfn_response_send_failure(event, context, "Datadog API returned error: {}".format(json_response)) + + +def cfn_response_send_failure(event, context, message): + LOGGER.info("Failed - Datadog API call failed.") + reason = None + response_status = "FAILED" + cfResponse = {"Message": message} + reason = json.dumps(cfResponse) + cfnresponse.send( + event, + context, + responseStatus=response_status, + responseData=cfResponse, + reason=reason, + ) + +def timeout_handler(_signal, _frame): + """Handle SIGALRM""" + raise TimeoutError("Lambda function timeout exceeded - increase the timeout set in the api_call Cloudformation template.") + +signal.signal(signal.SIGALRM, timeout_handler) diff --git a/aws_quickstart/main.yaml b/aws_quickstart/main.yaml deleted file mode 100644 index 14e7a6d..0000000 --- a/aws_quickstart/main.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# version: -AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Integration -Parameters: - APIKey: - Description: >- - API key for the Datadog account (find at https://app.datadoghq.com/organization-settings/api-keys) - Type: String - NoEcho: true - Default: "" - APPKey: - Description: >- - APP key for the Datadog account (find at https://app.datadoghq.com/organization-settings/application-keys). - If this template was launched from the Datadog app, this key is tied to the user that launched the template, - and is a key specifically generated for this integration. - Type: String - NoEcho: true - Default: "" - DatadogSite: - Type: String - Default: datadoghq.com - Description: Define your Datadog Site to send data to. - AllowedValues: - - datadoghq.com - - datadoghq.eu - - us3.datadoghq.com - - us5.datadoghq.com - - ddog-gov.com - Products: - Type: String - Description: >- - A comma separated list of relevant Datadog products you want to use with this account. - Chose from: Infrastructure Monitoring, Serverless Monitoring, Log Management, Cloud Security Posture Management - Default: "Infrastructure Monitoring,Serverless Monitoring,Log Management" - IAMRoleName: - Description: Customize the name of IAM role for Datadog AWS integration - Type: String - Default: DatadogIntegrationRole - InstallLambdaLogForwarder: - Type: String - AllowedValues: - - true - - false - Description: >- - Determines whether the default configuration for the Datadog Lambda Log Forwarder is installed - as part of this stack. This is useful for sending logs to Datadog for use in Log Management or Cloud SIEM. - Customers who want to customize this setup to include specific custom tags, data scrubbing or redaction rules, - or send logs using AWS PrivateLink should select “no” and install this independently - (https://docs.datadoghq.com/serverless/libraries_integrations/forwarder/#installation). - Default: true -Conditions: - InstallForwarder: - Fn::Equals: - - !Ref InstallLambdaLogForwarder - - true - # A workaround for being unable to check list contains - CloudSecurityPostureManagementPermissions: - Fn::Or: - - Fn::Equals: - - Fn::Select: - - 0 - - Fn::Split: - - "," - - Fn::Sub: - - ${ProductIds},,,,, - - { ProductIds: !Join [',', !Split [",", !Ref Products]] } - - "Cloud Security Posture Management" - - Fn::Equals: - - Fn::Select: - - 1 - - Fn::Split: - - "," - - Fn::Sub: - - ${ProductIds},,,,, - - { ProductIds: !Join [',', !Split [",", !Ref Products]] } - - "Cloud Security Posture Management" - - Fn::Equals: - - Fn::Select: - - 2 - - Fn::Split: - - "," - - Fn::Sub: - - ${ProductIds},,,,, - - { ProductIds: !Join [',', !Split [",", !Ref Products]] } - - "Cloud Security Posture Management" - - Fn::Equals: - - Fn::Select: - - 3 - - Fn::Split: - - "," - - Fn::Sub: - - ${ProductIds},,,,, - - { ProductIds: !Join [',', !Split [",", !Ref Products]] } - - "Cloud Security Posture Management" -Resources: - # A Macro used to generate policies for the integration IAM role based on user inputs - DatadogAPICall: - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: "https://.s3.amazonaws.com/aws//datadog_integration_api_call.yaml" - Parameters: - DatadogApiKey: !Ref APIKey - DatadogAppKey: !Ref APPKey - DatadogSite: !Ref DatadogSite - RoleName: !Ref IAMRoleName - Products: !Ref Products - # The IAM role for Datadog integration - DatadogIntegrationRoleStack: - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: "https://.s3.amazonaws.com/aws//datadog_integration_role.yaml" - Parameters: - ExternalId: !GetAtt DatadogAPICall.Outputs.ExternalId - IAMRoleName: !Ref IAMRoleName - CloudSecurityPostureManagementPermissions: !If [CloudSecurityPostureManagementPermissions, true, false] - DdAWSAccountId: 464622532012 - # The Lambda function to ship logs from S3 and CloudWatch, custom metrics and traces from Lambda functions to Datadog - # https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring - ForwarderStack: - Type: AWS::CloudFormation::Stack - Condition: InstallForwarder - Properties: - TemplateURL: "https://datadog-cloudformation-template.s3.amazonaws.com/aws/forwarder/latest.yaml" - Parameters: - DdApiKey: !Ref APIKey - DdSite: !Ref DatadogSite -Outputs: - IAMRoleName: - Description: AWS IAM Role named to be used with the DataDog AWS Integration - Value: !Ref IAMRoleName - AccountId: - Description: AWS Account number - Value: !Ref "AWS::AccountId" - Region: - Description: AWS Region - Value: !Ref "AWS::Region" - DatadogForwarderArn: - Description: Datadog Forwarder Lambda Function ARN - Condition: InstallForwarder - Value: - Fn::GetAtt: - - ForwarderStack - - Outputs.DatadogForwarderArn - Export: - Name: - Fn::Sub: ${AWS::StackName}-DatadogForwarderArn -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Parameters: - - Products - - APIKey - - APPKey - - DatadogSite - - IAMRoleName - - InstallLambdaLogForwarder - ParameterLabels: - APIKey: - default: "DatadogApiKey *" - APPKey: - default: "DatadogAppKey *" - DatadogSite: - default: "DatadogSite *" diff --git a/aws_quickstart/main_v2.yaml b/aws_quickstart/main_v2.yaml index 0ea395f..509d8b2 100644 --- a/aws_quickstart/main_v2.yaml +++ b/aws_quickstart/main_v2.yaml @@ -10,7 +10,7 @@ Parameters: Default: "" APPKey: Description: >- - APP key for the Datadog account (find at https://app.datadoghq.com/organization-settings/application-keys). + APP key for the Datadog account (find at https://app.datadoghq.com/organization-settings/application-keys). If this template was launched from the Datadog app, this key is tied to the user that launched the template, and is a key specifically generated for this integration. Type: String @@ -37,8 +37,8 @@ Parameters: - true - false Description: >- - Determines whether the default configuration for the Datadog Lambda Log Forwarder is installed - as part of this stack. This is useful for sending logs to Datadog for use in Log Management or Cloud SIEM. + Determines whether the default configuration for the Datadog Lambda Log Forwarder is installed + as part of this stack. This is useful for sending logs to Datadog for use in Log Management or Cloud SIEM. Customers who want to customize this setup to include specific custom tags, data scrubbing or redaction rules, or send logs using AWS PrivateLink should select “no” and install this independently (https://docs.datadoghq.com/serverless/libraries_integrations/forwarder/#installation). @@ -49,8 +49,8 @@ Parameters: - true - false Description: >- - Disabling metric collection for this account will lead to a loss in visibility into your AWS services. - Disable this if you only want to collect tags or resource configuration information from this AWS account, + Disabling metric collection for this account will lead to a loss in visibility into your AWS services. + Disable this if you only want to collect tags or resource configuration information from this AWS account, and do not want to use Datadog Infrastructure Monitoring. Default: false DisableResourceCollection: @@ -68,9 +68,9 @@ Parameters: - true - false Description: >- - Add the AWS Managed SecurityAudit policy to your Datadog AWS Integration role, and enable + Add the AWS Managed SecurityAudit policy to your Datadog AWS Integration role, and enable Datadog Cloud Security Posture Management (CSPM) to start performing configuration checks across your AWS account. - Datadog CSPM is a product that automatically detects resource misconfigurations in your AWS account according to + Datadog CSPM is a product that automatically detects resource misconfigurations in your AWS account according to industry benchmarks. More info: https://www.datadoghq.com/product/security-platform/cloud-security-posture-management/ Default: false Rules: @@ -109,7 +109,7 @@ Conditions: - aws-us-gov Resources: # A Macro used to generate policies for the integration IAM role based on user inputs - DatadogAPICall: + DatadogAWSAccountIntegration: Type: AWS::CloudFormation::Stack Properties: TemplateURL: "https://.s3.amazonaws.com/aws//datadog_integration_api_call_v2.yaml" @@ -117,7 +117,7 @@ Resources: DatadogApiKey: !Ref APIKey DatadogAppKey: !Ref APPKey DatadogSite: !Ref DatadogSite - RoleName: !Ref IAMRoleName + IAMRoleName: !Ref IAMRoleName CloudSecurityPostureManagement: !Ref CloudSecurityPostureManagement DisableMetricCollection: !Ref DisableMetricCollection DisableResourceCollection: !Ref DisableResourceCollection @@ -127,7 +127,7 @@ Resources: Properties: TemplateURL: "https://.s3.amazonaws.com/aws//datadog_integration_role.yaml" Parameters: - ExternalId: !GetAtt DatadogAPICall.Outputs.ExternalId + ExternalId: !GetAtt DatadogAWSAccountIntegration.Outputs.ExternalId IAMRoleName: !Ref IAMRoleName ResourceCollectionPermissions: !If [ResourceCollectionPermissions, true, false] DdAWSAccountId: !If @@ -152,7 +152,7 @@ Resources: DdSite: !Ref DatadogSite Outputs: IAMRoleName: - Description: AWS IAM Role named to be used with the DataDog AWS Integration + Description: AWS IAM Role named to be used with the DataDog AWS Integration Value: !Ref IAMRoleName AccountId: Description: AWS Account number diff --git a/aws_quickstart/taskcat/README.md b/aws_quickstart/taskcat/README.md new file mode 100644 index 0000000..d340ba1 --- /dev/null +++ b/aws_quickstart/taskcat/README.md @@ -0,0 +1,19 @@ +# How to run taskcat tests + +## Setup +``` +export DD_API_KEY= +export DD_APP_KEY= +export AWS_SSO_PROFILE_NAME= +``` + +### Run +``` +./run-taskcat-test.sh +``` + +### Cleanup +``` +# To delete test stacks, run: +taskcat test clean aws-quickstart -a ${AWS_SSO_PROFILE_NAME} +``` diff --git a/aws_quickstart/taskcat/run-taskcat-tests.sh b/aws_quickstart/taskcat/run-taskcat-tests.sh index 7db9058..d874a7c 100755 --- a/aws_quickstart/taskcat/run-taskcat-tests.sh +++ b/aws_quickstart/taskcat/run-taskcat-tests.sh @@ -20,6 +20,11 @@ if [ -z "$DD_APP_KEY" ]; then exit 1 fi +if ! docker info > /dev/null 2>&1; then + echo "This script uses docker, and it isn't running - please start docker and try again!" + exit 1 +fi + mkdir -p ./tmp for f in ../*.yaml; do diff --git a/aws_quickstart/version.txt b/aws_quickstart/version.txt index a6c5252..46b105a 100644 --- a/aws_quickstart/version.txt +++ b/aws_quickstart/version.txt @@ -1 +1 @@ -v1.2.6 +v2.0.0