CassandraCluster.yml

AWSTemplateFormatVersion: '2010-09-09'
Description: AWS CloudFormation template to create a 3 node cassandra cluster
Parameters:
  EcsInstanceType:
    Type: String
    Description: ECS EC2 instance type
    Default: m4.large
    AllowedValues: [m4.large, m4.xlarge, m4.2xlarge, m4.4xlarge, m4.10xlarge,
      c4.large, c4.xlarge, c4.2xlarge, c4.4xlarge, c4.8xlarge, c3.large, c3.xlarge,
      c3.2xlarge, c3.4xlarge, c3.8xlarge, r3.large, r3.xlarge, r3.2xlarge, r3.4xlarge,
      r3.8xlarge, i2.xlarge, i2.2xlarge, i2.4xlarge, i2.8xlarge, g2.2xlarge, g2.8xlarge,
      d2.xlarge, d2.2xlarge, d2.4xlarge, d2.8xlarge]
    ConstraintDescription: must be a valid EC2 instance type.
  KeyName:
    Type: AWS::EC2::KeyPair::KeyName
    Description: Name of an existing EC2 KeyPair to enable SSH access to the ECS instances
  EfsStackName:
    Type: String
    Description: The EFS stack which contains the file system needed to be mounted in the container instances
    Default: ''
  EfsMountPath:
    Type: String
    Description: The path at which to create the volume mount for EFS
    Default: /mnt/efs
  HostedZoneName:
    Type: String
    Description: The domain of the hosted zone ending with a period
  HostedZoneId:
    Type: String
    Description: The id of the hosted zone in which node records will be created
  CassandraConfigS3Bucket:
    Type: String
    Description: The S3 bucket which contains cassandra configuration files
  LogRetention:
    Type: Number
    Description: Number of days to retain logs for cassandra nodes
    Default: '1' 
  CloudFormationLogGroup:
    Type: String
    Description: The name of a CloudWatch log group to send CloudFormation related logs to
    Default: 'CloudFormation'
Mappings:

    # These are the latest ECS optimized AMIs as of June 2017:
    #
    #   amzn-ami-2017.03.e-amazon-ecs-optimized
    #   ECS agent:    1.14.3
    #   Docker:       17.03.1-ce
    #   ecs-init:     1.14.3-1
    #
    # You can find the latest available on this page of our documentation:
    # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
    # (note the AMI identifier is region specific)

    AWSRegionToAMI:
        us-east-1:
            AMI: ami-d61027ad
        us-east-2:
            AMI: ami-bb8eaede
        us-west-1:
            AMI: ami-514e6431
        us-west-2:
            AMI: ami-c6f81abe
        ca-central-1:
            AMI: ami-32bb0556

Resources:
  # Security Groups
  #   - Load Balancer Security Group
  #   - ECS Container Instance Security Group

  InstanceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: ECS Allowed Ports
      VpcId: !ImportValue
        'Fn::Sub': '${EfsStackName}-VpcId'
      SecurityGroupIngress:
      - IpProtocol: tcp
        FromPort: '7000'
        ToPort: '7001' 
        CidrIp: '10.0.0.0/16'  
      - IpProtocol: tcp
        FromPort: '9042'
        ToPort: '9042' 
        CidrIp: '10.0.0.0/16'  
      - IpProtocol: tcp
        FromPort: '7199'
        ToPort: '7199' 
        CidrIp: '10.0.0.0/16'
      - IpProtocol: tcp
        FromPort: '9160'
        ToPort: '9160' 
        CidrIp: '10.0.0.0/16'
      - IpProtocol: tcp
        FromPort: '22'
        ToPort: '22'
        CidrIp: '10.0.0.0/16'
      Tags:
      - Key: Name
        Value: !Sub 'Cassandra Cluster SG - ${AWS::StackName}'
        
  InstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      Policies:
      - PolicyName: EC2-CloudWatchLogs
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - logs:DescribeLogStreams
            Resource:
            - arn:aws:logs:*:*:*
      - PolicyName: EC2-Operations
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - ec2:DescribeTags
            - ec2:AssociateAddress
            - ec2:DisassociateAddress
            Resource: '*'
      - PolicyName: S3-Operations
        PolicyDocument:
          Version: '2012-10-17'
          Statement:  
          - Effect: Allow
            Action:
            - s3:GetObject
            Resource: !Sub 'arn:aws:s3:::${CassandraConfigS3Bucket}/${AWS::StackName}/*'
          - Effect: Allow
            Action:
            - s3:ListBucket
            Resource: !Sub 'arn:aws:s3:::${CassandraConfigS3Bucket}'
      - PolicyName: Route53-Operations
        PolicyDocument:
          Version: '2012-10-17'
          Statement:  
          - Effect: Allow
            Action:
            - route53:ChangeResourceRecordSets
            - route53:GetHostedZone
            - route53:ListResourceRecordSets
            Resource: !Sub 'arn:aws:route53:::hostedzone/${HostedZoneId}'
          - Effect: Allow
            Action:
            - route53:ListHostedZones
            Resource: '*'
          - Effect: Allow
            Action:
            - route53:GetChange
            Resource: 'arn:aws:route53:::change/*'
  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
      - !Ref 'InstanceRole'
      
  RecordA:
    Type: AWS::Route53::RecordSet
    Properties: 
      HostedZoneName: !Ref 'HostedZoneName'
      Name: !Sub 'nodea.cassandra.${HostedZoneName}'
      Type: A
      TTL: '60'
      ResourceRecords:
      - '127.0.0.1'
  RecordB:
    Type: AWS::Route53::RecordSet
    Properties: 
      HostedZoneName: !Ref 'HostedZoneName'
      Name: !Sub 'nodeb.cassandra.${HostedZoneName}'
      Type: A
      TTL: '60'
      ResourceRecords:
      - '127.0.0.1'       
  RecordC:
    Type: AWS::Route53::RecordSet
    Properties: 
      HostedZoneName: !Ref 'HostedZoneName'
      Name: !Sub 'nodec.cassandra.${HostedZoneName}'
      Type: A
      TTL: '60'
      ResourceRecords:
      - '127.0.0.1'       
  InstanceLc:
    Type: AWS::AutoScaling::LaunchConfiguration
    DependsOn: LogGroup
    Properties:
      ImageId: !FindInMap [AWSRegionToAMI, !Ref "AWS::Region", AMI]
      InstanceType: !Ref 'EcsInstanceType'
      AssociatePublicIpAddress: false
      IamInstanceProfile: !Ref 'InstanceProfile'
      KeyName: !Ref 'KeyName'
      SecurityGroups:
        - !Ref 'InstanceSecurityGroup'
        - !ImportValue
          'Fn::Sub': '${EfsStackName}-FSAccessSecurityGroupID'

      # 1. Ensure we have the latest version of cf helper scripts
      # 2. Run all init configs in the meta data section of this resource
      # 3. Once we are done running all our scripts and they successfully ran, signal success or failure to the ASG
      UserData:
        "Fn::Base64": !Sub |
          #!/bin/bash
          yum install -y aws-cfn-bootstrap awslogs jq
          export AWS_DEFAULT_REGION=$(curl http://169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/.$//')
          /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource InstanceLc --configsets full_install
          status=$?
          source /etc/cassandra-info
          /opt/aws/bin/cfn-signal -e $status --region ${AWS::Region} --stack ${AWS::StackName} --resource "InstanceAsg$NODE_ID"
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          full_install:
          - install_cfn
          - install_logging
          - configure_docker
          - install_efs
          - configure_instance
          - configure_cassandra
        install_cfn:
          files:
            /etc/cfn/cfn-hup.conf:
              content: !Sub |
                [main]
                stack=${AWS::StackId}
                region=${AWS::Region}
              mode: '000400'
              owner: root
              group: root
            /etc/cfn/hooks.d/cfn-auto-reloader.conf:
              content: !Sub |
                [cfn-auto-reloader-hook]
                triggers=post.update
                path=Resources.InstanceLc.Metadata.AWS::CloudFormation::Init
                action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource InstanceLc --configsets full_install
                runas=root
          services:
            sysvinit:
              cfn-hup:
                enabled: 'true'
                ensureRunning: 'true'
                files:
                - /etc/cfn/cfn-hup.conf
                - /etc/cfn/hooks.d/cfn-auto-reloader.conf
        install_logging:
          files:
            "/etc/awslogs/awslogs.conf":
              mode: '000644'
              owner: root
              group: root
              content: !Sub |
                [general]
                state_file = /var/lib/awslogs/agent-state

                [/var/log/dmesg]
                file = /var/log/dmesg
                log_group_name = /var/log/dmesg
                log_stream_name = ${AWS::StackName}

                [/var/log/messages]
                file = /var/log/messages
                log_group_name = /var/log/messages
                log_stream_name = ${AWS::StackName}
                datetime_format = %b %d %H:%M:%S

                [/var/log/docker]
                file = /var/log/docker
                log_group_name = /var/log/docker
                log_stream_name = ${AWS::StackName}
                datetime_format = %Y-%m-%dT%H:%M:%S.%f

                [/var/log/cloud-init.log]
                file = /var/log/cloud-init.log
                log_group_name = ${CloudFormationLogGroup}
                log_stream_name = {instance_id}/cloud-init.log
                datetime_format =

                [/var/log/cloud-init-output.log]
                file = /var/log/cloud-init-output.log
                log_group_name = ${CloudFormationLogGroup}
                log_stream_name = {instance_id}/cloud-init-output.log
                datetime_format =

                [/var/log/cfn-init.log]
                file = /var/log/cfn-init.log
                log_group_name = ${CloudFormationLogGroup}
                log_stream_name = {instance_id}/cfn-init.log
                datetime_format =

                [/var/log/cfn-init-cmd.log]
                file = /var/log/cfn-init-cmd.log
                log_group_name = ${CloudFormationLogGroup}
                log_stream_name = {instance_id}/cfn-init-cmd.log
                datetime_format =                

                [/var/log/cfn-hup.log]
                file = /var/log/cfn-hup.log
                log_group_name = ${CloudFormationLogGroup}
                log_stream_name = {instance_id}/cfn-hup.log
                datetime_format =

                [/var/log/cfn-wire.log]
                file = /var/log/cfn-wire.log
                log_group_name = ${CloudFormationLogGroup}
                log_stream_name = {instance_id}/cfn-wire.log
                datetime_format =
            "/etc/init/awslogs.conf":
              mode: '000644'
              owner: root
              group: root
              content: |
                #upstart-job
                description "Configure and start CloudWatch Logs agent on Amazon ECS container instance"
                author "Amazon Web Services"
                start on started ecs

                script
                  exec 2>>/var/log/ecs/cloudwatch-logs-start.log
                  set -x

                  until curl -s http://169.254.169.254/latest/meta-data
                  do
                      sleep 1
                  done

                  region=$(curl http://169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/.$//')
                  # Replace region with actual region
                  sed -i -e "s/region = us-east-1/region = $region/g" /etc/awslogs/awscli.conf
                  
                  service awslogs start
                  chkconfig awslogs on
                end script
        configure_docker:
          commands:
            01_docker_users:
              command: !Sub |
                #!/bin/bash
                
                # Create a docker user that has access to specific things on the host instead of running as root
                DOCKER_USER_ID=5001
                useradd -u $DOCKER_USER_ID -o -c "" docker-user
                CASSANDRA_USER_ID=999
                useradd -u $CASSANDRA_USER_ID -o -c "" cassandra
        install_efs:
          commands:
            01_install_efs:
              command: !Sub
                - |
                  #!/bin/bash
                  set -e
                  
                  LOG_FILE=/var/log/efs.log
                  
                  # include EFS config
                  EFS_FILE_SYSTEM_ID=${FileSystemId}
                  EFS_MOUNT_POINT=${MountPath}
                  
                  #Install NFS client
                  if ! rpm -qa | grep -qw nfs-utils; then
                      yum -y install nfs-utils
                      echo "Installed nfs-utils" >> $LOG_FILE
                  fi
                  
                  EC2_AVAIL_ZONE=`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone`
                  EC2_REGION="`echo \"$EC2_AVAIL_ZONE\" | sed -e 's:\([0-9][0-9]*\)[a-z]*\$:\\1:'`"
                  
                  mkdir -p $EFS_MOUNT_POINT
                  
                  DIR_SRC=$EC2_AVAIL_ZONE.$EFS_FILE_SYSTEM_ID.efs.$EC2_REGION.amazonaws.com
                  DIR_TGT=$EFS_MOUNT_POINT
                  
                  echo "$DIR_SRC:/ $DIR_TGT nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab
                  echo "Added mount entry to fstab:" >> $LOG_FILE
                  tail -n 1 /etc/fstab >> $LOG_FILE
                  
                  mount -a -t nfs4
                  echo "Mounted EFS: $EFS_FILE_SYSTEM_ID" >> $LOG_FILE
                  
                  # Create the EFS group and assign it as the group for the entire EFS volume
                  EFS_GID=555
                  groupadd -g $EFS_GID efs
                  
                  if [ ! -d "$DIR_TGT/cassandra" ]; then
                    echo "Creating cassandra dir on EFS" >> $LOG_FILE
                    mkdir "$DIR_TGT/cassandra"
                    chown -R cassandra:cassandra "$DIR_TGT/cassandra"
                  fi
                  
                - FileSystemId: !ImportValue
                    'Fn::Sub': '${EfsStackName}-FileSystemId'
                  MountPath: !Ref 'EfsMountPath'
            02_restart_docker:
              command: service docker restart
        configure_instance:
          files:
            "/opt/change-record-set-request.json":
              mode: '000644'
              owner: root
              group: root
              content: |
                {
                  "Comment": "Changing IP address",
                  "Changes": [
                    {
                      "Action": "UPSERT",
                      "ResourceRecordSet": {}
                    }
                  ]
                }        
          commands:
            01_get_config:
              command: !Sub |
                #!/bin/bash
                set -e
                
                instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)
                tags=$(aws ec2 describe-tags --filters "Name=resource-id,Values=$instance_id")
                
                SEED_IP=$(echo $tags | jq --raw-output '.Tags[] | select(.Key == "SeedIp") | .Value')
                NODE_ID=$(echo $tags | jq --raw-output '.Tags[] | select(.Key == "NodeId") | .Value')
                
                echo "SEED_IP=$SEED_IP" >> /etc/cassandra-info
                echo "NODE_ID=$NODE_ID" >> /etc/cassandra-info
                echo "INSTANCE_ID=$instance_id" >> /etc/cassandra-info
                echo "HOSTED_ZONE_ID=${HostedZoneId}" >> /etc/cassandra-info
                echo "NODE_ADDRESS=node$NODE_ID.cassandra.${HostedZoneName}" >> /etc/cassandra-info
            03_change_records:
              command: !Sub |
                #!/bin/bash
                
                source /etc/cassandra-info
                
                private_ip=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
                
                record=$(aws route53 list-resource-record-sets --hosted-zone-id $HOSTED_ZONE_ID --query "ResourceRecordSets[?Name=='$NODE_ADDRESS']" | jq '.[0]')
                newrecord=$(echo $record | jq ".ResourceRecords[0].Value = \"$private_ip\"")
                
                echo "Updating recordset to $private_ip" >> /var/log/setup.log
                cat /opt/change-record-set-request.json | jq ".Changes[0].ResourceRecordSet = $newrecord" > /opt/record.json
                result=$(aws route53 change-resource-record-sets --hosted-zone-id $HOSTED_ZONE_ID --change-batch file:///opt/record.json)
                
                echo "Waiting for recordset update" >> /var/log/setup.log
                changeid=$(echo $result | jq '.ChangeInfo.Id' --raw-output)
                aws route53 wait resource-record-sets-changed --id "$changeid"
                
                echo "Recordset update completed" >> /var/log/setup.log
        configure_cassandra:
          files:
            "/opt/cassandra-setup":
              mode: '000755'
              owner: root
              group: root
              content: !Sub |
                #!/bin/bash
                set -e
                
                # system optimizations
                sysctl -w vm.max_map_count=1048575
                
                source /etc/cassandra-info
                mount_path=/mnt/efs
                cassandra_folder=$mount_path/cassandra/${AWS::StackName}
                
                if [ ! -d "$cassandra_folder" ]; then
                  mkdir -p $cassandra_folder
                  chown -R cassandra:cassandra $cassandra_folder
                fi
                
                node_folder=$cassandra_folder/node$NODE_ID
                
                # retrieve cassandra config files
                mkdir -p /etc/cassandra
                chmod 755 /etc/cassandra
                echo "Downloading Cassandra configuration files from S3" >> /var/log/setup.log
                aws s3 sync s3://${CassandraConfigS3Bucket}/${AWS::StackName} /etc/cassandra --sse
                echo "Finished downloading Cassandra configuration files from S3" >> /var/log/setup.log
                chown -R cassandra:cassandra /etc/cassandra
                chmod -R 644 /etc/cassandra/*
                
                if [ -d "/etc/cassandra/conf/certs" ]; then
                  chmod 744 /etc/cassandra/conf
                  chmod -R 744 /etc/cassandra/conf/certs
                fi
                
                # create cassandra node data folder if it doesn't exist
                if [ ! -d "$node_folder" ]; then
                 mkdir -p $node_folder
                 chown cassandra:cassandra $node_folder
                fi
                
                # launch cassandra
                docker run -d --privileged --name cassandra --restart always --network=host -u $(id -u cassandra) \
                --log-driver=awslogs --log-opt awslogs-region="${AWS::Region}" --log-opt awslogs-group="${LogGroup}" --log-opt awslogs-stream="node$NODE_ID" \
                -e CASSANDRA_SEEDS="$SEED_IP" \
                -e CASSANDRA_BROADCAST_ADDRESS="$NODE_ADDRESS" \
                -e CASSANDRA_ENDPOINT_SNITCH="org.apache.cassandra.locator.Ec2Snitch" \
                -v $node_folder:/var/lib/cassandra \
                -v /etc/cassandra:/etc/cassandra cassandra:3.11 >> /var/log/docker                    
          commands:
            01_configure_cassandra:
              command: !Sub |
                #!/bin/bash
                
                /opt/cassandra-setup                                                                                                                  
  InstanceAsga:
    DependsOn: RecordA
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      VPCZoneIdentifier: 
        - !ImportValue
          'Fn::Sub': '${EfsStackName}-SubnetIdA'  
      LaunchConfigurationName: !Ref 'InstanceLc'
      MinSize: 1
      MaxSize: 1
      DesiredCapacity: 1      
      Tags:
      - Key: Name
        Value: !Sub 'Cassandra Cluster Instance - ${AWS::StackName}'
        PropagateAtLaunch: 'true'
      - Key: SeedIp
        Value: !Ref 'RecordA'
        PropagateAtLaunch: 'true'
      - Key: NodeId
        Value: 'a'
        PropagateAtLaunch: 'true'
    CreationPolicy:
      # Ensure we wait for a signal from each instance to know when its done bootstrapping
      ResourceSignal:
        Timeout: PT15M
  InstanceAsgb:
    DependsOn: RecordB
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      VPCZoneIdentifier: 
        - !ImportValue
          'Fn::Sub': '${EfsStackName}-SubnetIdB'  
      LaunchConfigurationName: !Ref 'InstanceLc'
      MinSize: 1
      MaxSize: 1
      DesiredCapacity: 1      
      Tags:
      - Key: Name
        Value: !Sub 'Cassandra Cluster Instance - ${AWS::StackName}'
        PropagateAtLaunch: 'true'
      - Key: SeedIp
        Value: !Ref 'RecordA'
        PropagateAtLaunch: 'true'
      - Key: NodeId
        Value: 'b'
        PropagateAtLaunch: 'true'
    CreationPolicy:
        # Ensure we wait for a signal from each instance to know when its done bootstrapping
        ResourceSignal:
            Timeout: PT15M
  InstanceAsgc:
    DependsOn: RecordC
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      VPCZoneIdentifier: 
        - !ImportValue
          'Fn::Sub': '${EfsStackName}-SubnetIdC'  
      LaunchConfigurationName: !Ref 'InstanceLc'
      MinSize: 1
      MaxSize: 1
      DesiredCapacity: 1      
      Tags:
      - Key: Name
        Value: !Sub 'Cassandra Cluster Instance - ${AWS::StackName}'
        PropagateAtLaunch: 'true'
      - Key: SeedIp
        Value: !Ref 'RecordA'
        PropagateAtLaunch: 'true'
      - Key: NodeId
        Value: 'c'
        PropagateAtLaunch: 'true'
    CreationPolicy:
        # Ensure we wait for a signal from each instance to know when its done bootstrapping
        ResourceSignal:
          Timeout: PT15M
  LogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      RetentionInDays: !Ref 'LogRetention'
      LogGroupName: !Sub '${AWS::StackName}'    
Outputs:
  # VpcId is required by various AWS Resources, so let the dependant stack refer to the VpcId
  InstanceSecurityGroup:
    Description: The ID of the security group which ECS container instances belong to
    Value: !GetAtt [InstanceSecurityGroup, GroupId]
    Export:
      Name: !Sub '${AWS::StackName}-InstanceSecurityGroupId'
  MountPath:
    Description: The path at which the EFS volume is mounted for cassandra
    Value: !Ref 'EfsMountPath'
    Export:
      Name: !Sub '${AWS::StackName}-MountPath'