AWS deployment

.github/workflows/deploy.yml

@@ -0,0 +1,63 @@
+name: Deploy Pipeline
+on: [workflow_dispatch]
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Display AWS CLI version
+        run: aws --version
+      - name: Restore cached .venv
+        uses: actions/cache/restore@v4
+        id: cache-venv
+        with:
+          key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+          path: .venv
+      - name: Setup environment
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry config --list
+      - name: Install dependencies
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      - name: Cache .venv
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+          path: .venv
+      - name: Create .env
+        uses: SpicyPizza/create-envfile@v2.0
+        with:
+          envkey_DEV: "dev"
+          envkey_MONGO_TOKEN: ${{ secrets.MONGO_TOKEN }}
+          file_name: .env
+      - name: AWS Setup
+        run: |
+          mkdir -p ~/.aws
+          cat > ~/.aws/config << EOF
+          [profile dev]
+          role_arn = ${{ secrets.LAMBDA_ARN }}
+          source_profile = default
+          EOF
+          cat > ~/.aws/credentials << EOF
+          [default]
+          aws_access_key_id = ${{ secrets.DEV_ID }}
+          aws_secret_access_key = ${{ secrets.DEV_KEY }}
+          EOF
+      - name: Build Lambda code
+        run: poetry run python aws/build_lambda_code.py
+      - name: Deploy stack
+        shell: bash
+        run: |
+          chmod +x ./aws/deploy.sh
+          ./aws/deploy.sh ${{ secrets.MONGO_TOKEN }}
+          wait

.gitignore

@@ -4,4 +4,7 @@
 **/__pycache__
 .pytest_cache
 .ruff_cache
-junit
+junit
+aws/dependencies
+*.zip
+packaged*

aws/SETUP.md

@@ -0,0 +1,38 @@
+# AWS Guide
+
+This guide will walk you through how to utilize the CloudFormation templates used to deploy this application!
+
+## Prerequisites
+1. Download the AWS CLI following [this documentation](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+
+## First Time Setup
+1. Build the stacks corresponding to the files `roles.yml` and `billing.yml` through the AWS Cloud Console.
+2. Determine the `dev` access credentials and `lambda_arn` from the `roles.yml` stack output for the next step 
+3. Configure the `~/.aws/config` and `~/.aws/credentials` files with the necessary data. The location of these files can be determined [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where):
+
+`~/.aws/config`:
+```
+[profile dev]
+role_arn = <lambda_arn>
+source_profile = default
+```
+
+`~/.aws/credentials`:
+```
+[default]
+aws_access_key_id = <dev_id> 
+aws_secret_access_key = <dev_key>
+```
+4. Add the following secrets to GitHub: `LAMBDA_ARN`, `DEV_ID`, `DEV_KEY`
+
+**NOTE**: Upon building the `billing.yml` stack you should receive a confirmation email to verify the notification subscription
+
+## Deploying / Updating Stack
+**NOTE:** These commands must be run from the root directory
+1. Build the updated Lambda code with: `python aws/build_lambda_code.py` which will create a `aws.zip` file
+2. Deploy the code with: `./aws/deploy.sh`
+
+
+## Documentation
+- [AWS authentication and access credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-authentication-user.html)
+- Viewing Lambda code: `aws lambda get-function --function-name <FUNCTION_NAME> --query 'Code.Location'`

aws/billing.yml

@@ -0,0 +1,95 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: CloudFormation template to monitor AWS Free Tier usage, budget limits, and billing thresholds, with email notifications.
+
+Parameters:
+  budgetLimit:
+    Type: Number
+    Default: 20
+    Description: Monthly limit - will be notified when exceeding 75% and 100% of input value in USD
+  billingThreshold:
+    Type: Number
+    Default: 10
+    Description: Outstanding charges - will be notified when exceeding input value in USD
+  emailAddress:
+    Type: String
+    AllowedPattern: '[^@]+@[^@]+\.[^@]+'
+    Default: email@example.com
+  currency:
+    Type: String
+    Default: USD
+
+Resources:
+  notificationTopic:
+    Type: AWS::SNS::Topic
+    Properties:
+      TopicName: BillingAndUsageNotifications
+      Subscription:
+        - Endpoint: !Ref emailAddress
+          Protocol: email
+
+  freeTierAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmName: FreeTierExceeded
+      AlarmDescription: Triggers when AWS Free Tier limits are exceeded
+      MetricName: EstimatedCharges
+      Namespace: AWS/Billing
+      Statistic: Maximum
+      Period: 3600 # Check every hour
+      EvaluationPeriods: 1
+      Threshold: 0.01
+      ComparisonOperator: GreaterThanThreshold
+      AlarmActions:
+        - Ref: notificationTopic
+      Dimensions:
+        - Name: Currency
+          Value: !Ref currency
+
+  billingAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmName: BillingThresholdExceeded
+      AlarmDescription: Triggers when the AWS bill exceeds the specified threshold
+      MetricName: EstimatedCharges
+      Namespace: AWS/Billing
+      Statistic: Maximum
+      Period: 3600 # Check every hour
+      EvaluationPeriods: 1
+      Threshold: !Ref billingThreshold
+      ComparisonOperator: GreaterThanOrEqualToThreshold
+      AlarmActions:
+        - Ref: notificationTopic
+      Dimensions:
+        - Name: Currency
+          Value: !Ref currency
+
+  monthlyBudget:
+    Type: AWS::Budgets::Budget
+    Properties:
+      Budget:
+        BudgetName: MonthlyBudgetLimit
+        BudgetLimit:
+          Amount: !Ref budgetLimit
+          Unit: !Ref currency
+        TimeUnit: MONTHLY
+        BudgetType: COST
+        CostFilters: {}
+        CostTypes:
+          IncludeTax: true
+          IncludeSubscription: true
+          UseBlended: false
+      NotificationsWithSubscribers:
+        - Notification:
+            NotificationType: ACTUAL
+            ComparisonOperator: GREATER_THAN
+            Threshold: 75 # 75% of budget is used
+          Subscribers:
+            - SubscriptionType: EMAIL
+              Address: !Ref emailAddress
+        - Notification:
+            NotificationType: ACTUAL
+            ComparisonOperator: GREATER_THAN
+            Threshold: 100 # 100% of budget is used
+          Subscribers:
+            - SubscriptionType: EMAIL
+              Address: !Ref emailAddress

aws/build_lambda_code.py

@@ -0,0 +1,94 @@
+import sys
+
+sys.path.append(
+  "."
+)  # Adds all root directories as a package, which in our case is: imaginate_api
+import inspect
+import re
+import os
+import subprocess
+import shutil
+import zipfile
+from imaginate_api.date.routes import images_by_date
+from imaginate_api.utils import build_result, calculate_date
+from imaginate_api.schemas.date_info import DateInfo
+
+
+ENV = "dev"
+DIR = "aws"
+CWD = os.path.dirname(os.path.realpath(__file__))
+LAMBDA_LIBRARIES = """import os
+import json
+from enum import Enum
+from http import HTTPStatus
+from base64 import b64encode
+
+# External libraries from pymongo:
+from pymongo import MongoClient
+from gridfs import GridFS
+from bson.objectid import ObjectId
+"""
+LAMBDA_SETUP = f"""db_name = 'imaginate_{ENV}' # Database names: ['imaginate_dev', 'imaginate_prod']
+conn_uri = os.environ.get('MONGO_TOKEN')
+client = MongoClient(conn_uri)
+db = client[db_name]
+fs = GridFS(db)
+"""
+LAMBDA_FUNC = """def handler(event, context):
+    if event and 'queryStringParameters' in event and event['queryStringParameters'] and 'day' in event['queryStringParameters']:
+        return images_by_date(event['queryStringParameters']['day'])
+    else:
+        return {'statusCode': HTTPStatus.BAD_REQUEST, 'body': json.dumps('Invalid date')}
+"""
+LAMBDA_SUBS = {
+  "abort": "return {'statusCode': HTTPStatus.BAD_REQUEST, 'body': json.dumps('Invalid date')}",
+  "@bp.route": "",  # Remove function decorator entirely
+  "return jsonify": "return {'statusCode': HTTPStatus.OK, 'body': json.dumps(out)}",
+}
+
+
+# Meta-program our source function to substitute Flask related libraries
+def edit_source_function(source_function: str) -> str:
+  for sub in LAMBDA_SUBS:
+    source_function = re.sub(
+      r"^(\s*)" + sub + r".*$",
+      r"\g<1>" + LAMBDA_SUBS[sub],
+      source_function,
+      flags=re.MULTILINE,
+    )
+  return source_function.strip()
+
+
+if __name__ == "__main__":
+  # The main function AWS Lambda will directly invoke
+  source_function = edit_source_function(inspect.getsource(images_by_date))
+
+  # Order all the required external code needed: helper functions, classes and source function
+  all_functions = [
+    inspect.getsource(DateInfo),
+    inspect.getsource(build_result),
+    inspect.getsource(calculate_date),
+    source_function,
+  ]
+
+  # Save our Lambda function code
+  with open(f"{DIR}/index.py", "w") as f:
+    f.write(LAMBDA_LIBRARIES + "\n")  # Libaries defined as constants
+    f.write(LAMBDA_SETUP + "\n")
+    f.write("\n".join(all_functions) + "\n\n")  # Functions retrieved from source code
+    f.write(LAMBDA_FUNC)
+
+  # Format our Lambda funtion code with ruff before packaging
+  subprocess.run("ruff format index.py", shell=True, cwd=CWD)
+
+  # Following documentation from here:
+  # https://www.mongodb.com/developer/products/atlas/awslambda-pymongo/
+  subprocess.run("mkdir dependencies", shell=True, cwd=CWD)
+  subprocess.run(
+    "pip install --upgrade --target ./dependencies pymongo", shell=True, cwd=CWD
+  )
+  shutil.make_archive(f"{DIR}/aws", "zip", f"{DIR}/dependencies")
+  zf = zipfile.ZipFile(f"{DIR}/aws.zip", "a")
+  zf.write(f"{DIR}/index.py", "index.py")
+  zf.close()
+  print(f"aws.zip successfully saved in {DIR} directory")

aws/deploy.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+# Static variables
+BUCKET_NAME="imaginate-templates-bucket"
+STACK_NAME="imaginate"
+REGION="us-east-1"
+PROFILE="dev"
+
+# Read environment variables
+source .env
+
+# Load first argument as MONGO_TOKEN if passed
+if [ "$1" ]; then
+  echo "Loading MONGO_TOKEN..."
+  MONGO_TOKEN=$1
+fi
+
+# Make sure MONGO_TOKEN is not empty
+if [ "$MONGO_TOKEN" = "" ]; then
+  echo "MONGO_TOKEN is empty"
+  exit 1
+fi
+
+# AWS build/deploy
+echo "Fetching bucket..."
+aws s3 mb s3://$BUCKET_NAME --region $REGION --profile $PROFILE
+echo "Packaging..."
+aws cloudformation package --template-file aws/deploy.yml --s3-bucket $BUCKET_NAME --output-template-file aws/packaged-deploy.yml --region $REGION --profile $PROFILE
+ls aws
+echo "Deploying..."
+aws cloudformation deploy --template-file aws/packaged-deploy.yml --stack-name $STACK_NAME --capabilities CAPABILITY_NAMED_IAM --region $REGION --profile $PROFILE --parameter-overrides mongoToken=$MONGO_TOKEN
+echo "Check status..."
+aws cloudformation describe-stacks --stack-name $STACK_NAME --region $REGION --profile $PROFILE --query "Stacks[0].StackStatus" --output text