diff --git a/.github/workflows/container_image.yml b/.github/workflows/container_image.yml new file mode 100644 index 0000000..f33828c --- /dev/null +++ b/.github/workflows/container_image.yml @@ -0,0 +1,75 @@ +name: Publish to GHCR.io + +on: [push] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build_tag_push_to_ghcr: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch,prefix= + type=ref,event=tag,prefix= + type=sha,format=short,prefix= + type=sha,format=long,prefix= + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + + + - name: Determine version + id: determine_version + run: | + if [ "${GITHUB_REF_TYPE}" == "tag" ]; then + VERSION=${GITHUB_REF_NAME} + else + VERSION=v0.0.0-${GITHUB_SHA::7} + fi + BUILD_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "COMMIT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV + echo "BUILD_TIMESTAMP=${BUILD_TIMESTAMP}" >> $GITHUB_ENV + + - name: Build and push Docker image + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ env.VERSION }} + COMMIT_SHA=${{ env.COMMIT_SHA }} + BUILD_TIMESTAMP=${{ env.BUILD_TIMESTAMP }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82f9275 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..41ce089 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# Use an official Python runtime as a parent image +FROM python:3.11.10-alpine@sha256:65c34f59d896f939f204e64c2f098db4a4c235be425bd8f0804fd389b1e5fd80 AS builder + +# Set working directory +WORKDIR /app + +# Copy the requirements file +COPY requirements.txt . + +# Install dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Use a smaller base image for the final image +FROM python:3.11.10-alpine@sha256:65c34f59d896f939f204e64c2f098db4a4c235be425bd8f0804fd389b1e5fd80 + +# Set working directory +WORKDIR /app + +# Copy the dependencies from the builder stage +COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +# Copy the application files +COPY app /app + +# Set environment variables +ENV PYTHONUNBUFFERED=1 + +# Accept build arguments for versioning +ARG VERSION=unknown +ARG COMMIT_SHA=unknown +ARG BUILD_TIMESTAMP=unknown + +ENV VERSION=${VERSION} +ENV COMMIT_SHA=${COMMIT_SHA} +ENV BUILD_TIMESTAMP=${BUILD_TIMESTAMP} + +# Create a non-root user and switch to it +RUN adduser -D appuser +USER appuser + +# Make port 8000 available to the world outside this container +EXPOSE 8000 + +# Use ENTRYPOINT to ensure the container runs as expected +ENTRYPOINT ["python", "-u", "main.py"] \ No newline at end of file diff --git a/README.md b/README.md index c8bbfaf..73da93a 100644 --- a/README.md +++ b/README.md @@ -1 +1,82 @@ # getoutline-docs-update-aws-organizations + +This project uses `boto3` to interact with AWS Organizations and IAM services to generate a markdown file containing information about AWS accounts and IAM users. The generated markdown includes details such as AWS Account ID, Account Name, Account Email, Created Date, SIGNIN URL, and Description for each account, as well as IAM User Name, Access Key ID, and Description for each IAM user. Once finished the Markdown will be added to our wiki hosted at getoutline.com + +## Prerequisites + +- Python 3.11 +- AWS credentials require these AWS permissions: + - `arn:aws:iam::aws:policy/AWSOrganizationsReadOnlyAccess` + - `arn:aws:iam::aws:policy/IAMReadOnlyAccess` + +## Installation + +1. Clone the repository: + ```sh + git clone https://github.com/yourusername/getoutline-docs-update-aws-organizations.git + cd getoutline-docs-update-aws-organizations + ``` + +2. Install the required Python packages: + ```sh + pip install -r requirements.txt + ``` + +## Usage + +1. Set the required environment variables: + ```sh + export GETOUTLINE_DOCUMENT_ID='your_outline_document_id' + export GETOUTLINE_API_TOKEN='your_outline_api_token' + export AWS_CREDENTIALS_JSON='your_aws_credentials_json' + ``` + + Example of `AWS_CREDENTIALS_JSON`: + ```json + { + "accounts": [ + { + "name": "org1", + "access_key": "your_access_key_id_for_org1", + "secret_key": "your_secret_access_key_for_org1" + }, + { + "name": "org2", + "access_key": "your_access_key_id_for_org2", + "secret_key": "your_secret_access_key_for_org2" + } + ] + } + ``` + +2. Run the script: + ```sh + python app/main.py + ``` + +3. The script will generate markdown and nest it under an existing AWS document within our getoutline docs. + +## Script Details + +### `main.py` + +- **get_aws_accounts(org_client)**: Retrieves a list of AWS accounts in the organization. +- **get_account_tags(org_client, account_id)**: Retrieves the tags for a given AWS account. +- **generate_signin_url(account_id)**: Generates the SIGNIN URL for a given AWS account ID. +- **list_iam_users(iam_client)**: Lists all IAM users in the root organization. +- **get_user_access_keys(iam_client, user_name)**: Retrieves the access keys for a given IAM user. +- **create_markdown(accounts, org_client)**: Generates the markdown content for AWS accounts and IAM users. + +## Example Output + +The generated markdow output will look like this: + +```md +| AWS Account ID | Account Name | Account Email | Created Date | SIGNIN URL | Description | +|----------------|--------------|---------------|--------------|------------|-------------| +| 123456789012 | ExampleName | example@domain.com | 2022-01-01 | https://123456789012.signin.aws.amazon.com/console | Example Description | + +| IAM User Name | Access Key ID | Description | +|---------------|---------------|-------------| +| example-user | AKIAIOSFODNN7EXAMPLE | Example Description | +``` \ No newline at end of file diff --git a/app/aws.py b/app/aws.py new file mode 100644 index 0000000..d207547 --- /dev/null +++ b/app/aws.py @@ -0,0 +1,160 @@ +import boto3 +import os +import json +import glueops.setup_logging + +class AWSOrganization: + def __init__(self, aws_account_name, aws_access_key_id, aws_secret_access_key, log_level="INFO"): + self.aws_account_name = aws_account_name + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + self.logger = glueops.setup_logging.configure(level=log_level) + self.logger.info(f"Logger initialized with level: {log_level}") + + def get_aws_client(self, service_name): + """ + Creates a boto3 client for the specified AWS service. + + Args: + service_name (str): The name of the AWS service. + + Returns: + boto3.client: The AWS service client. + """ + return boto3.client( + service_name, + aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key + ) + + def get_aws_accounts(self, org_client): + """ + Retrieves a list of AWS accounts in the organization. + + Args: + org_client (boto3.client): The AWS Organizations client. + + Returns: + list: A list of AWS accounts. + """ + try: + paginator = org_client.get_paginator('list_accounts') + accounts = [] + for page in paginator.paginate(): + accounts.extend(page['Accounts']) + self.logger.info(f"Retrieved {len(accounts)} accounts.") + return accounts + except Exception as e: + self.logger.error(f"Failed to retrieve AWS accounts: {e}") + raise + + def get_account_tags(self, org_client, account_id): + """ + Retrieves the tags for a given AWS account. + + Args: + org_client (boto3.client): The AWS Organizations client. + account_id (str): The AWS account ID. + + Returns: + str: The description tag value or 'No Description'. + """ + try: + response = org_client.list_tags_for_resource(ResourceId=account_id) + tags = response['Tags'] + for tag in tags: + if tag['Key'] == 'Description': + return tag['Value'] + return 'No Description' + except Exception as e: + self.logger.error(f"Failed to retrieve tags for account {account_id}: {e}") + raise + + def list_iam_users(self): + """ + Lists all IAM users in the root organization. + + Returns: + list: A list of IAM users. + """ + try: + iam_client = self.get_aws_client('iam') + paginator = iam_client.get_paginator('list_users') + users = [] + for page in paginator.paginate(): + users.extend(page['Users']) + self.logger.info(f"Retrieved {len(users)} IAM users.") + return users + except Exception as e: + self.logger.error(f"Failed to retrieve IAM users: {e}") + raise + + def get_user_access_keys(self, user_name): + """ + Retrieves the access keys for a given IAM user. + + Args: + user_name (str): The IAM user name. + + Returns: + list: A list of access key metadata. + """ + try: + iam_client = self.get_aws_client('iam') + response = iam_client.list_access_keys(UserName=user_name) + return response['AccessKeyMetadata'] + except Exception as e: + self.logger.error(f"Failed to retrieve access keys for user {user_name}: {e}") + raise + + def create_markdown(self, accounts, org_client): + """ + Creates a markdown table with details about the AWS accounts and IAM users. + + Args: + accounts (list): A list of AWS accounts. + org_client (boto3.client): The AWS Organizations client. + + Returns: + str: The markdown content. + """ + try: + # Sort accounts by created date (JoinedTimestamp) + accounts = sorted(accounts, key=lambda x: x['JoinedTimestamp']) + markdown_content = "> This page is automatically generated. Any manual changes will be lost. See: https://github.com/GlueOps/getoutline-docs-update-aws-organizations \n\n" + markdown_content += f"# AWS ROOT Organization Details for {self.aws_account_name}\n\n" + markdown_content += '| AWS Account ID | Account Name | Description | Account Email | Created Date |\n' + markdown_content += '|----------------|--------------|-------------|---------------|--------------|\n' + + for account in accounts: + account_id = account['Id'] + account_name = account['Name'] + account_email = account['Email'] + created_date = account['JoinedTimestamp'].strftime('%Y-%m-%d') + description = self.get_account_tags(org_client, account_id) + markdown_content += f'| {account_id} | {account_name} | {description} | {account_email} | {created_date} |\n' + + markdown_content += '\n\n| IAM User Name | Access Key ID | Description |\n' + markdown_content += '|---------------|---------------|-------------|\n' + users = self.list_iam_users() + for user in users: + user_name = user['UserName'] + access_keys = self.get_user_access_keys(user_name) + if not access_keys: + markdown_content += f'| {user_name} | No Access Key | No Description |\n' + else: + for access_key in access_keys: + access_key_id = access_key['AccessKeyId'] + description = 'No Description' + tags = self.get_aws_client('iam').list_user_tags(UserName=user_name)['Tags'] + for tag in tags: + if tag['Key'] == 'description': + description = tag['Value'] + markdown_content += f'| {user_name} | {access_key_id} | {description} |\n' + + self.logger.info("Markdown content created successfully.") + return markdown_content + except Exception as e: + self.logger.error(f"Failed to create markdown content: {e}") + raise + diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..f538b0e --- /dev/null +++ b/app/main.py @@ -0,0 +1,123 @@ +import boto3 +import os +import json +import requests +import glueops.setup_logging +import glueops.getoutline +from aws import AWSOrganization + +GETOUTLINE_API_URL = "https://app.getoutline.com" +REQUIRED_ENV_VARS = [ + "GETOUTLINE_DOCUMENT_ID", + "GETOUTLINE_API_TOKEN", + "AWS_CREDENTIALS_JSON" +] + +OPTIONAL_ENV_VARS = { + "VERSION": "unknown", + "COMMIT_SHA": "unknown", + "BUILD_TIMESTAMP": "unknown", +} + +def get_credentials(): + """ + Retrieve AWS credentials from the environment variable. + + :return: List of AWS account credentials. + :raises EnvironmentError: If the AWS_CREDENTIALS_JSON environment variable is not set. + :raises ValueError: If the JSON in AWS_CREDENTIALS_JSON is invalid. + """ + credentials_json = os.getenv('AWS_CREDENTIALS_JSON') + if not credentials_json: + raise EnvironmentError("AWS_CREDENTIALS_JSON environment variable not set.") + + try: + credentials = json.loads(credentials_json) + return credentials['accounts'] + except json.JSONDecodeError as e: + raise ValueError("Invalid JSON in AWS_CREDENTIALS_JSON.") + + +def get_env_variable(var_name: str, default=None): + """ + Retrieve environment variable or return default if not set. + + :param var_name: Name of the environment variable. + :param default: Default value if the environment variable is not set. + :return: Value of the environment variable or default. + :raises EnvironmentError: If a required environment variable is not set. + """ + value = os.getenv(var_name, default) + if var_name in REQUIRED_ENV_VARS and value is None: + logger.error(f"Environment variable '{var_name}' is not set.") + raise EnvironmentError(f"Environment variable '{var_name}' is required but not set.") + logger.debug(f"Environment variable '{var_name}' retrieved.") + return value + +# Configure logging +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") +logger = glueops.setup_logging.configure(level=LOG_LEVEL) +logger.info(f"Logger initialized with level: {LOG_LEVEL}") +logger.info({ + "version": os.getenv("VERSION", "unknown"), + "commit_sha": os.getenv("COMMIT_SHA", "unknown"), + "build_timestamp": os.getenv("BUILD_TIMESTAMP", "unknown") +}) + +try: + GETOUTLINE_DOCUMENT_ID = get_env_variable('GETOUTLINE_DOCUMENT_ID') + GETOUTLINE_API_TOKEN = get_env_variable('GETOUTLINE_API_TOKEN') + AWS_CREDENTIALS_JSON = get_env_variable('AWS_CREDENTIALS_JSON') + get_credentials() + VERSION = get_env_variable('VERSION', OPTIONAL_ENV_VARS['VERSION']) + COMMIT_SHA = get_env_variable('COMMIT_SHA', OPTIONAL_ENV_VARS['COMMIT_SHA']) + BUILD_TIMESTAMP = get_env_variable('BUILD_TIMESTAMP', OPTIONAL_ENV_VARS['BUILD_TIMESTAMP']) + logger.info("All required environment variables retrieved successfully.") +except EnvironmentError as env_err: + logger.critical(f"Environment setup failed: {env_err}") + raise + +def main(): + """ + Main function to execute the script. + """ + try: + logger.info("Starting script execution.") + # Initialize GetOutlineClient + GetOutlineClient = glueops.getoutline.GetOutlineClient(GETOUTLINE_API_URL, GETOUTLINE_DOCUMENT_ID, GETOUTLINE_API_TOKEN) + parent_id = GetOutlineClient.get_document_uuid() + children = GetOutlineClient.get_children_documents_to_delete(parent_id) + + # Delete existing child documents + for id in children: + GetOutlineClient.delete_document(id) + + # Retrieve AWS account credentials + accounts_creds = get_credentials() + all_accounts = [] + markdown_content = "" + + # Process each AWS account + for creds in accounts_creds: + aws_account_name = creds['name'] + access_key = creds['access_key'] + secret_key = creds['secret_key'] + + aws_org = AWSOrganization(aws_account_name, access_key, secret_key) + org_client = aws_org.get_aws_client('organizations') + accounts = aws_org.get_aws_accounts(org_client) + all_accounts.extend(accounts) + markdown_content = aws_org.create_markdown(accounts, org_client) + logger.debug(f"Markdown content: {markdown_content}") + logger.info(f"Generated Markdown for AWS ORG: {aws_account_name}") + + # Create new document in Outline + GetOutlineClient.create_document(parent_id, aws_account_name, markdown_content) + logger.info(f"Created {aws_account_name} doc successfully under parent doc: {GETOUTLINE_DOCUMENT_ID}") + logger.info("Script execution completed successfully.") + except Exception as e: + logger.error(f"Script execution failed: {e}") + raise + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9ea3e78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +boto3==1.35.66 +requests==2.32.3 +glueops-helpers @ https://github.com/GlueOps/python-glueops-helpers-library/archive/refs/tags/v0.6.0.zip \ No newline at end of file