Merge pull request #1120 from chaos-genius/develop

Merge develop into the main for v0.10.2 release

.github/dependabot.yml

@@ -12,7 +12,7 @@ updates:
       - "pip dependencies"
     reviewers:
       - "varunp2k"
-#     open-pull-requests-limit: 10
+    open-pull-requests-limit: 10
 
   - package-ecosystem: "npm"
     directory: "/"

.github/workflows/python-test.yml

@@ -27,8 +27,8 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
+        pip install --upgrade pip
+        pip install wheel flake8 pytest
         if [ -f requirements/dev.txt ]; then pip install -r requirements/dev.txt; fi
     - name: Lint with flake8
       run: |

Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:20.04
 WORKDIR /usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y python3-pip
+    && apt-get install -y python3-pip git
 
 COPY requirements /requirements
 

chaos_genius/alerts/base_alert_digests.py

@@ -65,18 +65,29 @@ def check_and_send_digests(self):
         )
 
         for triggered_alert in triggered_alerts:
-            triggered_alert = preprocess_triggered_alert(
-                triggered_alert, self.alert_config_cache, self.kpi_cache
-            )
-
-            if getattr(
-                self.alert_config_cache[triggered_alert.alert_conf_id],
-                ALERT_ATTRIBUTES_MAPPER[self.frequency],
-            ):
-                if triggered_alert.alert_channel == "slack":
-                    slack_alerts.append(triggered_alert)
-                if triggered_alert.alert_channel == "email":
-                    email_alerts.append(triggered_alert)
+            alert = self.alert_config_cache[triggered_alert.alert_conf_id]
+            if alert.active and alert.alert_status:
+                triggered_alert = preprocess_triggered_alert(
+                    triggered_alert, self.alert_config_cache, self.kpi_cache
+                )
+
+                if getattr(
+                    self.alert_config_cache[triggered_alert.alert_conf_id],
+                    ALERT_ATTRIBUTES_MAPPER[self.frequency],
+                ):
+                    if triggered_alert.alert_channel == "slack":
+                        slack_alerts.append(triggered_alert)
+                    if triggered_alert.alert_channel == "email":
+                        email_alerts.append(triggered_alert)
+            else:
+                logger.info(
+                    (
+                        "Alert %d is inactive/deleted. "
+                        "Not including triggered alert %d in report."
+                    ),
+                    triggered_alert.alert_conf_id,
+                    triggered_alert.id,
+                )
 
         if len(email_alerts) > 0:
             report_data = AlertsReportData.from_triggered_alerts(

chaos_genius/commands.py

@@ -93,7 +93,6 @@ def integration_connector():
 
 
 @click.command()
-@with_appcontext
 @click.option('--kpi', required=True, type=int, help="Perform the anomaly detection for given KPI.")
 @click.option('--end_date', type=str, help="Perform the anomaly detection for given KPI.")
 def run_anomaly(kpi, end_date):
@@ -112,7 +111,6 @@ def run_anomaly(kpi, end_date):
 
 
 @click.command()
-@with_appcontext
 @click.option('--kpi', required=True, type=int, help="Perform Root Cause Analysis for given KPI.")
 @click.option('--end_date', type=str, help="Set end date of analysis.")
 def run_rca(kpi, end_date):
@@ -147,7 +145,6 @@ def _fetch_metadata(id: int):
 
 
 @click.command()
-@with_appcontext
 @click.option(
     "--id", required=True, type=int, help="Fetch the metadata of provided data source."
 )
@@ -161,7 +158,6 @@ def fetch_metadata(id):
 
 
 @click.command()
-@with_appcontext
 @click.option(
     "--id",
     required=True,
@@ -182,7 +178,6 @@ def run_alert(id, last_anomaly_timestamp: datetime):
     click.echo(f"Completed the alert check for ID: {id}, with status: {status}.")
 
 @click.command()
-@with_appcontext
 @click.option("-f", "--frequency", required=True, type=str, help="Trigger Alert Digest for provided frequency.")
 def run_digest(frequency):
     """Trigger alert digests"""
@@ -193,7 +188,6 @@ def run_digest(frequency):
 
 
 @click.command()
-@with_appcontext
 def run_anomaly_rca_scheduler():
     """Run the anomaly_scheduler celery task.
 
@@ -206,7 +200,6 @@ def run_anomaly_rca_scheduler():
 
 
 @click.command()
-@with_appcontext
 def reinstall_db():
     """Delete the db and reinstall again."""
     from chaos_genius.databases.demo_data import install_demo_db
@@ -224,7 +217,6 @@ def reinstall_db():
 
 
 @click.command()
-@with_appcontext
 def insert_demo_data():
     """Insert the demo data."""
     install_demo_data()

chaos_genius/connectors/__init__.py

@@ -7,6 +7,7 @@
 from chaos_genius.connectors.druid import Druid
 from chaos_genius.connectors.aws_athena import AwsAthenaDb
 from chaos_genius.connectors.databricks import Databricks
+from chaos_genius.connectors.clickhouse import ClickHouseDb
 
 
 DB_CLASS_MAPPER = {
@@ -18,6 +19,7 @@
     "Druid": Druid,
     "AWS Athena": AwsAthenaDb,
     "Databricks": Databricks,
+    "ClickHouse": ClickHouseDb
 }
 
 

chaos_genius/connectors/base_db.py

@@ -1,4 +1,5 @@
 import logging
+import re
 
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy import exc as sqlalchemy_exc
@@ -104,7 +105,7 @@ def get_schema_metadata_from_query(self, query):
         query = query.strip()
         if query[-1] == ";":
             query = query[:-1]
-        if "limit" not in query:
+        if not re.search(r"\s+limit\s+", query, re.IGNORECASE):
             query += " limit 1 "
 
         query_text = text(query)

chaos_genius/connectors/clickhouse.py

@@ -0,0 +1,76 @@
+import pandas as pd
+from sqlalchemy import create_engine
+from sqlalchemy import text
+from .base_db import BaseDb
+from .connector_utils import merge_dataframe_chunks
+
+
+class ClickHouseDb(BaseDb):
+
+    __SQL_IDENTIFIER = "`"
+
+    @property
+    def sql_identifier(self):
+        """Used to quote any SQL identifier in case of it using special characters or keywords."""
+        return self.__SQL_IDENTIFIER
+
+    db_name = "ClickHouse"
+    test_db_query = "SELECT 1"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def get_db_uri(self):
+        db_info = self.ds_info
+        host = db_info.get("host")
+        port = int(db_info.get("port"))
+        username = db_info.get("username")
+        database = db_info.get("database")
+        password = db_info.get("password")
+        if not (host and port and username and password and database):
+            raise NotImplementedError(
+                "Database Credential not found for ClickHouse."
+            )
+        self.sqlalchemy_db_uri = (
+            f"clickhouse+native://{username}:{password}@{host}:{port}/{database}"
+        )
+        return self.sqlalchemy_db_uri
+
+    def get_db_engine(self):
+        db_uri = self.get_db_uri()
+        self.engine = create_engine(db_uri, echo=self.debug)
+        return self.engine
+
+    def test_connection(self):
+        if not hasattr(self, "engine") or not self.engine:
+            self.engine = self.get_db_engine()
+        query_text = text(self.test_db_query)
+        status, message = None, ""
+        try:
+            with self.engine.connect() as connection:
+                cursor = connection.execute(query_text)
+                results = cursor.all()
+                if results[0][0] == 1:
+                    status = True
+                else:
+                    status = False
+        except Exception as err_msg:
+            status = False
+            message = str(err_msg)
+        return status, message
+
+    def run_query(self, query, as_df=True):
+        engine = self.get_db_engine()
+        if as_df == True:
+            return merge_dataframe_chunks(
+                pd.read_sql_query(query, engine, chunksize=self.CHUNKSIZE)
+            )
+        else:
+            return []
+
+    def get_schema(self):
+        self.schema = self.ds_info.get("database")
+        return self.schema
+
+    def get_schema_names_list(self):
+        return None

chaos_genius/settings.py

@@ -149,7 +149,7 @@ def _make_bool(val: Union[str, bool]) -> bool:
 TASK_CHECKPOINT_LIMIT: int = int(os.getenv("TASK_CHECKPOINT_LIMIT", 1000))
 """Number of last checkpoints to retrieve in Task Monitor"""
 
-CHAOSGENIUS_VERSION_MAIN = os.getenv("CHAOSGENIUS_VERSION_MAIN", "0.10.1")
+CHAOSGENIUS_VERSION_MAIN = os.getenv("CHAOSGENIUS_VERSION_MAIN", "0.10.2")
 """ChaosGenius version - semver part only"""
 CHAOSGENIUS_VERSION_POSTFIX = os.getenv("CHAOSGENIUS_VERSION_POSTFIX", "git")
 """ChaosGenius version - postfix to identify deployment"""

chaos_genius/third_party/integration_server_config.py

@@ -66,6 +66,7 @@
     "5B45DB62-303C-4E70-92DA-419D3CDBD506": False,  # Druid
     "F2A65364-07BA-4542-BA1B-0805E5A0572D": False,  # AWS Athena
     "ba49e8e6-eed6-11ec-8ea0-0242ac120002": False,  # Databricks
+    "15323c3e-222d-11ed-861d-0242ac120002": False  # Clickhouse
     # "29b409d9-30a5-4cc8-ad50-886eb846fea3", # Quickbooks
 }
 
@@ -125,6 +126,14 @@
         "access_token": "access_token",
         "catalog": "catalog",
         "db_type": "databricks"
+    },
+    "15323c3e-222d-11ed-861d-0242ac120002": {
+        "host": "host",
+        "port": "port",
+        "database": "database",
+        "username": "username",
+        "password": "password",
+        "db_type": "ClickHouse"
     }
 }
 

chaos_genius/utils/metadata_api_config.py

@@ -6,7 +6,8 @@
     "Snowflake": True,
     "Druid": False,
     "AWS Athena": True,
-    "Databricks": True
+    "Databricks": True,
+    "ClickHouse": False
 }
 
 TABLE_VIEW_MATERIALIZED_VIEW_AVAILABILITY = {
@@ -66,6 +67,13 @@
         "supported_aggregations": ["mean", "sum", "count"],
         "supports_multidim_dd": True
     },
+    "ClickHouse": {
+        "tables": True,
+        "views": True,
+        "materialized_views": True,
+        "supported_aggregations": ["mean", "sum", "count"],
+        "supports_multidim_dd": True
+    },
 }
 
 TABLE_VIEW_MATERIALIZED_VIEW_AVAILABILITY_THIRD_PARTY = {

docker-compose.thirdparty.yml

@@ -38,7 +38,7 @@ services:
 
   chaosgenius-server:
     container_name: chaosgenius-server
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: sh setup/run-backend-docker.sh
     restart: unless-stopped
     volumes:
@@ -97,7 +97,7 @@ services:
 
   chaosgenius-webapp:
     container_name: chaosgenius-webapp
-    image: chaosgenius/chaosgenius-webapp:0.10.1
+    image: chaosgenius/chaosgenius-webapp:0.10.2
     command: >
       sh -c "npx react-inject-env set -d ./ &&
              nginx -g 'daemon off;'"
@@ -138,7 +138,7 @@ services:
 
   chaosgenius-scheduler:
     container_name: chaosgenius-scheduler
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: celery -A run.celery beat --loglevel=DEBUG
     restart: unless-stopped
     environment:
@@ -175,7 +175,7 @@ services:
 
   chaosgenius-worker-analytics:
     container_name: chaosgenius-worker-analytics
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: celery -A run.celery worker --loglevel=INFO --concurrency=2 -P processes -Q anomaly-rca
     restart: unless-stopped
     environment:
@@ -229,7 +229,7 @@ services:
 
   chaosgenius-worker-alerts:
     container_name: chaosgenius-worker-alerts
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: celery -A run.celery worker --loglevel=INFO --concurrency=2 -P processes -Q alerts
     restart: unless-stopped
     environment:

docker-compose.yml

@@ -12,7 +12,7 @@ x-version:
 services:
   chaosgenius-server:
     container_name: chaosgenius-server
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: sh setup/run-backend-docker.sh
     restart: unless-stopped
     volumes:
@@ -70,7 +70,7 @@ services:
 
   chaosgenius-webapp:
     container_name: chaosgenius-webapp
-    image: chaosgenius/chaosgenius-webapp:0.10.1
+    image: chaosgenius/chaosgenius-webapp:0.10.2
     command: >
       sh -c "npx react-inject-env set -d ./ &&
              nginx -g 'daemon off;'"
@@ -109,7 +109,7 @@ services:
 
   chaosgenius-scheduler:
     container_name: chaosgenius-scheduler
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: celery -A run.celery beat --loglevel=DEBUG
     restart: unless-stopped
     environment:
@@ -146,7 +146,7 @@ services:
 
   chaosgenius-worker-analytics:
     container_name: chaosgenius-worker-analytics
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: celery -A run.celery worker --loglevel=INFO --concurrency=2 -P processes -Q anomaly-rca
     restart: unless-stopped
     environment:
@@ -200,7 +200,7 @@ services:
 
   chaosgenius-worker-alerts:
     container_name: chaosgenius-worker-alerts
-    image: chaosgenius/chaosgenius-server:0.10.1
+    image: chaosgenius/chaosgenius-server:0.10.2
     command: celery -A run.celery worker --loglevel=INFO --concurrency=2 -P processes -Q alerts
     restart: unless-stopped
     environment: