Merge branch 'development' into 'stage'

Development

See merge request cometa/cometa!214

README.md

@@ -196,6 +196,6 @@ Copyright 2022 COMETA ROCKS S.L.
 
 Portions of this software are licensed as follows:
 
-* All content that resides under "ee/" directory of this repository (Enterprise Edition) is licensed under the license defined in "ee/LICENSE". (Work in porgress)
+* All content that resides under "ee/" directory of this repository (Enterprise Edition) is licensed under the license defined in "ee/LICENSE". (Work in progress)
 * All third party components incorporated into the cometa.rocks Software are licensed under the original license provided by the owner of the applicable component.
 * Content outside of the above mentioned directories or restrictions above is available under the "AGPLv3" license as defined in `LICENSE` file.

backend/behave/behave_django/schedules/tasks/runBrowser.py

@@ -2,6 +2,10 @@
 from django.conf import settings
 from django_rq import job
 from rq.timeouts import JobTimeoutException
+from rq.command import send_stop_job_command
+from rq.job import Job
+from rq import get_current_job
+import django_rq
 
 # just to import secrets
 sys.path.append("/code")
@@ -23,38 +27,42 @@
 @job
 def run_browser(json_path, env, **kwargs):
     # Start running feature with current browser
-    process = subprocess.Popen(["bash", settings.RUNTEST_COMMAND_PATH, json_path], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-    try:
-        # wait for the process to finish
-        process.wait()
-        logger.debug(f"Process Return Code: {process.returncode}")
-        if process.returncode > 0:
-            out, _ = process.communicate()
-            out = str(out.decode('utf-8'))
-            logger.error(f"Error ocurred during the feature startup ... please check the output:\n{out}")
-            if 'Parser failure' in out:
-                raise Exception("Parsing error in the feature, please recheck the steps.")
-    except JobTimeoutException as err:
-        # job was timed out, kill the process
-        logger.error("Job timed out.")
-        logger.exception(err)
-        subprocess.run(["pkill", "-TERM", "-P", "%d" % int(process.pid)])
-        raise
-        # TODO:
-        # Check if process has been stopped
-        # Send mail?
-    except Exception as e:
-        logger.error("run_browser threw an exception:")
-        logger.exception(e)
-        requests.post('http://cometa_socket:3001/feature/%s/error' % kwargs.get('feature_id', None), data={
-            "browser_info": kwargs.get('browser', None),
-            "feature_result_id": kwargs.get('feature_result_id', None),
-            "run_id": kwargs.get('feature_run', None),
-            "datetime": datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
-            "error": str(e),
-            "user_id": kwargs.get('user_data', {}).get('user_id', None)
-        })
+    with subprocess.Popen(["bash", settings.RUNTEST_COMMAND_PATH, json_path], env=env, stdout=subprocess.PIPE) as process:
+        try:
+            logger.debug(f"Process id: {process.pid}")
+            # wait for the process to finish
+            process.wait()
+            logger.debug(f"Process Return Code: {process.returncode}")
+            if process.returncode > 0:
+                out, _ = process.communicate()
+                out = str(out.decode('utf-8'))
+                logger.error(f"Error ocurred during the feature execution ... please check the output:\n{out}")
+                if 'Parser failure' in out:
+                    raise Exception("Parsing error in the feature, please recheck the steps.")
+        except JobTimeoutException as err:
+            # job was timed out, kill the process
+            logger.error("Job timed out.")
+            logger.exception(err)
+            with subprocess.Popen(f"ps -o pid --ppid {process.pid} --noheaders | xargs kill -15", shell=True) as p2:
+                p2.wait() 
+            process.wait()
+            job: Job = get_current_job()
+            send_stop_job_command(django_rq.get_connection(), job.id)
+            raise
+            # TODO:
+            # Check if process has been stopped
+            # Send mail?
+        except Exception as e:
+            logger.error("run_browser threw an exception:")
+            logger.exception(e)
+            requests.post('http://cometa_socket:3001/feature/%s/error' % kwargs.get('feature_id', None), data={
+                "browser_info": kwargs.get('browser', None),
+                "feature_result_id": kwargs.get('feature_result_id', None),
+                "run_id": kwargs.get('feature_run', None),
+                "datetime": datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
+                "error": str(e),
+                "user_id": kwargs.get('user_data', {}).get('user_id', None)
+            })
 
 @job
 def run_finished(feature_run, feature_id, user_data):

backend/behave/behave_django/schedules/views.py

@@ -91,7 +91,8 @@ def run_test(request):
         'PROXY_USER': PROXY_USER,
         'VARIABLES': VARIABLES,
         'PARAMETERS': PARAMETERS,
-        'department': department
+        'department': department,
+        'feature_id': feature_id
     }
     """
     os.environ['feature_run'] = str(feature_run)
@@ -212,7 +213,8 @@ def run_test(request):
             feature_id=feature_id, 
             feature_result_id=feature_result_id, 
             user_data=user_data,
-            feature_run=feature_run)
+            feature_run=feature_run,
+            job_timeout=7500)
         jobs.append(job)
 
     notify = run_finished.delay(feature_run, feature_id, user_data, depends_on=jobs)

backend/behave/cometa_itself/environment.py

@@ -15,9 +15,10 @@
 import secret_variables
 from src.backend.common import *
 
+LOGGER_FORMAT = '\33[96m[%(asctime)s][%(feature_id)s][%(current_step)s/%(total_steps)s][%(levelname)s][%(filename)s:%(lineno)d](%(funcName)s) -\33[0m %(message)s'
 # setup logging
 logging.setLoggerClass(CometaLogger)
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('FeatureExecution')
 logger.setLevel(BEHAVE_DEBUG_LEVEL)
 # create a formatter for the logger
 formatter = logging.Formatter(LOGGER_FORMAT, LOGGER_DATE_FORMAT)
@@ -37,6 +38,7 @@
 
 # handle SIGTERM when user stops the testcase
 def stopExecution(signum, frame, context):
+    logger.warn("SIGTERM Found, will stop the session")
     context.aborted = True
 
 # check if context has a variable 
@@ -82,6 +84,10 @@ def decorated(*args, **kwargs):
 
 @error_handling()
 def before_all(context):
+    # Create a logger for file handler
+    fileHandle = logging.FileHandler(f"/code/src/logs/{os.environ['feature_result_id']}.log")
+    fileHandle.setFormatter(formatter)
+    logger.addHandler(fileHandle)
     # handle SIGTERM signal
     signal.signal(signal.SIGTERM, lambda signum, frame, ctx=context: stopExecution(signum, frame, ctx))
     # create index counter for steps
@@ -311,6 +317,7 @@ def before_all(context):
 
     # update counters total
     context.counters['total'] = len(response.json()['results'])
+    os.environ['total_steps'] = str(context.counters['total'])
 
     # send a websocket request about that feature has been started
     request = requests.get('http://cometa_socket:3001/feature/%s/started' % context.feature_id, data={
@@ -319,7 +326,9 @@ def before_all(context):
         "feature_result_id": os.environ['feature_result_id'],
         "run_id": os.environ['feature_run'],
         "datetime": datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
-    })        
+    })
+
+    logger.info("Processing done ... will continue with the steps.")
 
 # Get video url with context of browser
 def get_video_url(context):
@@ -329,6 +338,8 @@ def get_video_url(context):
 
 @error_handling()
 def after_all(context):
+    del os.environ['current_step']
+    del os.environ['total_steps']
     # check if any alertboxes are open before quiting the browser
     try:
         while(context.browser.switch_to.alert):
@@ -492,8 +503,10 @@ def after_all(context):
 
 @error_handling()
 def before_step(context, step):
+    os.environ['current_step'] = str(context.counters['index'] + 1)
     # complete step name to let front know about the step that will be executed next
     step_name = "%s %s" % (step.keyword, step.name)
+    logger.info(f"-> {step_name}")
     # step index
     index = context.counters['index']
     # pass all the data about the step to the step_data in context, step_data has name, screenshot, compare, enabled and type

backend/behave/cometa_itself/steps/actions.py

@@ -75,7 +75,7 @@
 ENCRYPTION_START = getattr(secret_variables, 'COMETA_ENCRYPTION_START', '')
 
 # setup logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('FeatureExecution')
 
 DATETIMESTRING=time.strftime("%Y%m%d-%H%M%S")
 

backend/behave/cometa_itself/steps/tools/cognos.py

@@ -8,17 +8,7 @@
 from src.backend.common import *
 
 # setup logging
-logger = logging.getLogger(__name__)
-logger.setLevel(BEHAVE_DEBUG_LEVEL)
-# create a formatter for the logger
-formatter = logging.Formatter(LOGGER_FORMAT, LOGGER_DATE_FORMAT)
-# create a stream logger
-streamLogger = logging.StreamHandler()
-# set the format of streamLogger to formatter
-streamLogger.setFormatter(formatter)
-# add the stream handle to logger
-logger.addHandler(streamLogger)
-
+logger = logging.getLogger('FeatureExecution')
 
 """
 Python library with functions used for IBM Cognos
@@ -469,7 +459,7 @@ def selectCognosPrompt_rro(context, **kwargs):
                     elm[0].click()
                     logger.debug("Clicked on option")
                 except:
-                    elm[0].selected=true
+                    elm[0].selected=True
                     logger.debug("Setting selected to true")
         # we have no value for this selector ... fallback to choosing the one with index=optionIndex
         else:

backend/behave/cometa_itself/steps/tools/common.py

@@ -5,6 +5,7 @@
 from .variables import *
 from functools import wraps
 from selenium.webdriver.remote.webelement import WebElement
+from selenium.common.exceptions import InvalidSelectorException
 import time, requests, json, os, datetime, sys, subprocess, re, shutil
 from src.backend.common import *
 from src.backend.utility.cometa_logger import CometaLogger
@@ -13,16 +14,7 @@
 
 # setup logging
 logging.setLoggerClass(CometaLogger)
-logger = logging.getLogger(__name__)
-logger.setLevel(BEHAVE_DEBUG_LEVEL)
-# create a formatter for the logger
-formatter = logging.Formatter(LOGGER_FORMAT, LOGGER_DATE_FORMAT)
-# create a stream logger
-streamLogger = logging.StreamHandler()
-# set the format of streamLogger to formatter
-streamLogger.setFormatter(formatter)
-# add the stream handle to logger
-logger.addHandler(streamLogger)
+logger = logging.getLogger('FeatureExecution')
 
 """
 Python library with common utility functions
@@ -131,6 +123,8 @@ def waitSelector(context, selector_type, selector, max_timeout=None):
                 logger.exception(err)
                 # Max retries exceeded, raise error
                 raise
+            except InvalidSelectorException as err: 
+                logger.debug(f"Invalid Selector Exception: Selector Type: {selec_type}, Selector: {selector}.")
             except Exception as err:
                 # logger.error("Exception occured during the selector find, will continue looking for the element.")
                 # logger.exception(err)

backend/src/backend/middlewares/authentication.py

@@ -36,6 +36,8 @@ def __call__(self, request):
             try:
                 # get the host from request
                 HTTP_HOST = request.META.get('HTTP_HOST', DOMAIN)
+                if HTTP_HOST == 'cometa.local':
+                    raise Exception("User session none existent from behave.")
                 if not re.match(r'^(cometa.*\.amvara\..*)|(.*\.cometa\.rocks)$', HTTP_HOST):
                     HTTP_HOST = 'cometa_front'
                 # make a request to cometa_front to get info about the logged in user
@@ -46,7 +48,7 @@ def __call__(self, request):
                 })
                 # save user_info to self
                 self.user_info = response.json().get('userinfo', {})
-            except Exception as error: # if executed from crontab
+            except Exception as error: # if executed from crontab or sent by behave
                 self.user_info = {}
 
             # create a session variable

backend/src/backend/utility/cometa_logger.py

@@ -1,4 +1,4 @@
-import logging, threading, re
+import logging, threading, re, os
 
 
 class CometaLogger(logging.Logger):
@@ -22,6 +22,9 @@ def mask_values(self, msg):
             msg = re.sub(rf"(?:{words_to_mask})\b", '[MASKED]', str(msg))
         return msg
 
-    def _log(self, level, msg, args, exc_info = None, extra = None, stack_info = False, stacklevel = 1):
+    def _log(self, level, msg, args, exc_info = None, extra = {}, stack_info = False, stacklevel = 1):
         msg = self.mask_values(msg)
+        extra['feature_id'] = os.environ.get('feature_id', "n/a")
+        extra['current_step'] = os.environ.get('current_step', "?")
+        extra['total_steps'] = os.environ.get('total_steps', "?")
         return super()._log(level, msg, args, exc_info, extra, stack_info, stacklevel)

backend/src/backend/views.py

@@ -2148,22 +2148,18 @@ def patch(self, request, *args, **kwargs):
         Process schedule if requested
         """
         # Set schedule of feature if provided in data, if schedule is empty will be removed
+        set_schedule = False
         if 'schedule' in data:
             schedule = data['schedule']
             logger.debug("Saveing schedule: "+str(schedule) )
             # Check if schedule is not 'now'
-            if schedule != 'now':
+            if schedule != '' and schedule != 'now':
                 # Validate cron format before sending to Behave
                 if schedule != "" and not CronSlices.is_valid(schedule):
                     return JsonResponse({ 'success': False, "error": 'Schedule format is invalid.' }, status=200)
-                # Save schedule in Behave docker Crontab
-                response = set_test_schedule(feature.feature_id, schedule, request.session['user']['user_id'])
-                if response.status_code != 200:
-                    # Oops, something went wrong while saving schedule
-                    logger.debug("Ooops - something went wrong saveing the schedule. You should probably check the crontab file mounted into docker to be a file and not a directory.")
-                    json_data = response.json()
-                    return JsonResponse({ 'success': False, "error": json_data.get('error', 'Something went wrong while saving schedule. Check crontab directory of docker.') }, status=200)
+                set_schedule = True
             # Save schedule, at this point is 100% valid and saved
+            logger.debug("Adding schedule to database")
             feature.schedule = schedule
 
         """
@@ -2179,6 +2175,15 @@ def patch(self, request, *args, **kwargs):
             # Save without steps
             result = feature.save()
 
+        if set_schedule:
+            # Save schedule in Behave docker Crontab
+            response = set_test_schedule(feature.feature_id, schedule, request.session['user']['user_id'])
+            if response.status_code != 200:
+                # Oops, something went wrong while saving schedule
+                logger.debug("Ooops - something went wrong saveing the schedule. You should probably check the crontab file mounted into docker to be a file and not a directory.")
+                json_data = response.json()
+                return JsonResponse({ 'success': False, "error": json_data.get('error', 'Something went wrong while saving schedule. Check crontab directory of docker.') }, status=200)
+
         """
         Send WebSockets
         """

front/src/app/app.module.ts

@@ -40,6 +40,7 @@ import { NgxNetworkErrorModule } from 'ngx-network-error';
 
 /* Services */
 import { ApiService } from '@services/api.service';
+import { DownloadService } from '@services/download.service';
 import { PaymentsService } from '@services/payments.service';
 import { SocketService } from '@services/socket.service';
 import { ConfigService } from '@services/config.service';
@@ -282,6 +283,7 @@ export function getStripeApiKey() {
   providers: [
     ConfigService,
     ApiService,
+    DownloadService,
     PaymentsService,
     SocketService,
     ConfigService,

front/src/app/components/feature-run/feature-run.component.ts

@@ -1,4 +1,4 @@
-import { Component, Input, ChangeDetectionStrategy, Optional, Host } from '@angular/core';
+import { Component, Input, ChangeDetectionStrategy, Optional, Host, OnInit } from '@angular/core';
 import { Router, ActivatedRoute } from '@angular/router';
 import { MatLegacyDialog as MatDialog } from '@angular/material/legacy-dialog';
 import { VideoComponent } from '@dialogs/video/video.component';

front/src/app/components/feature-titles/feature-titles.component.html

@@ -1,5 +1,6 @@
 <ng-container *ngIf="feature$ | async as feature">
-    <span *ngIf="feature.app_name as app"><b>App</b>: {{ app }}</span>
+    <span *ngIf="feature.department_name as dep"><b>Department</b>: {{ dep }}</span>
+    <span *ngIf="feature.app_name as app"> | <b>Application</b>: {{ app }}</span>
     <span *ngIf="feature.environment_name as env"> | <b>Environment</b>: {{ env }}</span>
     <span *ngIf="feature.feature_name as name" [matTooltip]="feature.description"> | <b>Test</b>: {{ name }}</span>
 </ng-container>

front/src/app/components/l1-feature-list/l1-feature-list.component.scss

@@ -67,6 +67,7 @@
             }
         }
     }
+
     td {
         font-weight: bold;
         color: $dark;

front/src/app/dialogs/edit-feature/edit-feature.component.html

@@ -54,7 +54,7 @@ <h2 mat-dialog-title>Clone feature</h2>
               <textarea spellcheck="false" formControlName="description" matInput rows="1"></textarea>
             </mat-form-field>
           </div>
-          <div *ngIf="featureForm.value.department_name.value === 'Default' && departments$.length > 1" class="department-warning">
+          <div *ngIf="selected_department === 'Default' && departments$.length > 1" class="department-warning">
             <mat-icon>info</mat-icon>
             <span>Selecting <strong>Default</strong> department will make this feature visible to everyone, use it with caution!</span>
           </div>

front/src/app/dialogs/video/video.component.scss

@@ -1,13 +1,6 @@
 @import 'color';
 @import 'breakpoints';
 
-:host {
-  display: block;
-  margin-bottom: -1px;
-  margin-right: -1px;
-  min-width: 300px;
-}
-
 video {
   width: 100%;
   height: auto;