diff --git a/README.md b/README.md
index 83f70d12..5d1b41e9 100644
--- a/README.md
+++ b/README.md
@@ -196,6 +196,6 @@ Copyright 2022 COMETA ROCKS S.L.
Portions of this software are licensed as follows:
-* All content that resides under "ee/" directory of this repository (Enterprise Edition) is licensed under the license defined in "ee/LICENSE". (Work in porgress)
+* All content that resides under "ee/" directory of this repository (Enterprise Edition) is licensed under the license defined in "ee/LICENSE". (Work in progress)
* All third party components incorporated into the cometa.rocks Software are licensed under the original license provided by the owner of the applicable component.
* Content outside of the above mentioned directories or restrictions above is available under the "AGPLv3" license as defined in `LICENSE` file.
diff --git a/backend/behave/behave_django/schedules/tasks/runBrowser.py b/backend/behave/behave_django/schedules/tasks/runBrowser.py
index 11e2832d..cda48641 100644
--- a/backend/behave/behave_django/schedules/tasks/runBrowser.py
+++ b/backend/behave/behave_django/schedules/tasks/runBrowser.py
@@ -2,6 +2,10 @@
from django.conf import settings
from django_rq import job
from rq.timeouts import JobTimeoutException
+from rq.command import send_stop_job_command
+from rq.job import Job
+from rq import get_current_job
+import django_rq
# just to import secrets
sys.path.append("/code")
@@ -23,38 +27,42 @@
@job
def run_browser(json_path, env, **kwargs):
# Start running feature with current browser
- process = subprocess.Popen(["bash", settings.RUNTEST_COMMAND_PATH, json_path], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
- try:
- # wait for the process to finish
- process.wait()
- logger.debug(f"Process Return Code: {process.returncode}")
- if process.returncode > 0:
- out, _ = process.communicate()
- out = str(out.decode('utf-8'))
- logger.error(f"Error ocurred during the feature startup ... please check the output:\n{out}")
- if 'Parser failure' in out:
- raise Exception("Parsing error in the feature, please recheck the steps.")
- except JobTimeoutException as err:
- # job was timed out, kill the process
- logger.error("Job timed out.")
- logger.exception(err)
- subprocess.run(["pkill", "-TERM", "-P", "%d" % int(process.pid)])
- raise
- # TODO:
- # Check if process has been stopped
- # Send mail?
- except Exception as e:
- logger.error("run_browser threw an exception:")
- logger.exception(e)
- requests.post('http://cometa_socket:3001/feature/%s/error' % kwargs.get('feature_id', None), data={
- "browser_info": kwargs.get('browser', None),
- "feature_result_id": kwargs.get('feature_result_id', None),
- "run_id": kwargs.get('feature_run', None),
- "datetime": datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
- "error": str(e),
- "user_id": kwargs.get('user_data', {}).get('user_id', None)
- })
+ with subprocess.Popen(["bash", settings.RUNTEST_COMMAND_PATH, json_path], env=env, stdout=subprocess.PIPE) as process:
+ try:
+ logger.debug(f"Process id: {process.pid}")
+ # wait for the process to finish
+ process.wait()
+ logger.debug(f"Process Return Code: {process.returncode}")
+ if process.returncode > 0:
+ out, _ = process.communicate()
+ out = str(out.decode('utf-8'))
+ logger.error(f"Error ocurred during the feature execution ... please check the output:\n{out}")
+ if 'Parser failure' in out:
+ raise Exception("Parsing error in the feature, please recheck the steps.")
+ except JobTimeoutException as err:
+ # job was timed out, kill the process
+ logger.error("Job timed out.")
+ logger.exception(err)
+ with subprocess.Popen(f"ps -o pid --ppid {process.pid} --noheaders | xargs kill -15", shell=True) as p2:
+ p2.wait()
+ process.wait()
+ job: Job = get_current_job()
+ send_stop_job_command(django_rq.get_connection(), job.id)
+ raise
+ # TODO:
+ # Check if process has been stopped
+ # Send mail?
+ except Exception as e:
+ logger.error("run_browser threw an exception:")
+ logger.exception(e)
+ requests.post('http://cometa_socket:3001/feature/%s/error' % kwargs.get('feature_id', None), data={
+ "browser_info": kwargs.get('browser', None),
+ "feature_result_id": kwargs.get('feature_result_id', None),
+ "run_id": kwargs.get('feature_run', None),
+ "datetime": datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
+ "error": str(e),
+ "user_id": kwargs.get('user_data', {}).get('user_id', None)
+ })
@job
def run_finished(feature_run, feature_id, user_data):
diff --git a/backend/behave/behave_django/schedules/views.py b/backend/behave/behave_django/schedules/views.py
index 458481ab..3ff9a897 100755
--- a/backend/behave/behave_django/schedules/views.py
+++ b/backend/behave/behave_django/schedules/views.py
@@ -91,7 +91,8 @@ def run_test(request):
'PROXY_USER': PROXY_USER,
'VARIABLES': VARIABLES,
'PARAMETERS': PARAMETERS,
- 'department': department
+ 'department': department,
+ 'feature_id': feature_id
}
"""
os.environ['feature_run'] = str(feature_run)
@@ -212,7 +213,8 @@ def run_test(request):
feature_id=feature_id,
feature_result_id=feature_result_id,
user_data=user_data,
- feature_run=feature_run)
+ feature_run=feature_run,
+ job_timeout=7500)
jobs.append(job)
notify = run_finished.delay(feature_run, feature_id, user_data, depends_on=jobs)
diff --git a/backend/behave/cometa_itself/environment.py b/backend/behave/cometa_itself/environment.py
index f64c42ec..ca185a29 100755
--- a/backend/behave/cometa_itself/environment.py
+++ b/backend/behave/cometa_itself/environment.py
@@ -15,9 +15,10 @@
import secret_variables
from src.backend.common import *
+LOGGER_FORMAT = '\33[96m[%(asctime)s][%(feature_id)s][%(current_step)s/%(total_steps)s][%(levelname)s][%(filename)s:%(lineno)d](%(funcName)s) -\33[0m %(message)s'
# setup logging
logging.setLoggerClass(CometaLogger)
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('FeatureExecution')
logger.setLevel(BEHAVE_DEBUG_LEVEL)
# create a formatter for the logger
formatter = logging.Formatter(LOGGER_FORMAT, LOGGER_DATE_FORMAT)
@@ -37,6 +38,7 @@
# handle SIGTERM when user stops the testcase
def stopExecution(signum, frame, context):
+ logger.warn("SIGTERM Found, will stop the session")
context.aborted = True
# check if context has a variable
@@ -82,6 +84,10 @@ def decorated(*args, **kwargs):
@error_handling()
def before_all(context):
+ # Create a logger for file handler
+ fileHandle = logging.FileHandler(f"/code/src/logs/{os.environ['feature_result_id']}.log")
+ fileHandle.setFormatter(formatter)
+ logger.addHandler(fileHandle)
# handle SIGTERM signal
signal.signal(signal.SIGTERM, lambda signum, frame, ctx=context: stopExecution(signum, frame, ctx))
# create index counter for steps
@@ -311,6 +317,7 @@ def before_all(context):
# update counters total
context.counters['total'] = len(response.json()['results'])
+ os.environ['total_steps'] = str(context.counters['total'])
# send a websocket request about that feature has been started
request = requests.get('http://cometa_socket:3001/feature/%s/started' % context.feature_id, data={
@@ -319,7 +326,9 @@ def before_all(context):
"feature_result_id": os.environ['feature_result_id'],
"run_id": os.environ['feature_run'],
"datetime": datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
- })
+ })
+
+ logger.info("Processing done ... will continue with the steps.")
# Get video url with context of browser
def get_video_url(context):
@@ -329,6 +338,8 @@ def get_video_url(context):
@error_handling()
def after_all(context):
+ del os.environ['current_step']
+ del os.environ['total_steps']
# check if any alertboxes are open before quiting the browser
try:
while(context.browser.switch_to.alert):
@@ -492,8 +503,10 @@ def after_all(context):
@error_handling()
def before_step(context, step):
+ os.environ['current_step'] = str(context.counters['index'] + 1)
# complete step name to let front know about the step that will be executed next
step_name = "%s %s" % (step.keyword, step.name)
+ logger.info(f"-> {step_name}")
# step index
index = context.counters['index']
# pass all the data about the step to the step_data in context, step_data has name, screenshot, compare, enabled and type
diff --git a/backend/behave/cometa_itself/steps/actions.py b/backend/behave/cometa_itself/steps/actions.py
index a81aaa44..31aab7dd 100755
--- a/backend/behave/cometa_itself/steps/actions.py
+++ b/backend/behave/cometa_itself/steps/actions.py
@@ -75,7 +75,7 @@
ENCRYPTION_START = getattr(secret_variables, 'COMETA_ENCRYPTION_START', '')
# setup logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('FeatureExecution')
DATETIMESTRING=time.strftime("%Y%m%d-%H%M%S")
diff --git a/backend/behave/cometa_itself/steps/tools/cognos.py b/backend/behave/cometa_itself/steps/tools/cognos.py
index 716eb4f3..891f6a98 100644
--- a/backend/behave/cometa_itself/steps/tools/cognos.py
+++ b/backend/behave/cometa_itself/steps/tools/cognos.py
@@ -8,17 +8,7 @@
from src.backend.common import *
# setup logging
-logger = logging.getLogger(__name__)
-logger.setLevel(BEHAVE_DEBUG_LEVEL)
-# create a formatter for the logger
-formatter = logging.Formatter(LOGGER_FORMAT, LOGGER_DATE_FORMAT)
-# create a stream logger
-streamLogger = logging.StreamHandler()
-# set the format of streamLogger to formatter
-streamLogger.setFormatter(formatter)
-# add the stream handle to logger
-logger.addHandler(streamLogger)
-
+logger = logging.getLogger('FeatureExecution')
"""
Python library with functions used for IBM Cognos
@@ -469,7 +459,7 @@ def selectCognosPrompt_rro(context, **kwargs):
elm[0].click()
logger.debug("Clicked on option")
except:
- elm[0].selected=true
+ elm[0].selected=True
logger.debug("Setting selected to true")
# we have no value for this selector ... fallback to choosing the one with index=optionIndex
else:
diff --git a/backend/behave/cometa_itself/steps/tools/common.py b/backend/behave/cometa_itself/steps/tools/common.py
index bf300d54..52b9cdf2 100644
--- a/backend/behave/cometa_itself/steps/tools/common.py
+++ b/backend/behave/cometa_itself/steps/tools/common.py
@@ -5,6 +5,7 @@
from .variables import *
from functools import wraps
from selenium.webdriver.remote.webelement import WebElement
+from selenium.common.exceptions import InvalidSelectorException
import time, requests, json, os, datetime, sys, subprocess, re, shutil
from src.backend.common import *
from src.backend.utility.cometa_logger import CometaLogger
@@ -13,16 +14,7 @@
# setup logging
logging.setLoggerClass(CometaLogger)
-logger = logging.getLogger(__name__)
-logger.setLevel(BEHAVE_DEBUG_LEVEL)
-# create a formatter for the logger
-formatter = logging.Formatter(LOGGER_FORMAT, LOGGER_DATE_FORMAT)
-# create a stream logger
-streamLogger = logging.StreamHandler()
-# set the format of streamLogger to formatter
-streamLogger.setFormatter(formatter)
-# add the stream handle to logger
-logger.addHandler(streamLogger)
+logger = logging.getLogger('FeatureExecution')
"""
Python library with common utility functions
@@ -131,6 +123,8 @@ def waitSelector(context, selector_type, selector, max_timeout=None):
logger.exception(err)
# Max retries exceeded, raise error
raise
+ except InvalidSelectorException as err:
+ logger.debug(f"Invalid Selector Exception: Selector Type: {selec_type}, Selector: {selector}.")
except Exception as err:
# logger.error("Exception occured during the selector find, will continue looking for the element.")
# logger.exception(err)
diff --git a/backend/src/backend/middlewares/authentication.py b/backend/src/backend/middlewares/authentication.py
index d4ccecff..4f63c6f5 100644
--- a/backend/src/backend/middlewares/authentication.py
+++ b/backend/src/backend/middlewares/authentication.py
@@ -36,6 +36,8 @@ def __call__(self, request):
try:
# get the host from request
HTTP_HOST = request.META.get('HTTP_HOST', DOMAIN)
+ if HTTP_HOST == 'cometa.local':
+ raise Exception("User session none existent from behave.")
if not re.match(r'^(cometa.*\.amvara\..*)|(.*\.cometa\.rocks)$', HTTP_HOST):
HTTP_HOST = 'cometa_front'
# make a request to cometa_front to get info about the logged in user
@@ -46,7 +48,7 @@ def __call__(self, request):
})
# save user_info to self
self.user_info = response.json().get('userinfo', {})
- except Exception as error: # if executed from crontab
+ except Exception as error: # if executed from crontab or sent by behave
self.user_info = {}
# create a session variable
diff --git a/backend/src/backend/utility/cometa_logger.py b/backend/src/backend/utility/cometa_logger.py
index 26d26788..09002e81 100644
--- a/backend/src/backend/utility/cometa_logger.py
+++ b/backend/src/backend/utility/cometa_logger.py
@@ -1,4 +1,4 @@
-import logging, threading, re
+import logging, threading, re, os
class CometaLogger(logging.Logger):
@@ -22,6 +22,9 @@ def mask_values(self, msg):
msg = re.sub(rf"(?:{words_to_mask})\b", '[MASKED]', str(msg))
return msg
- def _log(self, level, msg, args, exc_info = None, extra = None, stack_info = False, stacklevel = 1):
+ def _log(self, level, msg, args, exc_info = None, extra = {}, stack_info = False, stacklevel = 1):
msg = self.mask_values(msg)
+ extra['feature_id'] = os.environ.get('feature_id', "n/a")
+ extra['current_step'] = os.environ.get('current_step', "?")
+ extra['total_steps'] = os.environ.get('total_steps', "?")
return super()._log(level, msg, args, exc_info, extra, stack_info, stacklevel)
\ No newline at end of file
diff --git a/backend/src/backend/views.py b/backend/src/backend/views.py
index a581ba2f..5b39ccef 100755
--- a/backend/src/backend/views.py
+++ b/backend/src/backend/views.py
@@ -2148,22 +2148,18 @@ def patch(self, request, *args, **kwargs):
Process schedule if requested
"""
# Set schedule of feature if provided in data, if schedule is empty will be removed
+ set_schedule = False
if 'schedule' in data:
schedule = data['schedule']
logger.debug("Saveing schedule: "+str(schedule) )
# Check if schedule is not 'now'
- if schedule != 'now':
+ if schedule != '' and schedule != 'now':
# Validate cron format before sending to Behave
if schedule != "" and not CronSlices.is_valid(schedule):
return JsonResponse({ 'success': False, "error": 'Schedule format is invalid.' }, status=200)
- # Save schedule in Behave docker Crontab
- response = set_test_schedule(feature.feature_id, schedule, request.session['user']['user_id'])
- if response.status_code != 200:
- # Oops, something went wrong while saving schedule
- logger.debug("Ooops - something went wrong saveing the schedule. You should probably check the crontab file mounted into docker to be a file and not a directory.")
- json_data = response.json()
- return JsonResponse({ 'success': False, "error": json_data.get('error', 'Something went wrong while saving schedule. Check crontab directory of docker.') }, status=200)
+ set_schedule = True
# Save schedule, at this point is 100% valid and saved
+ logger.debug("Adding schedule to database")
feature.schedule = schedule
"""
@@ -2179,6 +2175,15 @@ def patch(self, request, *args, **kwargs):
# Save without steps
result = feature.save()
+ if set_schedule:
+ # Save schedule in Behave docker Crontab
+ response = set_test_schedule(feature.feature_id, schedule, request.session['user']['user_id'])
+ if response.status_code != 200:
+ # Oops, something went wrong while saving schedule
+ logger.debug("Ooops - something went wrong saveing the schedule. You should probably check the crontab file mounted into docker to be a file and not a directory.")
+ json_data = response.json()
+ return JsonResponse({ 'success': False, "error": json_data.get('error', 'Something went wrong while saving schedule. Check crontab directory of docker.') }, status=200)
+
"""
Send WebSockets
"""
diff --git a/front/src/app/app.module.ts b/front/src/app/app.module.ts
index 35e452aa..669b5706 100755
--- a/front/src/app/app.module.ts
+++ b/front/src/app/app.module.ts
@@ -40,6 +40,7 @@ import { NgxNetworkErrorModule } from 'ngx-network-error';
/* Services */
import { ApiService } from '@services/api.service';
+import { DownloadService } from '@services/download.service';
import { PaymentsService } from '@services/payments.service';
import { SocketService } from '@services/socket.service';
import { ConfigService } from '@services/config.service';
@@ -282,6 +283,7 @@ export function getStripeApiKey() {
providers: [
ConfigService,
ApiService,
+ DownloadService,
PaymentsService,
SocketService,
ConfigService,
diff --git a/front/src/app/components/feature-run/feature-run.component.ts b/front/src/app/components/feature-run/feature-run.component.ts
index d54b8d97..b41e83cb 100644
--- a/front/src/app/components/feature-run/feature-run.component.ts
+++ b/front/src/app/components/feature-run/feature-run.component.ts
@@ -1,4 +1,4 @@
-import { Component, Input, ChangeDetectionStrategy, Optional, Host } from '@angular/core';
+import { Component, Input, ChangeDetectionStrategy, Optional, Host, OnInit } from '@angular/core';
import { Router, ActivatedRoute } from '@angular/router';
import { MatLegacyDialog as MatDialog } from '@angular/material/legacy-dialog';
import { VideoComponent } from '@dialogs/video/video.component';
diff --git a/front/src/app/components/feature-titles/feature-titles.component.html b/front/src/app/components/feature-titles/feature-titles.component.html
index a99ac0a3..7318f4db 100755
--- a/front/src/app/components/feature-titles/feature-titles.component.html
+++ b/front/src/app/components/feature-titles/feature-titles.component.html
@@ -1,5 +1,6 @@
This screen shows the results of your testruns.
Once you have more then 10 results, co.meta will show you a beautiful linechart with execution times and more.
Please execute your first feature clicking the blue run-button.
-If you just added a feature and it's executing now you will have to wait until it's finished.
-These results are reloaded automatically.
-Please execute your first feature clicking the blue run-button.
+If you just added a feature and it's executing now you will have to wait until it's finished.
+These results are reloaded automatically.
+