diff --git a/.gitignore b/.gitignore index 4abbe19..6620678 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ /logs /output +/input /.idea *.ipr *.iml *.iws +*.DS_Store .git-bitbucket -__pycache__ +__pycache__ \ No newline at end of file diff --git a/README.md b/README.md index b6d4a61..8bdb11d 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,71 @@ # config-assessment-tool -This project aims to provide a single source of truth for performing AppDynamics Health Checks. +This project aims to provide a single source of truth for performing +AppDynamics Instrumentation Health Checks. These health checks provide +metrics on how well your applications are instrumented based on some of the +field best practices. ## Usage +### Prerequisite +Ensure the DefaultJob.json file correctly reflects the controller +credentials you are running the tool for. The file format is outlined +below with some default placeholder values. Change according to your setup. +You may also create your own custom named job file. + +``` +{ + "host": "acme.saas.appdynamics.com", + "port": 443, + "account": "acme", + "username": "foo", + "pwd": "hunter1", + "authType": "", # see Note ** + "verifySsl": true, + "useProxy": true, + "applicationFilter": { + "apm": ".*", + "mrum": ".*", + "brum": ".*" + }, + "timeRangeMins": 1440 +} +``` +Note ** +The tool supports both basic UI authentication or API Client authentication +(oAuth tokens) using either client secrets or temporary access tokens. The +authType field is used to specify the authentication method. See +Administration -> API Clients in controller UI for setup. + +authType: +- "basic" use controller web UI username for "username" and controller web + UI password for "pwd". This method will be deprecated in a future release. +- "token" use API Clients -> Client Name for "username", and API Clients -> + Temporary Access Token for "pwd" field. This authentication method is + preferred. Note "username" is not strictly required if use token + authentication. +- "secret" use API Clients -> Client Name for "username", and API Clients -> + Client Secret for "pwd" fields. + +If using API Client authentication, ensure the Default/Temporary Token +expiration times are sufficient to complete the run + +### How to run There are four options to run the tool: 1. [UI Method](https://github.com/Appdynamics/config-assessment-tool#ui-method) - - Run jobs from a convenient web based UI - - Easier way to configure jobs but requires Python and Docker installation + - Run jobs from a convenient web based UI + - Easier way to configure jobs but requires Python and Docker installation 2. [Platform executable](https://github.com/Appdynamics/config-assessment-tool#platform-executable). (Preferred method for most users running Windows and Linux) - - A self contained OS specific bundle if you are not using Docker and Python. Bundles available for Linux and Windows - - Recommended for users that do not wish to install Python and Docker and/or can not access external repositories + - A self contained OS specific bundle if you are not using Docker and Python. Bundles available for Linux and Windows + - Recommended for users that do not wish to install Python and Docker and/or can not access external repositories 3. [Directly via Docker](https://github.com/Appdynamics/config-assessment-tool#directly-via-docker) - - The backend container can be run manually from the command line - - Recommended for users with Docker who do not want to use the UI + - The backend container can be run manually from the command line + - Recommended for users with Docker who do not want to use the UI 4. [From Source](https://github.com/Appdynamics/config-assessment-tool#directly-via-docker) - - Manually install dependencies and run the `backend.py` script directly - - Recommended for users who want to build the tool from source + - Manually install dependencies and run the `backend.py` script directly + - Recommended for users who want to build the tool from source ### Important step for running on Windows (Ignore this step if using method 2 or 4 above) @@ -168,32 +215,32 @@ The frontend can be invoked by navigating to `config_assessment_tool/frontend` a This program will create the following files in the `out` directory. - `{jobName}-MaturityAssessment-apm.xlsx` - - MaturityAssessment report for APM + - MaturityAssessment report for APM - `{jobName}-MaturityAssessment-brum.xlsx` - - MaturityAssessment report for BRUM + - MaturityAssessment report for BRUM - `{jobName}-MaturityAssessment-mrum.xlsx` - - MaturityAssessment report for MRUM + - MaturityAssessment report for MRUM - `{jobName}-AgentMatrix.xlsx` - - Details agent versions rolled up by application - - Lists the details of individual without any rollup + - Details agent versions rolled up by application + - Lists the details of individual without any rollup - `{jobName}-CustomMetrics.xlsx` - - Lists which applications are leveraging Custom Extensions + - Lists which applications are leveraging Custom Extensions - `{jobName}-License.xlsx` - - Export of the License Usage page in the Controller + - Export of the License Usage page in the Controller - `{jobName}-MaturityAssessmentRaw-apm.xlsx` - - Raw metrics which go into MaturityAssessment for APM report + - Raw metrics which go into MaturityAssessment for APM report - `{jobName}-MaturityAssessmentRaw-brum.xlsx` - - Raw metrics which go into MaturityAssessment for BRUM report + - Raw metrics which go into MaturityAssessment for BRUM report - `{jobName}-MaturityAssessmentRaw-mrum.xlsx` - - Raw metrics which go into MaturityAssessment for MRUM report + - Raw metrics which go into MaturityAssessment for MRUM report - `{jobName}-HybridApplicationMonitoringUseCaseMaturityAssessment-presentation.pptx - Primarily used by customers that have purchased the Hybrid App Monitoring(HAM) SKU's - `{jobName}-ConfigurationAnalysisReport.xlsx` - - Configuration Analysis Report used primarily by AppD Services team, generated separately + - Configuration Analysis Report used primarily by AppD Services team, generated separately - `controllerData.json` - - Contains all raw data used in analysis. + - Contains all raw data used in analysis. - `info.json` - - Contains information on previous job execution. + - Contains information on previous job execution. ## Program Architecture @@ -213,7 +260,7 @@ There is no other communication from the tool to any other external services. We Consult the links below for the aforementioned references: - AppDynamics API's: https://docs.appdynamics.com/appd/22.x/latest/en/extend-appdynamics/appdynamics-apis#AppDynamicsAPIs-apiindex -- Platform executable bundles: https://github.com/Appdynamics/config-assessment-tool/releases +- Platform executable bundles: https://github.com/Appdynamics/config-assessment-tool/releases - Job file: https://github.com/Appdynamics/config-assessment-tool/blob/master/input/jobs/DefaultJob.json - Build from source: https://github.com/Appdynamics/config-assessment-tool#from-source - config-assessment-tool GitHub open source project: https://github.com/Appdynamics/config-assessment-tool @@ -243,36 +290,36 @@ For example: Use HTTPS_PROXY environment variable if your controller is accessib [DefaultJob.json](https://github.com/Appdynamics/config-assessment-tool/blob/master/input/jobs/DefaultJob.json) defines a number of optional configurations. - verifySsl - - enabled by default, disable it to disable SSL cert checking (equivalent to `curl -k`) + - enabled by default, disable it to disable SSL cert checking (equivalent to `curl -k`) - useProxy - - As defined above under [Proxy Support](https://github.com/Appdynamics/config-assessment-tool#proxy-support), enable this to use a configured proxy + - As defined above under [Proxy Support](https://github.com/Appdynamics/config-assessment-tool#proxy-support), enable this to use a configured proxy - applicationFilter - - Three filters are available, one for `apm`, `mrum`, and `brum` - - The filter value accepts any valid regex, set to `.*` by default - - Set the value to null to filter out all applications for the set type + - Three filters are available, one for `apm`, `mrum`, and `brum` + - The filter value accepts any valid regex, set to `.*` by default + - Set the value to null to filter out all applications for the set type - timeRangeMins - - Configure the data pull time range, by default set to 1 day (1440 mins) + - Configure the data pull time range, by default set to 1 day (1440 mins) - pwd - - Your password will be automatically encrypted to base64 when it is persisted to disk - - If your password is not entered as base64, it will be automatically converted + - Your password will be automatically encrypted to base64 when it is persisted to disk + - If your password is not entered as base64, it will be automatically converted ## Requirements -- Python 3.9 if running from source method. In addition, Docker engine is required if running using either th UI or Docker methods +- Python 3.9 if running from source method. In addition, Docker engine is required if running using either th UI or Docker methods - No Python/Docker needed if using Platform executable bundles. Linux and Windows(10/11/Server), x86 architectures are supported only. No ARM architecture support currenlty provided. ## Limitations - Data Collectors - - The API to directly find snapshots containing data collectors of type `Session Key` or `HTTP Header` does not work. - - The API does however work for `Business Data` (POJO match rule), `HTTP Parameter`, and `Cookie` types. - - As far as I can tell this is a product limitation, the transaction snapshot filtering UI does not even have an option for `Session Key` or `HTTP Header`. - - The only way to check for `Session Key` or `HTTP Header` data collector existence within snapshots would be to inspect ALL snapshots (prohibitively time intensive). - - As a workaround, we will assume any `Session Key` or `HTTP Header` data collectors are present in snapshots. + - The API to directly find snapshots containing data collectors of type `Session Key` or `HTTP Header` does not work. + - The API does however work for `Business Data` (POJO match rule), `HTTP Parameter`, and `Cookie` types. + - As far as I can tell this is a product limitation, the transaction snapshot filtering UI does not even have an option for `Session Key` or `HTTP Header`. + - The only way to check for `Session Key` or `HTTP Header` data collector existence within snapshots would be to inspect ALL snapshots (prohibitively time intensive). + - As a workaround, we will assume any `Session Key` or `HTTP Header` data collectors are present in snapshots. ## Support -For general feature requests or questions/feedback please create an issue in this Github repository. Ensure that no proprietary information is included in the issue or attachments as this is an open source project with public visibility. +For general feature requests or questions/feedback please create an issue in this Github repository. Ensure that no proprietary information is included in the issue or attachments as this is an open source project with public visibility. If you are having difficulty running the tool email Alex Afshar at aleafsha@cisco.com and attach any relevant information including debug logs. diff --git a/VERSION b/VERSION index 98610aa..0a35bd8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.6.2 \ No newline at end of file +v1.7.0-beta \ No newline at end of file diff --git a/backend/api/appd/AppDController.py b/backend/api/appd/AppDController.py index 93c0e77..a14219e 100644 --- a/backend/api/appd/AppDController.py +++ b/backend/api/appd/AppDController.py @@ -16,10 +16,23 @@ class AppdController(Consumer): jsessionid: str = None xcsrftoken: str = None + def __init__(self, *args, session=None, **kwargs): + super().__init__(*args, **kwargs) + self.client_session=session + + def get_client_session(self): + return self.client_session + @params({"action": "login"}) @get("/controller/auth") def login(self): - """Verifies Login Success""" + """Verifies Login Success (Basic Auth)""" + + @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) + @post("/controller/api/oauth/access_token") + def loginOAuth(self, data: Body): + """Method to get a token.""" @params({"output": "json"}) @get("/controller/rest/applications") @@ -67,11 +80,13 @@ def getConfigurations(self): """Retrieves Controller Configurations""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/customExitPoint/getAllCustomExitPoints") def getAllCustomExitPoints(self, application: Body): """Retrieves Custom Edit Point Configurations""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/backendConfig/getBackendDiscoveryConfigs") def getBackendDiscoveryConfigs(self, body: Body): """Retrieves Controller Configurations""" @@ -87,6 +102,11 @@ def getInstrumentationLevel(self, applicationID: Path): """Retrieves Instrumentation Level""" @params({"output": "json"}) + @headers( + { + "Accept": "application/json, text/plain, */*", + } + ) @get("/controller/restui/agentManager/getAllApplicationComponentsWithNodes/{applicationID}") def getAllApplicationComponentsWithNodes(self, applicationID: Path): """Retrieves Node Configurations""" @@ -112,11 +132,13 @@ def getApplicationComponents(self, applicationID: Path): """Retrieves Application Components for Later to get getServiceEndpointCustomMatchRules""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/serviceEndpoint/getAll") def getServiceEndpointCustomMatchRules(self, body: Body): """Retrieves Service Endpoint Custom Match Rules for an individual Application Tier""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/serviceEndpoint/getServiceEndpointMatchConfigs") def getServiceEndpointDefaultMatchRules(self, body: Body): """Retrieves Service Endpoint Custom Match Rules for an individual Application Tier""" @@ -124,15 +146,16 @@ def getServiceEndpointDefaultMatchRules(self, body: Body): @params({"output": "json"}) @get("/controller/restui/events/eventCounts") def getEventCounts( - self, - applicationID: Query("applicationId"), - entityType: Query("entityType"), - entityID: Query("entityId"), - timeRangeString: Query("timeRangeString"), + self, + applicationID: Query("applicationId"), + entityType: Query("entityType"), + entityID: Query("entityId"), + timeRangeString: Query("timeRangeString"), ): """Retrieves Event Counts""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/metricBrowser/async/metric-tree/root") def getMetricTree(self, body: Body): """Retrieves Metrics""" @@ -140,28 +163,28 @@ def getMetricTree(self, body: Body): @params({"output": "json"}) @get("/controller/rest/applications/{applicationID}/metric-data") def getMetricData( - self, - applicationID: Path, - metric_path: Query("metric-path"), - rollup: Query("rollup"), - time_range_type: Query("time-range-type"), - duration_in_mins: Query("duration-in-mins"), - start_time: Query("start-time"), - end_time: Query("end-time"), + self, + applicationID: Path, + metric_path: Query("metric-path"), + rollup: Query("rollup"), + time_range_type: Query("time-range-type"), + duration_in_mins: Query("duration-in-mins"), + start_time: Query("start-time"), + end_time: Query("end-time"), ): """Retrieves Metrics""" @params({"output": "json"}) @get("/controller/rest/applications/{applicationID}/events") def getApplicationEvents( - self, - applicationID: Path, - event_types: Query("event-types"), - severities: Query("severities"), - time_range_type: Query("time-range-type"), - duration_in_mins: Query("duration-in-mins"), - start_time: Query("start-time"), - end_time: Query("end-time"), + self, + applicationID: Path, + event_types: Query("event-types"), + severities: Query("severities"), + time_range_type: Query("time-range-type"), + duration_in_mins: Query("duration-in-mins"), + start_time: Query("start-time"), + end_time: Query("end-time"), ): """Retrieves Events""" @@ -186,6 +209,7 @@ def getDataCollectors(self, applicationID: Path): """Retrieves Data Collectors""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/snapshot/snapshotListDataWithFilterHandle") def getSnapshotsWithDataCollector(self, body: Body): """Retrieves Snapshots""" @@ -196,11 +220,13 @@ def getAnalyticsEnabledStatusForAllApplications(self): """Retrieves Analytics Enabled Status for app Applications""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @get("/controller/restui/dashboards/getAllDashboardsByType/false") def getAllDashboardsMetadata(self): """Retrieves all Dashboards""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @get("/controller/CustomDashboardImportExportServlet") def getDashboard(self, dashboardId: Query("dashboardId")): """Retrieves a single Dashboard""" @@ -216,26 +242,41 @@ def getUser(self, userID: Path): """Retrieves permission set of a given user""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/licenseRule/getAllLicenseModuleProperties") def getAccountUsageSummary(self, body: Body): """Retrieves license usage summary""" @params({"output": "json"}) + @get("/controller/restui/apiClientAdministrationUiService/apiClients") + def getApiClients(self): + """Retrieves list of API Clients""" + + @params({"output": "json"}) + @get("/controller/restui/accountRoleAdministrationUiService/accountRoleSummaries") + def getRoles(self): + """Retrieves list of Roles""" + + @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/licenseRule/getEumLicenseUsage") def getEumLicenseUsage(self, body: Body): """Retrieves EUM license usage""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/agents/list/appserver") def getAppServerAgents(self, body: Body): """Retrieves app server agent summary list""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/agents/list/machine") def getMachineAgents(self, body: Body): """Retrieves machine agent summary list""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/agents/list/appserver/ids") def getAppServerAgentsIds(self, body: Body): """Retrieves app server agent summary list""" @@ -246,6 +287,7 @@ def getAppServerAgentsMetadata(self, applicationId: Path, nodeId: Path): """Retrieves app agent metadata""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/agents/list/machine/ids") def getMachineAgentsIds(self, body: Body): """Retrieves machine agent summary list""" @@ -261,6 +303,7 @@ def getAnalyticsAgents(self): """Retrieves analytics agent summary list""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/sim/v2/user/machines/keys") def getServersKeys(self, body: Body): """Retrieves machine agents in bulk""" @@ -271,6 +314,7 @@ def getServer(self, machineId: Path): """Retrieves server agent info""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/sim/v2/user/metrics/query/machines") def getServerAvailability(self, body: Body): """Retrieves server availability info""" @@ -281,11 +325,13 @@ def getEumApplications(self, timeRange: Query("time-range")): """Retrieves all Eum Applications""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/pageList/getEumPageListViewData") def getEumPageListViewData(self, body: Body): """Retrieves Eum Page List View Data""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/web/pagelist") def getEumNetworkRequestList(self, body: Body): """Retrieves Eum Network Request List""" @@ -306,6 +352,7 @@ def getVirtualPagesConfig(self, applicationId: Path): """Retrieves virtual pages config""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/browserSnapshotList/getSnapshots") def getBrowserSnapshots(self, body: Body): """Retrieves browser snapshots""" @@ -326,26 +373,31 @@ def getNetworkRequestLimit(self, applicationId: Path): """Retrieves network request limit""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/mobileSnapshotListUiService/getMobileSnapshotSummaries") def getMobileSnapshots(self, body: Body): """Retrieves mobile snapshots""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/synthetic/schedule/getJobList/{applicationId}") def getSyntheticJobs(self, applicationId: Path): """Retrieves Synthetic Job List""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/eumSyntheticJobListUiService/getBillableTimeData") def getSyntheticBillableTime(self, body: Body): """Retrieves Synthetic Billable Time""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/synthetic/schedule/{applicationId}/getJobPAUtilizations") def getSyntheticPrivateAgentUtilization(self, applicationId: Path, body: Body): """Retrieves Synthetic Private Agent Utilization""" @params({"output": "json"}) + @headers({"Content-Type": "application/json"}) @post("/controller/restui/eumSyntheticJobListUiService/getSessionData") def getSyntheticSessionData(self, body: Body): """Retrieves Synthetic Session Data""" diff --git a/backend/api/appd/AppDService.py b/backend/api/appd/AppDService.py index 54efb85..6b6eed9 100644 --- a/backend/api/appd/AppDService.py +++ b/backend/api/appd/AppDService.py @@ -1,4 +1,3 @@ -import ipaddress import json import logging import re @@ -8,61 +7,36 @@ from math import ceil from typing import List -import aiohttp -from api.appd.AppDController import AppdController from api.Result import Result -from uplink import AiohttpClient -from uplink.auth import BasicAuth, MultiAuth, ProxyAuth +from api.appd.AppDController import AppdController +from api.appd.AuthMethod import AuthMethod from util.asyncio_utils import AsyncioUtils from util.stdlib_utils import get_recursively class AppDService: controller: AppdController + authMethod: AuthMethod + + def __init__(self, + applicationFilter: dict = None, + timeRangeMins: int = 1440, + authMethod: AuthMethod = None): - def __init__( - self, - host: str, - port: int, - ssl: bool, - account: str, - username: str, - pwd: str, - verifySsl: bool = True, - useProxy: bool = False, - applicationFilter: dict = None, - timeRangeMins: int = 1440, - ): - logging.debug(f"{host} - Initializing controller service") - connection_url = f'{"https" if ssl else "http"}://{host}:{port}' - auth = BasicAuth(f"{username}@{account}", pwd) - self.host = host - self.username = username self.applicationFilter = applicationFilter self.timeRangeMins = timeRangeMins self.endTime = int(round(time.time() * 1000)) self.startTime = self.endTime - (1 * 60 * self.timeRangeMins * 1000) - - cookie_jar = aiohttp.CookieJar() - try: - if ipaddress.ip_address(host): - logging.warning(f"Configured host {host} is an IP address. Consider using the DNS instead.") - logging.warning(f"RFC 2109 explicitly forbids cookie accepting from URLs with IP address instead of DNS name.") - logging.warning(f"Using unsafe Cookie Jar.") - cookie_jar = aiohttp.CookieJar(unsafe=True) - except ValueError: - pass - - connector = aiohttp.TCPConnector(limit=AsyncioUtils.concurrentConnections, verify_ssl=verifySsl) - self.session = aiohttp.ClientSession(connector=connector, trust_env=useProxy, cookie_jar=cookie_jar) - - self.controller = AppdController( - base_url=connection_url, - auth=auth, - client=AiohttpClient(session=self.session), - ) self.totalCallsProcessed = 0 + self.authMethod = authMethod + self.host = authMethod.host + self.controller = authMethod.controller + self.username = authMethod.username + + def getAuthMethod(self) -> AuthMethod: + return self.authMethod + def __json__(self): return { "host": self.host, @@ -80,31 +54,44 @@ async def loginToController(self) -> Result: Result.Error(f"{self.host} - {e}."), ) if response.status_code != 200: - logging.error(f"{self.host} - Controller login failed with {response.status_code}. Check username and password.") + err_msg = f"{self.host} - Controller login failed with " \ + f"{response.status_code}. Check username and password." + logging.error(err_msg) return Result( response, - Result.Error(f"{self.host} - Controller login failed with {response.status_code}. Check username and password."), + Result.Error(err_msg), ) try: - jsessionid = re.search("JSESSIONID=(\\w|\\d)*", str(response.headers)).group(0).split("JSESSIONID=")[1] + jsessionid = \ + re.search("JSESSIONID=(\\w|\\d)*", str(response.headers)).group( + 0).split("JSESSIONID=")[1] self.controller.jsessionid = jsessionid except AttributeError: - logging.debug(f"{self.host} - Unable to find JSESSIONID in login response. Please verify credentials.") + logging.debug( + f"{self.host} - Unable to find JSESSIONID in login response. Please verify credentials.") try: - xcsrftoken = re.search("X-CSRF-TOKEN=(\\w|\\d)*", str(response.headers)).group(0).split("X-CSRF-TOKEN=")[1] + xcsrftoken = \ + re.search("X-CSRF-TOKEN=(\\w|\\d)*", + str(response.headers)).group( + 0).split("X-CSRF-TOKEN=")[1] self.controller.xcsrftoken = xcsrftoken except AttributeError: - logging.debug(f"{self.host} - Unable to find X-CSRF-TOKEN in login response. Please verify credentials.") + logging.debug( + f"{self.host} - Unable to find X-CSRF-TOKEN in login response. Please verify credentials.") if self.controller.jsessionid is None or self.controller.xcsrftoken is None: return Result( response, - Result.Error(f"{self.host} - Valid authentication headers not cached from previous login call. Please verify credentials."), + Result.Error( + f"{self.host} - Valid authentication headers not cached from previous login call. Please verify credentials."), ) - self.controller.session.headers["X-CSRF-TOKEN"] = self.controller.xcsrftoken - self.controller.session.headers["Set-Cookie"] = f"JSESSIONID={self.controller.jsessionid};X-CSRF-TOKEN={self.controller.xcsrftoken};" - self.controller.session.headers["Content-Type"] = "application/json;charset=UTF-8" + self.controller.session.headers[ + "X-CSRF-TOKEN"] = self.controller.xcsrftoken + self.controller.session.headers[ + "Set-Cookie"] = f"JSESSIONID={self.controller.jsessionid};X-CSRF-TOKEN={self.controller.xcsrftoken};" + self.controller.session.headers[ + "Content-Type"] = "application/json;charset=UTF-8" logging.debug(f"{self.host} - Controller initialization successful.") return Result(self.controller, None) @@ -121,7 +108,8 @@ async def getApmApplications(self) -> Result: if self.applicationFilter is not None: if self.applicationFilter.get("apm") is None: - logging.warning(f"Filtered out all APM applications from analysis by match rule {self.applicationFilter['apm']}") + logging.warning( + f"Filtered out all APM applications from analysis by match rule {self.applicationFilter['apm']}") return Result([], None) response = await self.controller.getApmApplications() @@ -136,8 +124,10 @@ async def getApmApplications(self) -> Result: pattern = re.compile(self.applicationFilter["apm"]) for application in result.data: if not pattern.search(application["name"]): - logging.warning(f"Filtered out APM application {application['name']} from analysis by match rule {self.applicationFilter['apm']}") - result.data = [application for application in result.data if pattern.search(application["name"])] + logging.warning( + f"Filtered out APM application {application['name']} from analysis by match rule {self.applicationFilter['apm']}") + result.data = [application for application in result.data if + pattern.search(application["name"])] return result @@ -207,13 +197,17 @@ async def getInstrumentationLevel(self, applicationID: int) -> Result: debugString = f"Gathering Instrumentation Level for Application:{applicationID}" logging.debug(f"{self.host} - {debugString}") response = await self.controller.getInstrumentationLevel(applicationID) - return await self.getResultFromResponse(response, debugString, isResponseJSON=False) + return await self.getResultFromResponse(response, debugString, + isResponseJSON=False) - async def getAllNodePropertiesForCustomizedComponents(self, applicationID: int) -> Result: + async def getAllNodePropertiesForCustomizedComponents(self, + applicationID: int) -> Result: debugString = f"Gathering All Application Components With Nodes for Application:{applicationID}" logging.debug(f"{self.host} - {debugString}") - response = await self.controller.getAllApplicationComponentsWithNodes(applicationID) - applicationComponentsWithNodes = await self.getResultFromResponse(response, debugString) + response = await self.controller.getAllApplicationComponentsWithNodes( + applicationID) + applicationComponentsWithNodes = await self.getResultFromResponse( + response, debugString) getAgentConfigurationFutures = [] for applicationConfiguration in applicationComponentsWithNodes.data: @@ -260,7 +254,8 @@ async def getAllNodePropertiesForCustomizedComponents(self, applicationID: int) agentConfigurations = await AsyncioUtils.gatherWithConcurrency(*getAgentConfigurationFutures) return Result(agentConfigurations, None) - async def getAgentConfiguration(self, applicationID: int, agentType: str, entityType: str, entityId: int) -> Result: + async def getAgentConfiguration(self, applicationID: int, agentType: str, + entityType: str, entityId: int) -> Result: debugString = f"Gathering Agent Configuration for Application:{applicationID} entity:{entityId}" logging.debug(f"{self.host} - {debugString}") body = ( @@ -276,8 +271,10 @@ async def getAgentConfiguration(self, applicationID: int, agentType: str, entity async def getApplicationConfiguration(self, applicationID: int) -> Result: debugString = f"Gathering Application Call Graph Settings for Application:{applicationID}" logging.debug(f"{self.host} - {debugString}") - response = await self.controller.getApplicationConfiguration(applicationID) - return await self.getResultFromResponse(response, debugString, isResponseList=False) + response = await self.controller.getApplicationConfiguration( + applicationID) + return await self.getResultFromResponse(response, debugString, + isResponseList=False) async def getServiceEndpointMatchRules(self, applicationID: int) -> Result: debugString = f"Gathering Service Endpoint Custom Match Rules for Application:{applicationID}" @@ -290,22 +287,31 @@ async def getServiceEndpointMatchRules(self, applicationID: int) -> Result: body = '{"agentType":"APP_AGENT","attachedEntity":{"entityId":{entityId},"entityType":"APPLICATION"}}'.replace( "{entityId}", f"{applicationID}" ) - defaultMatchRulesFutures.append(self.controller.getServiceEndpointDefaultMatchRules(body)) + defaultMatchRulesFutures.append( + self.controller.getServiceEndpointDefaultMatchRules(body)) for entity in response.data: body = '{"attachedEntity":{"entityType":"APPLICATION_COMPONENT","entityId":{entityId}},"agentType":"{agentType}"}'.replace( "{entityId}", str(entity["id"]) ).replace("{agentType}", str(entity["componentType"]["agentType"])) - customMatchRulesFutures.append(self.controller.getServiceEndpointCustomMatchRules(body)) + customMatchRulesFutures.append( + self.controller.getServiceEndpointCustomMatchRules(body)) body = '{"agentType":"APP_AGENT","attachedEntity":{"entityId":{entityId},"entityType":"APPLICATION_COMPONENT"}}'.replace( "{entityId}", str(entity["id"]) ) - defaultMatchRulesFutures.append(self.controller.getServiceEndpointDefaultMatchRules(body)) - - response = await AsyncioUtils.gatherWithConcurrency(*customMatchRulesFutures) - customMatchRules = [await self.getResultFromResponse(response, debugString) for response in response] - response = await AsyncioUtils.gatherWithConcurrency(*defaultMatchRulesFutures) - defaultMatchRules = [await self.getResultFromResponse(response, debugString) for response in response] + defaultMatchRulesFutures.append( + self.controller.getServiceEndpointDefaultMatchRules(body)) + + response = await AsyncioUtils.gatherWithConcurrency( + *customMatchRulesFutures) + customMatchRules = [ + await self.getResultFromResponse(response, debugString) for response + in response] + response = await AsyncioUtils.gatherWithConcurrency( + *defaultMatchRulesFutures) + defaultMatchRules = [ + await self.getResultFromResponse(response, debugString) for response + in response] return Result((customMatchRules, defaultMatchRules), None) @@ -315,14 +321,16 @@ async def getAppLevelBTConfig(self, applicationID: int) -> Result: response = await self.controller.getAppLevelBTConfig(applicationID) return await self.getResultFromResponse(response, debugString) - async def getCustomMetrics(self, applicationID: int, tierName: str) -> Result: + async def getCustomMetrics(self, applicationID: int, + tierName: str) -> Result: debugString = f"Gathering Custom Metrics for Application:{applicationID}" logging.debug(f"{self.host} - {debugString}") body = { "request": None, "applicationId": applicationID, "livenessStatus": "ALL", - "pathData": ["Application Infrastructure Performance", tierName, "Custom Metrics"], + "pathData": ["Application Infrastructure Performance", tierName, + "Custom Metrics"], "timeRangeSpecifier": { "type": "BEFORE_NOW", "durationInMinutes": self.timeRangeMins, @@ -336,14 +344,14 @@ async def getCustomMetrics(self, applicationID: int, tierName: str) -> Result: return await self.getResultFromResponse(response, debugString) async def getMetricData( - self, - applicationID: int, - metric_path: str, - rollup: bool, - time_range_type: str, - duration_in_mins: int = "", - start_time: int = "", - end_time: int = 1440, + self, + applicationID: int, + metric_path: str, + rollup: bool, + time_range_type: str, + duration_in_mins: int = "", + start_time: int = "", + end_time: int = 1440, ) -> Result: debugString = f'Gathering Metrics for:"{metric_path}" on application:{applicationID}' logging.debug(f"{self.host} - {debugString}") @@ -359,14 +367,14 @@ async def getMetricData( return await self.getResultFromResponse(response, debugString) async def getApplicationEvents( - self, - applicationID: int, - event_types: List[str], - severities: List[str], - time_range_type: str, - duration_in_mins: str = "", - start_time: str = "", - end_time: str = "", + self, + applicationID: int, + event_types: List[str], + severities: List[str], + time_range_type: str, + duration_in_mins: str = "", + start_time: str = "", + end_time: str = "", ) -> Result: debugString = f'Gathering Application Events for:"{event_types}" with severities {severities} on application:{applicationID}' logging.debug(f"{self.host} - {debugString}") @@ -381,11 +389,13 @@ async def getApplicationEvents( ) return await self.getResultFromResponse(response, debugString) - async def getEventCounts(self, applicationID: int, entityType: str, entityID: int) -> Result: + async def getEventCounts(self, applicationID: int, entityType: str, + entityID: int) -> Result: debugString = f'Gathering Event Counts for:"{entityType}" {entityID} on application:{applicationID}' logging.debug(f"{self.host} - {debugString}") response = await self.controller.getEventCounts( - applicationID, entityType, entityID, f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}" + applicationID, entityType, entityID, + f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}" ) return await self.getResultFromResponse(response, debugString) @@ -397,14 +407,16 @@ async def getHealthRules(self, applicationID: int) -> Result: healthRuleDetails = [] for healthRule in healthRules.data: - healthRuleDetails.append(self.controller.getHealthRule(applicationID, healthRule["id"])) + healthRuleDetails.append( + self.controller.getHealthRule(applicationID, healthRule["id"])) responses = await AsyncioUtils.gatherWithConcurrency(*healthRuleDetails) healthRulesData = [] for response, healthRule in zip(responses, healthRules.data): debugString = f"Gathering Health Rule Data for Application:{applicationID} HealthRule:'{healthRule['name']}'" - healthRulesData.append(await self.getResultFromResponse(response, debugString)) + healthRulesData.append( + await self.getResultFromResponse(response, debugString)) return Result(healthRulesData, None) @@ -415,12 +427,12 @@ async def getPolicies(self, applicationID: int) -> Result: return await self.getResultFromResponse(response, debugString) async def getSnapshotsWithDataCollector( - self, - applicationID: int, - data_collector_name: str, - data_collector_type: str, - maximum_results: int = 1, - data_collector_value: str = "", + self, + applicationID: int, + data_collector_name: str, + data_collector_type: str, + maximum_results: int = 1, + data_collector_value: str = "", ) -> Result: debugString = f"Gathering Snapshots for Application:{applicationID}" logging.debug(f"{self.host} - {debugString}") @@ -439,15 +451,19 @@ async def getSnapshotsWithDataCollector( "url": None, "sessionId": None, "userPrincipalId": None, - "dataCollectorFilter": {"collectorType": data_collector_type, "query": {"name": data_collector_name, "value": ""}}, + "dataCollectorFilter": {"collectorType": data_collector_type, + "query": {"name": data_collector_name, + "value": ""}}, "archived": None, "guids": [], "diagnosticSnapshot": None, "badRequest": None, "deepDivePolicy": [], - "rangeSpecifier": {"type": "BEFORE_NOW", "durationInMinutes": self.timeRangeMins}, + "rangeSpecifier": {"type": "BEFORE_NOW", + "durationInMinutes": self.timeRangeMins}, } - response = await self.controller.getSnapshotsWithDataCollector(json.dumps(body)) + response = await self.controller.getSnapshotsWithDataCollector( + json.dumps(body)) return await self.getResultFromResponse(response, debugString) @@ -457,9 +473,13 @@ async def getDataCollectorUsage(self, applicationID: int) -> Result: response = await self.controller.getDataCollectors(applicationID) dataCollectors = await self.getResultFromResponse(response, debugString) - snapshotEnabledDataCollectors = [dataCollector for dataCollector in dataCollectors.data if dataCollector["enabledForApm"]] + snapshotEnabledDataCollectors = [dataCollector for dataCollector in + dataCollectors.data if + dataCollector["enabledForApm"]] - httpDataCollectors = [dataCollector for dataCollector in snapshotEnabledDataCollectors if dataCollector["type"] == "http"] + httpDataCollectors = [dataCollector for dataCollector in + snapshotEnabledDataCollectors if + dataCollector["type"] == "http"] dataCollectorFields = [] for dataCollector in httpDataCollectors: for field in dataCollector["requestParameters"]: @@ -471,15 +491,21 @@ async def getDataCollectorUsage(self, applicationID: int) -> Result: ) ) for field in dataCollector["cookieNames"]: - dataCollectorFields.append(("Cookie", field, dataCollector["enabledForAnalytics"])) + dataCollectorFields.append( + ("Cookie", field, dataCollector["enabledForAnalytics"])) for field in dataCollector["sessionKeys"]: - dataCollectorFields.append(("Session Key", field, dataCollector["enabledForAnalytics"])) + dataCollectorFields.append(("Session Key", field, dataCollector[ + "enabledForAnalytics"])) for field in dataCollector["headers"]: - dataCollectorFields.append(("HTTP Header", field, dataCollector["enabledForAnalytics"])) + dataCollectorFields.append(("HTTP Header", field, dataCollector[ + "enabledForAnalytics"])) - pojoDataCollectors = [dataCollector for dataCollector in snapshotEnabledDataCollectors if dataCollector["type"] == "pojo"] + pojoDataCollectors = [dataCollector for dataCollector in + snapshotEnabledDataCollectors if + dataCollector["type"] == "pojo"] for dataCollector in pojoDataCollectors: - for methodDataGathererConfig in dataCollector["methodDataGathererConfigs"]: + for methodDataGathererConfig in dataCollector[ + "methodDataGathererConfigs"]: dataCollectorFields.append( ( "Business Data", @@ -491,7 +517,8 @@ async def getDataCollectorUsage(self, applicationID: int) -> Result: snapshotsContainingDataCollectorFields = [] distinctDataCollectors = set() for dataCollectorField in dataCollectorFields: - if (applicationID, dataCollectorField[1], dataCollectorField[0]) not in distinctDataCollectors: + if (applicationID, dataCollectorField[1], + dataCollectorField[0]) not in distinctDataCollectors: snapshotsContainingDataCollectorFields.append( self.getSnapshotsWithDataCollector( applicationID=applicationID, @@ -499,12 +526,16 @@ async def getDataCollectorUsage(self, applicationID: int) -> Result: data_collector_type=dataCollectorField[0], ) ) - distinctDataCollectors.add((applicationID, dataCollectorField[1], dataCollectorField[0])) - snapshotResults = await AsyncioUtils.gatherWithConcurrency(*snapshotsContainingDataCollectorFields) + distinctDataCollectors.add( + (applicationID, dataCollectorField[1], dataCollectorField[0])) + snapshotResults = await AsyncioUtils.gatherWithConcurrency( + *snapshotsContainingDataCollectorFields) dataCollectorFieldsWithSnapshots = [] - for collector, snapshotResult in zip(dataCollectorFields, snapshotResults): - if snapshotResult.error is None and len(snapshotResult.data["requestSegmentDataListItems"]) == 1: + for collector, snapshotResult in zip(dataCollectorFields, + snapshotResults): + if snapshotResult.error is None and len( + snapshotResult.data["requestSegmentDataListItems"]) == 1: dataCollectorFieldsWithSnapshots.append(collector) # This API does not work for either session keys or headers, as far as I know there is no way to get this info without inspecting ALL snapshots (won't do). # The API comes from the Transaction Snapshot filtering UI. No UI option for session keys or headers exists there. @@ -515,7 +546,10 @@ async def getDataCollectorUsage(self, applicationID: int) -> Result: result = { "allDataCollectors": dataCollectorFields, "dataCollectorsPresentInSnapshots": dataCollectorFieldsWithSnapshots, - "dataCollectorsPresentInAnalytics": [dataCollector for dataCollector in dataCollectorFieldsWithSnapshots if dataCollector[2]], + "dataCollectorsPresentInAnalytics": [dataCollector for dataCollector + in + dataCollectorFieldsWithSnapshots + if dataCollector[2]], } return Result(result, None) @@ -526,25 +560,41 @@ async def getAnalyticsEnabledStatusForAllApplications(self) -> Result: return await self.getResultFromResponse(response, debugString) async def getDashboards(self) -> Result: + + # self.controller.jsessionid = "c618719074f568d36fa97fca95c7" + # self.controller.xcsrftoken= "8c6b20c6ee1128ae47b8c4c782a6a28b55b5dc3d" + # self.controller.session.headers["X-CSRF-TOKEN"] = ( self.controller.xcsrftoken) + # self.controller.session.headers["Set-Cookie"] = (f"JSESSIONID=" + # f"{self.controller.jsessionid};X-CSRF-TOKEN={self.controller.xcsrftoken};") + debugString = f"Gathering Dashboards" logging.debug(f"{self.host} - {debugString}") response = await self.controller.getAllDashboardsMetadata() - allDashboardsMetadata = await self.getResultFromResponse(response, debugString) + allDashboardsMetadata = await self.getResultFromResponse(response, + debugString) dashboards = [] batch_size = AsyncioUtils.concurrentConnections for i in range(0, len(allDashboardsMetadata.data), batch_size): dashboardsFutures = [] - logging.debug(f"Batch iteration {int(i / batch_size)} of {ceil(len(allDashboardsMetadata.data) / batch_size)}") - chunk = allDashboardsMetadata.data[i : i + batch_size] + logging.debug( + f"Batch iteration {int(i / batch_size)} of {ceil(len(allDashboardsMetadata.data) / batch_size)}") + chunk = allDashboardsMetadata.data[i: i + batch_size] for dashboard in chunk: - dashboardsFutures.append(self.controller.getDashboard(dashboard["id"])) - - response = await AsyncioUtils.gatherWithConcurrency(*dashboardsFutures) - for dashboard in [await self.getResultFromResponse(response, debugString) for response in response]: + dashboardsFutures.append( + self.controller.getDashboard(dashboard["id"])) + + response = await AsyncioUtils.gatherWithConcurrency( + *dashboardsFutures) + for dashboard in [ + await self.getResultFromResponse(response, debugString) for + response in response]: dashboards.append(dashboard.data) + # logging.info(f'{self.host} - DASHBOARD: ' + # f'{dashboard.data["name"]}' + # f' {dashboard.data["name"]}') # The above implementation shouldn't be necessary since gatherWithConcurrency uses a semaphore to limit number of concurrent calls. # But on controllers with a large number of dashboards the coroutines will get stuck unless explicitly batched. @@ -558,7 +608,8 @@ async def getDashboards(self) -> Result: # dashboards = [dashboard.data for dashboard in dashboards if dashboard.error is None] returnedDashboards = [] - for dashboardSchema, dashboardOverview in zip(dashboards, allDashboardsMetadata.data): + for dashboardSchema, dashboardOverview in zip(dashboards, + allDashboardsMetadata.data): if "schemaVersion" in dashboardSchema: dashboardSchema["createdBy"] = dashboardOverview["createdBy"] dashboardSchema["createdOn"] = dashboardOverview["createdOn"] @@ -570,19 +621,20 @@ async def getDashboards(self) -> Result: async def getUserPermissions(self, username: str) -> Result: debugString = f"Gathering Permission set for user: {username}" logging.debug(f"{self.host} - {debugString}") - response = await self.controller.getUsers() - users = await self.getResultFromResponse(response, debugString) + response = await self.getAuthMethod().validatePermissions() - if users.error is not None: - logging.error(f"{self.host} - Call to Get User Permissions failed. Is user '{self.username}' an Account Owner?") + # response = await self.controller.getUsers() + # users = await self.getResultFromResponse(response, debugString) + + if response.error is not None: + logging.error( + f"{self.host} - Call to Get User Permissions failed. Is user '{self.username}' an Account Owner?") return Result( response, - Result.Error(f"{self.host} - Call to Get User Permissions failed. Is user '{self.username}' an Account Owner?"), + Result.Error( + f"{self.host} - Call to Get User Permissions failed. Is user '{self.username}' an Account Owner?"), ) - userID = next(user["id"] for user in users.data if user["name"].lower() == username.lower()) - - response = await self.controller.getUser(userID) return await self.getResultFromResponse(response, debugString) async def getAccountUsageSummary(self) -> Result: @@ -596,7 +648,8 @@ async def getAccountUsageSummary(self) -> Result: "timeRange": None, "timeRangeAdjusted": False, } - response = await self.controller.getAccountUsageSummary(json.dumps(body)) + response = await self.controller.getAccountUsageSummary( + json.dumps(body)) return await self.getResultFromResponse(response, debugString) async def getEumLicenseUsage(self) -> Result: @@ -613,23 +666,30 @@ async def getEumLicenseUsage(self) -> Result: response = await self.controller.getEumLicenseUsage(json.dumps(body)) return await self.getResultFromResponse(response, debugString) - async def getAppAgentMetadata(self, applicationId: int, agentIDs: list[str]) -> Result: + async def getAppAgentMetadata(self, applicationId: int, + agentIDs: list[str]) -> Result: debugString = f"Gathering App Agent Metadata" logging.debug(f"{self.host} - {debugString}") if len(agentIDs) == 0: return Result([], None) - futures = [self.controller.getAppServerAgentsMetadata(applicationId, agentId) for agentId in agentIDs] + futures = [ + self.controller.getAppServerAgentsMetadata(applicationId, agentId) + for agentId in agentIDs] response = await AsyncioUtils.gatherWithConcurrency(*futures) - results = [(await self.getResultFromResponse(response, debugString)).data for response in response] + results = [ + (await self.getResultFromResponse(response, debugString)).data for + response in response] return Result(results, None) async def getAppServerAgents(self) -> Result: debugString = f"Gathering App Server Agents Agents" logging.debug(f"{self.host} - {debugString}") body = { - "requestFilter": {"queryParams": {"applicationAssociationType": "ALL"}, "filters": []}, + "requestFilter": { + "queryParams": {"applicationAssociationType": "ALL"}, + "filters": []}, "resultColumns": [], "offset": 0, "limit": -1, @@ -643,13 +703,14 @@ async def getAppServerAgents(self) -> Result: if result.error is not None: return result - agentIds = [agent["applicationComponentNodeId"] for agent in result.data["data"]] + agentIds = [agent["applicationComponentNodeId"] for agent in + result.data["data"]] debugString = f"Gathering App Server Agents Agents List" agentFutures = [] batch_size = 50 for i in range(0, len(agentIds), batch_size): - chunk = agentIds[i : i + batch_size] + chunk = agentIds[i: i + batch_size] body = { "requestFilter": chunk, "resultColumns": [ @@ -668,10 +729,13 @@ async def getAppServerAgents(self) -> Result: "timeRangeStart": self.startTime, "timeRangeEnd": self.endTime, } - agentFutures.append(self.controller.getAppServerAgentsIds(json.dumps(body))) + agentFutures.append( + self.controller.getAppServerAgentsIds(json.dumps(body))) response = await AsyncioUtils.gatherWithConcurrency(*agentFutures) - results = [(await self.getResultFromResponse(response, debugString)).data["data"] for response in response] + results = [ + (await self.getResultFromResponse(response, debugString)).data[ + "data"] for response in response] out = [] for result in results: out.extend(result) @@ -681,7 +745,9 @@ async def getMachineAgents(self) -> Result: debugString = f"Gathering App Server Agents Agents" logging.debug(f"{self.host} - {debugString}") body = { - "requestFilter": {"queryParams": {"applicationAssociationType": "ALL"}, "filters": []}, + "requestFilter": { + "queryParams": {"applicationAssociationType": "ALL"}, + "filters": []}, "resultColumns": [], "offset": 0, "limit": -1, @@ -701,10 +767,11 @@ async def getMachineAgents(self) -> Result: agentFutures = [] batch_size = 50 for i in range(0, len(agentIds), batch_size): - chunk = agentIds[i : i + batch_size] + chunk = agentIds[i: i + batch_size] body = { "requestFilter": chunk, - "resultColumns": ["AGENT_VERSION", "APPLICATION_NAMES", "ENABLED"], + "resultColumns": ["AGENT_VERSION", "APPLICATION_NAMES", + "ENABLED"], "offset": 0, "limit": -1, "searchFilters": [], @@ -713,10 +780,13 @@ async def getMachineAgents(self) -> Result: "timeRangeEnd": self.endTime, } - agentFutures.append(self.controller.getMachineAgentsIds(json.dumps(body))) + agentFutures.append( + self.controller.getMachineAgentsIds(json.dumps(body))) response = await AsyncioUtils.gatherWithConcurrency(*agentFutures) - results = [(await self.getResultFromResponse(response, debugString)).data["data"] for response in response] + results = [ + (await self.getResultFromResponse(response, debugString)).data[ + "data"] for response in response] out = [] for result in results: out.extend(result) @@ -755,7 +825,7 @@ async def getServers(self) -> Result: machineIds = [] if not isinstance(serverKeys.data["machineKeys"], list): logging.warning("Expected 'serverKeys.data[\"machineKeys\"]' to be a " - "list, but got {}".format(type(serverKeys.data["machineKeys"]))) + "list, but got {}".format(type(serverKeys.data["machineKeys"]))) else: for serverKey in serverKeys.data["machineKeys"]: if isinstance(serverKey, dict) and "machineId" in serverKey: @@ -771,8 +841,10 @@ async def getServers(self) -> Result: logging.warning("Expected a dictionary, but found type: {}".format(type(serverKey))) - serverFutures = [self.controller.getServer(serverId) for serverId in machineIds] - serversResponses = await AsyncioUtils.gatherWithConcurrency(*serverFutures) + serverFutures = [self.controller.getServer(serverId) for serverId in + machineIds] + serversResponses = await AsyncioUtils.gatherWithConcurrency( + *serverFutures) serverAvailabilityFutures = [] for machineId in machineIds: @@ -783,20 +855,27 @@ async def getServers(self) -> Result: "ids": [machineId], "baselineId": None, } - serverAvailabilityFutures.append(self.controller.getServerAvailability(json.dumps(body))) - serversAvailabilityResponses = await AsyncioUtils.gatherWithConcurrency(*serverAvailabilityFutures) + serverAvailabilityFutures.append( + self.controller.getServerAvailability(json.dumps(body))) + serversAvailabilityResponses = await AsyncioUtils.gatherWithConcurrency( + *serverAvailabilityFutures) debugString = f"Gathering Machine Agents Agents List" - serversResults = [(await self.getResultFromResponse(serversResponse, debugString)) for serversResponse in serversResponses] + serversResults = [ + (await self.getResultFromResponse(serversResponse, debugString)) for + serversResponse in serversResponses] serversAvailabilityResults = [ - (await self.getResultFromResponse(serversAvailabilityResponse, debugString)) + (await self.getResultFromResponse(serversAvailabilityResponse, + debugString)) for serversAvailabilityResponse in serversAvailabilityResponses ] machineIdMap = {} - for serverResult, serverAvailabilityResult in zip(serversResults, serversAvailabilityResults): + for serverResult, serverAvailabilityResult in zip(serversResults, + serversAvailabilityResults): machine = serverResult.data - value = get_recursively(serverAvailabilityResult.data["data"], "value") + value = get_recursively(serverAvailabilityResult.data["data"], + "value") if value: availability = next(iter(value)) machine["availability"] = availability @@ -821,10 +900,12 @@ async def getEumApplications(self) -> Result: if self.applicationFilter is not None: if self.applicationFilter.get("brum") is None: - logging.warning(f"Filtered out all BRUM applications from analysis by match rule {self.applicationFilter['brum']}") + logging.warning( + f"Filtered out all BRUM applications from analysis by match rule {self.applicationFilter['brum']}") return Result([], None) - response = await self.controller.getEumApplications(f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}") + response = await self.controller.getEumApplications( + f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}") result = await self.getResultFromResponse(response, debugString) if self.applicationFilter is not None: @@ -834,7 +915,8 @@ async def getEumApplications(self) -> Result: logging.warning( f"Filtered out BRUM application {application['name']} from analysis by match rule {self.applicationFilter['brum']}" ) - result.data = [application for application in result.data if pattern.search(application["name"])] + result.data = [application for application in result.data if + pattern.search(application["name"])] return result @@ -847,15 +929,19 @@ async def getEumPageListViewData(self, applicationId: int) -> Result: "timeRangeString": f"Custom_Time_Range|BETWEEN_TIMES|{self.endTime}|{self.startTime}|{self.timeRangeMins}", "fetchSyntheticData": False, } - response = await self.controller.getEumPageListViewData(json.dumps(body)) + response = await self.controller.getEumPageListViewData( + json.dumps(body)) return await self.getResultFromResponse(response, debugString) async def getEumNetworkRequestList(self, applicationId: int) -> Result: debugString = f"Gathering EUM Page List View Data for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") body = { - "requestFilter": {"applicationId": applicationId, "fetchSyntheticData": False}, - "resultColumns": ["PAGE_TYPE", "PAGE_NAME", "TOTAL_REQUESTS", "END_USER_RESPONSE_TIME", "VISUALLY_COMPLETE_TIME"], + "requestFilter": {"applicationId": applicationId, + "fetchSyntheticData": False}, + "resultColumns": ["PAGE_TYPE", "PAGE_NAME", "TOTAL_REQUESTS", + "END_USER_RESPONSE_TIME", + "VISUALLY_COMPLETE_TIME"], "offset": 0, "limit": -1, "searchFilters": [], @@ -863,7 +949,8 @@ async def getEumNetworkRequestList(self, applicationId: int) -> Result: "timeRangeStart": self.startTime, "timeRangeEnd": self.endTime, } - response = await self.controller.getEumNetworkRequestList(json.dumps(body)) + response = await self.controller.getEumNetworkRequestList( + json.dumps(body)) return await self.getResultFromResponse(response, debugString) async def getPagesAndFramesConfig(self, applicationId: int) -> Result: @@ -884,7 +971,8 @@ async def getVirtualPagesConfig(self, applicationId: int) -> Result: response = await self.controller.getVirtualPagesConfig(applicationId) return await self.getResultFromResponse(response, debugString) - async def getBrowserSnapshotsWithServerSnapshots(self, applicationId: int) -> Result: + async def getBrowserSnapshotsWithServerSnapshots(self, + applicationId: int) -> Result: debugString = f"Gathering Browser Snapshots for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") body = { @@ -892,7 +980,9 @@ async def getBrowserSnapshotsWithServerSnapshots(self, applicationId: int) -> Re "timeRangeString": f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}", "filters": { "_classType": "BrowserSnapshotFilters", - "serverSnapshotExists": {"type": "BOOLEAN", "name": "ms_serverSnapshotExists", "value": True}, + "serverSnapshotExists": {"type": "BOOLEAN", + "name": "ms_serverSnapshotExists", + "value": True}, "pages": { "type": "FLY_OUT_SELECT", "name": "ms_pagesAndAjaxRequestsNavLabel", @@ -917,7 +1007,8 @@ async def getBrowserSnapshotsWithServerSnapshots(self, applicationId: int) -> Re }, } response = await self.controller.getBrowserSnapshots(json.dumps(body)) - return await self.getResultFromResponse(response, debugString, isResponseList=False) + return await self.getResultFromResponse(response, debugString, + isResponseList=False) async def getMRUMApplications(self) -> Result: debugString = f"Gathering MRUM Applications" @@ -925,10 +1016,12 @@ async def getMRUMApplications(self) -> Result: if self.applicationFilter is not None: if self.applicationFilter.get("mrum") is None: - logging.warning(f"Filtered out all MRUM applications from analysis by match rule {self.applicationFilter['mrum']}") + logging.warning( + f"Filtered out all MRUM applications from analysis by match rule {self.applicationFilter['mrum']}") return Result([], None) - response = await self.controller.getMRUMApplications(f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}") + response = await self.controller.getMRUMApplications( + f"Custom_Time_Range.BETWEEN_TIMES.{self.endTime}.{self.startTime}.{self.timeRangeMins}") result = await self.getResultFromResponse(response, debugString) tempData = result.data.copy() @@ -936,7 +1029,8 @@ async def getMRUMApplications(self) -> Result: for mrumApplicationGroup in tempData: for mrumApplication in mrumApplicationGroup["children"]: mrumApplication["name"] = mrumApplication["internalName"] - mrumApplication["taggedName"] = f"{mrumApplicationGroup['appKey']}-{mrumApplication['name']}" + mrumApplication[ + "taggedName"] = f"{mrumApplicationGroup['appKey']}-{mrumApplication['name']}" result.data.append(mrumApplication) if self.applicationFilter is not None: @@ -946,14 +1040,16 @@ async def getMRUMApplications(self) -> Result: logging.warning( f"Filtered out MRUM application {application['name']} from analysis by match rule {self.applicationFilter['mrum']}" ) - result.data = [application for application in result.data if pattern.search(application["name"])] + result.data = [application for application in result.data if + pattern.search(application["name"])] return result async def getMRUMNetworkRequestConfig(self, applicationId: int) -> Result: debugString = f"Gathering MRUM Network Request Config for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") - response = await self.controller.getMRUMNetworkRequestConfig(applicationId) + response = await self.controller.getMRUMNetworkRequestConfig( + applicationId) return await self.getResultFromResponse(response, debugString) async def getNetworkRequestLimit(self, applicationId: int) -> Result: @@ -962,7 +1058,9 @@ async def getNetworkRequestLimit(self, applicationId: int) -> Result: response = await self.controller.getNetworkRequestLimit(applicationId) return await self.getResultFromResponse(response, debugString) - async def getMobileSnapshotsWithServerSnapshots(self, applicationId: int, mobileApplicationId: int, platform: str) -> Result: + async def getMobileSnapshotsWithServerSnapshots(self, applicationId: int, + mobileApplicationId: int, + platform: str) -> Result: debugString = f"Gathering Mobile Snapshots for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") body = { @@ -981,7 +1079,8 @@ async def getSyntheticJobs(self, applicationId: int): response = await self.controller.getSyntheticJobs(applicationId) return await self.getResultFromResponse(response, debugString) - async def getSyntheticBillableTime(self, applicationId: int, scheduleIds: List[str]) -> Result: + async def getSyntheticBillableTime(self, applicationId: int, + scheduleIds: List[str]) -> Result: debugString = f"Gathering Synthetic Billable Time for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") body = { @@ -990,24 +1089,32 @@ async def getSyntheticBillableTime(self, applicationId: int, scheduleIds: List[s "startTime": self.startTime, "currentTime": self.endTime, } - response = await self.controller.getSyntheticBillableTime(json.dumps(body)) + response = await self.controller.getSyntheticBillableTime( + json.dumps(body)) return await self.getResultFromResponse(response, debugString) - async def getSyntheticPrivateAgentUtilization(self, applicationId: int, jobsJson: List[dict]) -> Result: + async def getSyntheticPrivateAgentUtilization(self, applicationId: int, + jobsJson: List[ + dict]) -> Result: debugString = f"Gathering Synthetic Private Agent Utilization for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") - response = await self.controller.getSyntheticPrivateAgentUtilization(applicationId, json.dumps(jobsJson)) + response = await self.controller.getSyntheticPrivateAgentUtilization( + applicationId, json.dumps(jobsJson)) return await self.getResultFromResponse(response, debugString) - async def getSyntheticSessionData(self, applicationId: int, jobsJson: List[dict]) -> Result: + async def getSyntheticSessionData(self, applicationId: int, + jobsJson: List[dict]) -> Result: debugString = f"Gathering Synthetic Session Data for Application {applicationId}" logging.debug(f"{self.host} - {debugString}") # get the last 24 hours in milliseconds lastMonth = self.endTime - (1 * 60 * 60 * 24 * 30 * 1000) - monthStart = datetime.timestamp(datetime.today().replace(day=1, hour=0, minute=0, second=0, microsecond=0)) + monthStart = datetime.timestamp( + datetime.today().replace(day=1, hour=0, minute=0, second=0, + microsecond=0)) ws = (date.today() - timedelta(date.today().weekday())) weekStart = datetime.timestamp( - datetime.today().replace(year=ws.year, month=ws.month, day=ws.day, hour=0, minute=0, second=0, microsecond=0)) + datetime.today().replace(year=ws.year, month=ws.month, day=ws.day, + hour=0, minute=0, second=0, microsecond=0)) body = { "appId": applicationId, "scheduleIds": jobsJson, @@ -1021,14 +1128,17 @@ async def getSyntheticSessionData(self, applicationId: int, jobsJson: List[dict] "utcOffset": -14400000, }, } - response = await self.controller.getSyntheticSessionData(json.dumps(body)) + response = await self.controller.getSyntheticSessionData( + json.dumps(body)) return await self.getResultFromResponse(response, debugString) async def close(self): logging.debug(f"{self.host} - Closing connection") - await self.session.close() + await self.authMethod.cleanup() - async def getResultFromResponse(self, response, debugString, isResponseJSON=True, isResponseList=True) -> Result: + async def getResultFromResponse(self, response, debugString, + isResponseJSON=True, + isResponseList=True) -> Result: body = (await response.content.read()).decode("ISO-8859-1") self.totalCallsProcessed += 1 @@ -1041,7 +1151,8 @@ async def getResultFromResponse(self, response, debugString, isResponseJSON=True except JSONDecodeError: pass logging.debug(msg) - return Result([] if isResponseList else {}, Result.Error(f"{response.status_code}")) + return Result([] if isResponseList else {}, + Result.Error(f"{response.status_code}")) if isResponseJSON: try: return Result(json.loads(body), None) diff --git a/backend/api/appd/AuthMethod.py b/backend/api/appd/AuthMethod.py new file mode 100644 index 0000000..0cb7cfe --- /dev/null +++ b/backend/api/appd/AuthMethod.py @@ -0,0 +1,383 @@ +import asyncio +import ipaddress +import json +import logging +import re + +import aiohttp +from uplink import AiohttpClient +from uplink.auth import BasicAuth, BearerToken + +from api.Result import Result +from api.appd.AppDController import AppdController +from util.asyncio_utils import AsyncioUtils +from util.logging_utils import initLogging + + +class AuthMethod(): + + controller: AppdController + + def __init__(self, + auth_method: str, + host, + port, + ssl=True, + account=None, + username=None, + password=None, + useProxy=True, + verifySsl=True, + controller: AppdController = None): + + self.auth_method = auth_method + self.host = host + self.port = port + self.ssl = ssl + self.account = account + self.username = username + self.password = password + self.useProxy = useProxy + self.verifySSL = verifySsl + self.session = None + connection_url = (f'{"https" if ssl else "http"}://{host}:{port}') + + + # poor man's DI + #TODO: replace with proper DI + if controller is None: + cookie_jar = aiohttp.CookieJar() + try: + if ipaddress.ip_address(host): + logging.warning( + f"Configured host {host} is an IP address. Consider using the DNS instead.") + logging.warning( + f"RFC 2109 explicitly forbids cookie accepting from URLs with IP address instead of DNS name.") + logging.warning(f"Using unsafe Cookie Jar.") + cookie_jar = aiohttp.CookieJar(unsafe=True) + except ValueError: + pass + + connector = aiohttp.TCPConnector( + limit=AsyncioUtils.concurrentConnections, verify_ssl=True) + + self.session = aiohttp.ClientSession(connector=connector, + trust_env=True, + cookie_jar=cookie_jar) + + self.controller = AppdController( + base_url=connection_url, + client=AiohttpClient(session=self.session), + session=self.session + ) + else: + self.session = controller.get_client_session() + self.controller = controller + + async def loginBasicAuthentication(self): + if self.auth_method.lower() == "basic": + self.controller.session.auth = ( + BasicAuth(f"{self.username}@{self.account}", + self.password)) + + try: + response = await self.controller.login() + except Exception as e: + logging.error( + f"Controller login failed with {e}. Check username and password.") + raise e + + if response.status_code != 200: + err_msg = f"{self.host} - Controller login failed with " \ + f"{response.status_code}. Check username and " \ + f"password" + logging.error(err_msg) + return Result( + response, + Result.Error(err_msg), + ) + try: + jsessionid = \ + re.search("JSESSIONID=(\\w|\\d)*", str(response.headers)).group( + 0).split("JSESSIONID=")[1] + self.controller.jsessionid = jsessionid + except AttributeError: + logging.info( + f"{self.host} - Unable to find JSESSIONID in login " + f"response. Please verify credentials.") + try: + xcsrftoken = re.search("X-CSRF-TOKEN=(\\w|\\d)*", + str(response.headers)).group(0).split( + "X-CSRF-TOKEN=")[1] + self.controller.xcsrftoken = xcsrftoken + except AttributeError: + logging.info( + f"{self.host} - Unable to find X-CSRF-TOKEN in login " + f"response. Please verify credentials.") + + if (self.controller.jsessionid is None or self.controller.xcsrftoken is + None): + return Result( + response, + Result.Error( + f"{self.host} - Valid authentication headers not " + f"cached from previous login call. Please verify credentials."), + ) + + self.controller.session.headers["X-CSRF-TOKEN"] = ( + self.controller.xcsrftoken) + self.controller.session.headers["Set-Cookie"] = (f"JSESSIONID=" + f"{self.controller.jsessionid};X-CSRF-TOKEN={self.controller.xcsrftoken};") + self.controller.session.headers["Content-Type"] = \ + "application/json;charset=UTF-8" + + return await self._validateBasicAuthPermissions(self.username) + + async def loginClientSecretOauthAuthentication(self): + payload = { + "grant_type": "client_credentials", + "client_id": f"{self.username}@{self.account}", + "client_secret": self.password, + } + + try: + response = await self.controller.loginOAuth(data=payload) + except Exception as e: + logging.error(f"Controller login failed with {e}. Check username and password.") + return Result(None, Result.Error(f"{self.host} - Controller login failed with ")) + + if response.status_code != 200: + err_msg = f"{self.host} - Controller login failed with " \ + f"{response.status_code}. Check username and " \ + f"password 1." + logging.error(err_msg) + return Result( + response, + Result.Error(err_msg), + ) + + token_data = await response.json() + token = token_data["access_token"] + expires_in = token_data.get("expires_in", 300) + self.controller.session.headers["Authorization"] = (f"Bearer" + f" {token}") + self.controller.session.auth = BearerToken(token) + + return await self._validateAPIClientPermissions(self.username) + + async def loginTokenOauthAuthentication(self): + self.controller.session.auth = BearerToken(self.password) + return await self._validateAPIClientPermissions(self.username) + + async def authenticate(self): + + if self.auth_method.lower() == "basic": + logging.info(f"Authenticating user {self.username}, using: " + f"<<<{self.auth_method} >>> " f"authentication for {self.host}") + return await self.loginBasicAuthentication() + + + if self.auth_method.lower() == "secret": + logging.info( + f"Authenticating user {self.username}, using API Client " + f"authentication <<< {self.auth_method} >>> (Client " + f"name/secret) for {self.host}") + return await self.loginClientSecretOauthAuthentication() + + if self.auth_method.lower() == "token": + logging.info(f"Authenticating user {self.username}, using API " + f"Client authentication <<< {self.auth_method} >>> " + f"(Temporary Access Token) for {self.host}") + return await self.loginTokenOauthAuthentication() + + + + async def getAdminRoleId(self): + response = await self.controller.getRoles() + roles = await self.getResultFromResponse(response, + "Get roles") + for role in roles.data: + if role["name"] == "Account Administrator": + return role["id"] + + async def _validateBasicAuthPermissions(self, username) -> Result: + if username is None: + msg = " - Username is required for basic auth authentication" + logging.error(f"{self.host} {msg}") + return Result( + None, + Result.Error( + f"{self.host} {msg}"), + ) + + response = await self.controller.getUsers() + users = await (self.getResultFromResponse(response, "Get users")) + + errorMsg = f"{self.host} - Unable to validate permissions. Does {username} have Account Ownership role? " + + if users.error is not None: + logging.error(errorMsg) + return Result( + None, + Result.Error(errorMsg), + ) + + userID = next(user["id"] for user in users.data if + user["name"].lower() == username.lower()) + response = await self.controller.getUser(userID) + result = await self.getResultFromResponse(response, "Get user") + + if result.error is not None: + return Result(response.Error(errorMsg)) + + adminRole = next((role for role in result.data["roles"] if role[ + "name"] == "super-admin"), None) + if not adminRole: + logging.error(errorMsg) + return Result(None, Result.Error(errorMsg)) + # If adminRole is found, continue with this logic + + logging.debug( + f"User permissions: Role: {adminRole['name']}, idx: {adminRole['id']}") + logging.info(f"{self.host} - {username} admin " + f"role validated. User has Account Ownership role!") + + return Result(result.data, None) + + + async def _validateAPIClientPermissions(self, username) -> Result: + ''' validate permissions for token user + :param username: username to validate permissions for + :return: Result object with dict containing the roles for the user + ''' + if username is None: + logging.error( + f"{self.host} - API Client username is required for token authentication") + return Result( + None, + Result.Error( + f"{self.host} - API Client username is required for token authentication"), + ) + + response = await self.controller.getApiClients() + apiClients = await ( + self.getResultFromResponse(response, "Get API " "clients")) + + # if no api clients exist, then the user is not an admin + if len(apiClients.data) == 0 or apiClients.error is not None: + errorMesg = (f"{self.host} - Not able to validate permissions. " + f"Does {username} have Account Ownership role? ") + logging.error(errorMesg) + return Result( + response, + Result.Error(errorMesg), + ) + + for apiClient in apiClients.data: + if apiClient["name"] == username: + roles = apiClient["accountRoleIds"] + # get the admin id for the admin role name and see if it exists in the list of roles for the user + adminRoleId = await self.getAdminRoleId() + if adminRoleId is not None and adminRoleId in roles: + logging.info(f"{self.host} - API client {username} admin " + f"role validated. User has Account Ownership role!") + # keep the interface consistent with basic auth perm check + roles = {"roles": [{"name": "Account Administrator"}], } + return Result(roles, None) + else: + msg = f"{self.host} - Unable to validate user administrative roles/permission. " + logging.error(msg) + return Result( + response, + Result.Error(msg) + ) + + return Result(apiClients, None) + + async def cleanup(self): + logging.info(f"Cleaning up closing connection for this service using " + f"{self.host}") + await self.session.close() + + async def getResultFromResponse(self, response, debugString, + isResponseJSON=True, + isResponseList=True) -> Result: + body = (await response.content.read()).decode("ISO-8859-1") + + if response.status_code >= 400: + msg = (f"{self.host} - {debugString} failed with code" + f":{response.status_code} body:{body}") + try: + responseJSON = json.loads(body) + if "message" in responseJSON: + msg = (f"{self.host} - {debugString} failed with code" + f":{response.status_code} body:{responseJSON['message']}") + except json.JSONDecodeError: + pass + logging.debug(msg) + return Result([] if isResponseList else {}, + Result.Error(f"{response.status_code}")) + if isResponseJSON: + try: + return Result(json.loads(body), None) + except json.JSONDecodeError: + msg = (f"{self.host} - {debugString} failed to parse json from " + f"body. Returned code:{response.status_code} body:{body}") + logging.error(msg) + return Result([] if isResponseList else {}, Result.Error(msg)) + else: + return Result(body, None) + + + +async def main(): + + authMethod = AuthMethod( + auth_method="", + host="", + port=443, + ssl=True, + account="", + username="", + password="", + useProxy=False, + verifySsl=True + ) + + await authMethod.authenticate() + await authMethod.cleanup() + + # inject controller + host = "" + ssl = True + port = 443 + connection_url = (f'{"https" if ssl else "http"}://{host}:{port}') + + cookie_jar = aiohttp.CookieJar() + try: + if ipaddress.ip_address(host): + logging.warning( + f"Configured host {host} is an IP address. Consider using the DNS instead.") + logging.warning( + f"RFC 2109 explicitly forbids cookie accepting from URLs with IP address instead of DNS name.") + logging.warning(f"Using unsafe Cookie Jar.") + cookie_jar = aiohttp.CookieJar(unsafe=True) + except ValueError: + pass + + connector = aiohttp.TCPConnector( + limit=AsyncioUtils.concurrentConnections, verify_ssl=True) + + client_session = aiohttp.ClientSession(connector=connector, + trust_env=True, + cookie_jar=cookie_jar) + + controller = AppdController( + base_url=connection_url, + client=AiohttpClient(session=client_session), + session=client_session + ) + +if __name__ == '__main__': + initLogging(True) + asyncio.run(main()) diff --git a/backend/core/Engine.py b/backend/core/Engine.py index 32d7085..cf47790 100644 --- a/backend/core/Engine.py +++ b/backend/core/Engine.py @@ -11,6 +11,7 @@ import requests from api.appd.AppDService import AppDService +from api.appd.AuthMethod import AuthMethod from extractionSteps.general.ControllerLevelDetails import ControllerLevelDetails from extractionSteps.general.CustomMetrics import CustomMetrics from extractionSteps.general.Synthetics import Synthetics @@ -50,6 +51,7 @@ class Engine: def __init__(self, jobFileName: str, thresholdsFileName: str, concurrentConnections: int, username: str, password: str, car: bool): # should we run the configuration analysis report in post-processing? + self.controllers = [] self.car = car if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): @@ -138,26 +140,50 @@ def __init__(self, jobFileName: str, thresholdsFileName: str, concurrentConnecti indent=4, ) - # Instantiate controllers, jobs, and report lists - self.controllers = [ - AppDService( + for controller in self.job: + + if controller.get("authType") is None: + logging.warn(f'\'authType\' is not ' + f'specified for host {controller["host"]} in ' + f'the input job file. Will ' + f'default to basic authentication for ' + f'backward compatibility.') + controller["authType"] = "basic" + + logging.debug( + f'authenticationMethod: {controller["authType"]} ' + f'for host {controller["host"]}') + + authMethod = AuthMethod( + auth_method=controller["authType"], host=controller["host"], port=controller["port"], ssl=controller["ssl"], account=controller["account"], username=username if username else controller["username"], - pwd=password if password else base64Decode(controller["pwd"])[len("CAT-ENCODED-") :], + password=password if password else base64Decode(controller[ + "pwd"])[len("CAT-ENCODED-") :], verifySsl=controller.get("verifySsl", True), - useProxy=controller.get("useProxy", False), + useProxy=controller.get("useProxy", False) + ) + + controllerService = AppDService( applicationFilter=controller.get("applicationFilter", None), timeRangeMins=controller.get("timeRangeMins", 1440), + authMethod=authMethod ) - for controller in self.job - ] + + + self.controllers.append(controllerService) + username = None + password = None + + if password: # I will let it here until it's the final version, so that we will logging.info("Dynamic password change was used!") # have confirmation, that it's working as intended else: logging.info("Using password from jobfile") + self.controllerData = OrderedDict() self.otherSteps = [ ControllerLevelDetails(), @@ -214,31 +240,15 @@ async def run(self): error=False, ) + async def initControllers(self) -> ([AppDService], str): logging.info(f"Validating Controller Login(s) for Job - {self.jobFileName} ") - - loginFutures = [controller.loginToController() for controller in self.controllers] + loginFutures = [controller.getAuthMethod().authenticate() for controller in + self.controllers] loginResults = await AsyncioUtils.gatherWithConcurrency(*loginFutures) if any(login.error is not None for login in loginResults): await self.abortAndCleanup(f"Unable to connect to one or more controllers. Aborting.") - userPermissionFutures = [controller.getUserPermissions(controller.username) for controller in self.controllers] - userPermissionsResults = await AsyncioUtils.gatherWithConcurrency(*userPermissionFutures) - if any(userPermissions.error is not None for userPermissions in userPermissionsResults): - await self.abortAndCleanup(f"Get user permissions failed for one or more controllers. Aborting.") - - anyUserNotAdmin = False - for idx, userPermissions in enumerate(userPermissionsResults): - adminRole = next( - (role for role in userPermissions.data["roles"] if role["name"] == "super-admin"), - None, - ) - if adminRole is None: - anyUserNotAdmin = True - logging.error(f"{self.controllers[idx].host} - Login user does not have Account Owner role. Please modify permissions.") - - if anyUserNotAdmin: - await self.abortAndCleanup(f"Login user not admin on one or more controllers. Aborting.") for idx, controller in enumerate(self.controllers): self.controllerData[controller.host] = OrderedDict() @@ -263,21 +273,21 @@ async def validateThresholdsFile(self): def thresholdStrictlyDecreasing(jobStep, thresholdMetric, componentType: str) -> bool: thresholds = self.thresholds[componentType][jobStep] if all( - value is True - for value in [ - thresholds["platinum"][thresholdMetric], - thresholds["gold"][thresholdMetric], - thresholds["silver"][thresholdMetric], - ] + value is True + for value in [ + thresholds["platinum"][thresholdMetric], + thresholds["gold"][thresholdMetric], + thresholds["silver"][thresholdMetric], + ] ): return True if all( - value is False - for value in [ - thresholds["platinum"][thresholdMetric], - thresholds["gold"][thresholdMetric], - thresholds["silver"][thresholdMetric], - ] + value is False + for value in [ + thresholds["platinum"][thresholdMetric], + thresholds["gold"][thresholdMetric], + thresholds["silver"][thresholdMetric], + ] ): return False return thresholds["platinum"][thresholdMetric] >= thresholds["gold"][thresholdMetric] >= thresholds["silver"][thresholdMetric] @@ -285,21 +295,21 @@ def thresholdStrictlyDecreasing(jobStep, thresholdMetric, componentType: str) -> def thresholdStrictlyIncreasing(jobStep, thresholdMetric, componentType: str) -> bool: thresholds = self.thresholds[componentType][jobStep] if all( - value is False - for value in [ - thresholds["platinum"][thresholdMetric], - thresholds["gold"][thresholdMetric], - thresholds["silver"][thresholdMetric], - ] + value is False + for value in [ + thresholds["platinum"][thresholdMetric], + thresholds["gold"][thresholdMetric], + thresholds["silver"][thresholdMetric], + ] ): return True if all( - value is True - for value in [ - thresholds["platinum"][thresholdMetric], - thresholds["gold"][thresholdMetric], - thresholds["silver"][thresholdMetric], - ] + value is True + for value in [ + thresholds["platinum"][thresholdMetric], + thresholds["gold"][thresholdMetric], + thresholds["silver"][thresholdMetric], + ] ): return False return thresholds["platinum"][thresholdMetric] <= thresholds["gold"][thresholdMetric] <= thresholds["silver"][thresholdMetric] diff --git a/backend/output/presentations/cxPptFsoUseCases.py b/backend/output/presentations/cxPptFsoUseCases.py index 913a514..6513021 100644 --- a/backend/output/presentations/cxPptFsoUseCases.py +++ b/backend/output/presentations/cxPptFsoUseCases.py @@ -112,7 +112,7 @@ def validColumn(self, filename, sheet_name, column_name): def getColumnTotal(self, filename, sheet_name, column_name): df = None if self.validColumn(filename, sheet_name, column_name) \ - and column_name in self.workbooks[filename][sheet_name].columns: + and column_name in self.workbooks[filename][sheet_name].columns: df = self.workbooks[filename][sheet_name] else: logging.error(f"No column found with name: {column_name}") @@ -122,7 +122,7 @@ def getColumnTotal(self, filename, sheet_name, column_name): def getColumnAverage(self, filename, sheet_name, column_name): df = None if self.validColumn(filename, sheet_name, column_name) \ - and column_name in self.workbooks[filename][sheet_name].columns: + and column_name in self.workbooks[filename][sheet_name].columns: df = self.workbooks[filename][sheet_name] else: logging.error(f"No column found with name: {column_name}") @@ -544,6 +544,8 @@ def createCxHamUseCasePpt(folder: str): f"{file_prefix}-Synthetics.xlsx" )) + assert len(excels.getWorkBooks()) >= 10 + # currently only 1st controller in the job file is examined. controller = getValuesInColumn(apm_wb["Analysis"], "controller")[0] @@ -664,9 +666,7 @@ def cleanup_slides(root: Presentation, uc: UseCase): def calculate_kpis(apm_wb, agent_wb, uc: UseCase): # currently only supports one controller report out of the workbook controller = getValuesInColumn(apm_wb["Analysis"], "controller")[0] - logging.info(f"processing CX HAM Use Case report for 1st controller only " - f"as multiple " - f"controllers are not supported yet: {controller}") + logging.info(f"processing report for 1st controller only as multiple controllers are not supported yet: {controller}") totalApplications = getRowCountForController(apm_wb["Analysis"], controller) percentAgentsReportingData = getValuesInColumnForController(apm_wb["AppAgentsAPM"], "percentAgentsReportingData", controller) diff --git a/backend/util/logging_utils.py b/backend/util/logging_utils.py index 1ed214e..e0a4d48 100644 --- a/backend/util/logging_utils.py +++ b/backend/util/logging_utils.py @@ -14,7 +14,7 @@ def initLogging(debug: bool): logging.basicConfig( level=logging.DEBUG if debug else logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", + format="%(asctime)s [%(levelname)s] %(name)s %(funcName)s: %(message)s", handlers=[ logging.FileHandler("logs/config-assessment-tool-backend.log"), logging.StreamHandler(), diff --git a/input/thresholds/DefaultThresholds.json b/input/thresholds/DefaultThresholds.json index 0b994c5..d399796 100644 --- a/input/thresholds/DefaultThresholds.json +++ b/input/thresholds/DefaultThresholds.json @@ -1,5 +1,5 @@ { - "version": "v1.6.2", + "version": "v1.7.0-beta", "apm": { "AppAgentsAPM": { "platinum": { diff --git a/tests/test_api.py b/tests/test_api.py index cb86835..652db55 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,12 +3,29 @@ from distutils.util import strtobool import pytest +import logging from api.appd.AppDService import AppDService +from api.appd.AuthMethod import AuthMethod +from util.logging_utils import initLogging APPLICATION_ID = int(os.getenv("TEST_CONTROLLER_APPLICATION_ID")) EUM_APPLICATION_ID = int(os.getenv("TEST_CONTROLLER_EUM_APPLICATION_ID")) -USERNAME = os.getenv("TEST_CONTROLLER_USERNAME") +host = os.getenv("TEST_CONTROLLER_HOST") +port = int(os.getenv("TEST_CONTROLLER_PORT")) +ssl = strtobool(os.getenv("TEST_CONTROLLER_SSL")) +account = os.getenv("TEST_CONTROLLER_ACCOUNT") +username = os.getenv("TEST_CONTROLLER_USERNAME") +pwd = os.getenv("TEST_CONTROLLER_PASSWORD") +oauth_client_id = os.getenv("TEST_OAUTH_CLIENT_ID") +client_id = f"{oauth_client_id}" +client_secret = os.getenv("TEST_OAUTH_CLIENT_SECRET") +bearer_token = os.getenv("TEST_OAUTH_BEARER_TOKEN") +connection_url = f'{"https" if ssl else "http"}://{host}:{port}' +token_url=f"{connection_url}/controller/api/oauth/access_token" + + +initLogging(True) @pytest.fixture def event_loop(): @@ -20,38 +37,49 @@ def event_loop(): loop.close() -@pytest.fixture -async def controller(): - if os.name == "nt": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - - host = os.getenv("TEST_CONTROLLER_HOST") - port = int(os.getenv("TEST_CONTROLLER_PORT")) - ssl = strtobool(os.getenv("TEST_CONTROLLER_SSL")) - account = os.getenv("TEST_CONTROLLER_ACCOUNT") - username = os.getenv("TEST_CONTROLLER_USERNAME") - pwd = os.getenv("TEST_CONTROLLER_PASSWORD") - - controller = AppDService( - host=host, - port=port, - ssl=ssl, - account=account, - username=username, - pwd=pwd, - ) - yield controller - - +def authentication_factory(auth_method): + method = None + if auth_method == "basic": + method = AuthMethod( auth_method=auth_method, host=host, + port=port, ssl=ssl, account=account, username=username, password=pwd, + verifySsl=True, useProxy=False + ) + elif auth_method == "secret": + method = AuthMethod( auth_method=auth_method, host=host, + port=port, ssl=ssl, account=account, username=client_id, + password=client_secret, verifySsl=True, useProxy=False + ) + elif auth_method == "token": + method = AuthMethod( auth_method=auth_method, host=host, port=port, + ssl=ssl, account=account, username=client_id, password=bearer_token, + verifySsl=True, useProxy=False + ) + else: + raise ValueError(f"Invalid auth method: {auth_method}") + + return AppDService( + applicationFilter={"apm": ".*", "mrum": ".*", "brum": ".*"}, + timeRangeMins=1440, + authMethod=method) + +@pytest.fixture() +def appdServiceFactory(): + return authentication_factory + +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) @pytest.mark.asyncio -async def testLogin(controller): - assert (await controller.loginToController()).error is None - await controller.close() +async def testLogin(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + await appdService.close() @pytest.mark.asyncio -async def testGetApmApplications(controller): - applications = await controller.getApmApplications() +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetApmApplications(appdServiceFactory, auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + applications = await appdService.getApmApplications() assert applications.error is None application = next( @@ -59,14 +87,15 @@ async def testGetApmApplications(controller): None, ) assert application is not None - - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetBtMatchRules(controller): - assert (await controller.loginToController()).error is None - btMatchRules = await controller.getBtMatchRules(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetBtMatchRules(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + btMatchRules = await appdService.getBtMatchRules(APPLICATION_ID) assert btMatchRules.error is None assert "ruleScopeSummaryMappings" in btMatchRules.data @@ -78,13 +107,16 @@ async def testGetBtMatchRules(controller): assert "summary" in rule["rule"] assert "name" in rule["rule"]["summary"] - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetConfigurations(controller): - assert (await controller.loginToController()).error is None - configurations = await controller.getConfigurations() +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetConfigurations(appdServiceFactory,auth_method): + logging.info("testGetConfig...") + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + configurations = await appdService.getConfigurations() assert configurations.error is None assert len(configurations.data) > 0 @@ -109,13 +141,15 @@ async def testGetConfigurations(controller): assert "value" in serviceEndpointLimitProp assert int(serviceEndpointLimitProp["value"]) >= 0 - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetAllCustomExitPoints(controller): - assert (await controller.loginToController()).error is None - customExitPoints = await controller.getAllCustomExitPoints(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetAllCustomExitPoints(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + customExitPoints = await appdService.getAllCustomExitPoints(APPLICATION_ID) assert customExitPoints.error is None @@ -124,21 +158,22 @@ async def testGetAllCustomExitPoints(controller): assert "agentType" in customExitPoint assert ( - next( - customExitPoint - for customExitPoint in customExitPoints.data - if customExitPoint["name"] == "FOO" and customExitPoint["agentType"] == "APP_AGENT" - ), - None, - ) is not None - - await controller.close() + next( + customExitPoint + for customExitPoint in customExitPoints.data + if customExitPoint["name"] == "FOO" and customExitPoint["agentType"] == "APP_AGENT" + ), + None, + ) is not None + await appdService.close() @pytest.mark.asyncio -async def testGetBackendDiscoveryConfigs(controller): - assert (await controller.loginToController()).error is None - backendDiscoveryConfigs = await controller.getBackendDiscoveryConfigs(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetBackendDiscoveryConfigs(appdServiceFactory, auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + backendDiscoveryConfigs = await appdService.getBackendDiscoveryConfigs(APPLICATION_ID) assert backendDiscoveryConfigs.error is None @@ -148,13 +183,15 @@ async def testGetBackendDiscoveryConfigs(controller): numberOfModifiedDefaultBackendDiscoveryConfigs = len([config for config in backendDiscoveryConfigs.data if config["version"] != 0]) assert numberOfModifiedDefaultBackendDiscoveryConfigs != 0 - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetDevModeConfig(controller): - assert (await controller.loginToController()).error is None - devModeConfig = await controller.getDevModeConfig(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetDevModeConfig(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + devModeConfig = await appdService.getDevModeConfig(APPLICATION_ID) assert devModeConfig.error is None @@ -163,24 +200,29 @@ async def testGetDevModeConfig(controller): for child in config["children"]: assert "enabled" in child - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetInstrumentationLevel(controller): - assert (await controller.loginToController()).error is None - instrumentationLevel = await controller.getInstrumentationLevel(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetInstrumentationLevel(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + instrumentationLevel = await appdService.getInstrumentationLevel(APPLICATION_ID) assert instrumentationLevel.error is None assert instrumentationLevel.data == "DEVELOPMENT" or instrumentationLevel.data == "PRODUCTION" - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetAllNodePropertiesForCustomizedComponents(controller): - assert (await controller.loginToController()).error is None - allNodeProperties = await controller.getAllNodePropertiesForCustomizedComponents(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["token"]) +async def testGetAllNodePropertiesForCustomizedComponents(appdServiceFactory, + auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + allNodeProperties = await appdService.getAllNodePropertiesForCustomizedComponents(APPLICATION_ID) assert allNodeProperties.error is None @@ -192,13 +234,15 @@ async def testGetAllNodePropertiesForCustomizedComponents(controller): assert "definition" in nodeProperty assert "name" in nodeProperty["definition"] - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetApplicationConfiguration(controller): - assert (await controller.loginToController()).error is None - applicationConfiguration = await controller.getApplicationConfiguration(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetApplicationConfiguration(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + applicationConfiguration = await appdService.getApplicationConfiguration(APPLICATION_ID) assert applicationConfiguration.error is None @@ -211,13 +255,15 @@ async def testGetApplicationConfiguration(controller): assert "rawSQL" in config assert type(config["rawSQL"]) == bool - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetServiceEndpointCustomMatchRules(controller): - assert (await controller.loginToController()).error is None - serviceEndpointMatchRules = await controller.getServiceEndpointMatchRules(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetServiceEndpointCustomMatchRules(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + serviceEndpointMatchRules = await appdService.getServiceEndpointMatchRules(APPLICATION_ID) assert serviceEndpointMatchRules.error is None @@ -239,13 +285,15 @@ async def testGetServiceEndpointCustomMatchRules(controller): assert "version" in definition assert "agentType" in definition - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetAppLevelBTConfig(controller): - assert (await controller.loginToController()).error is None - appLevelBTConfig = await controller.getAppLevelBTConfig(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetAppLevelBTConfig(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + appLevelBTConfig = await appdService.getAppLevelBTConfig(APPLICATION_ID) assert appLevelBTConfig.error is None @@ -254,14 +302,16 @@ async def testGetAppLevelBTConfig(controller): assert "btAutoCleanupTimeFrame" in appLevelBTConfig.data assert "btAutoCleanupCallCountThreshold" in appLevelBTConfig.data - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetMetricData(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetMetricData(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - metricData = await controller.getMetricData( + metricData = await appdService.getMetricData( applicationID=APPLICATION_ID, metric_path="Business Transaction Performance|Business Transactions|*|*|Calls per Minute", rollup=True, @@ -290,14 +340,16 @@ async def testGetMetricData(controller): assert "value" in value assert "standardDeviation" in value - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetEventCountsLastDay(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetEventCountsLastDay(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - eventCounts = await controller.getEventCounts( + eventCounts = await appdService.getEventCounts( applicationID=APPLICATION_ID, entityType="APPLICATION", entityID=APPLICATION_ID, @@ -313,14 +365,16 @@ async def testGetEventCountsLastDay(controller): assert eventCounts.data["policyViolationEventCounts"]["totalPolicyViolations"]["warning"] >= 0 assert eventCounts.data["policyViolationEventCounts"]["totalPolicyViolations"]["critical"] >= 0 - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetHealthRules(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetHealthRules(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - healthRules = await controller.getHealthRules(APPLICATION_ID) + healthRules = await appdService.getHealthRules(APPLICATION_ID) assert healthRules.error is None @@ -336,14 +390,16 @@ async def testGetHealthRules(controller): assert "affects" in healthRule.data assert "evalCriterias" in healthRule.data - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetPolicies(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetPolicies(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - policies = await controller.getPolicies(APPLICATION_ID) + policies = await appdService.getPolicies(APPLICATION_ID) assert policies.error is None assert len(policies.data) > 0 @@ -356,14 +412,15 @@ async def testGetPolicies(controller): assert "events" in policy assert "selectedEntityType" in policy - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetPolicies(controller): - assert (await controller.loginToController()).error is None - - policies = await controller.getPolicies(APPLICATION_ID) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetPolicies(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + policies = await appdService.getPolicies(APPLICATION_ID) assert policies.error is None assert len(policies.data) > 0 @@ -376,14 +433,16 @@ async def testGetPolicies(controller): assert "events" in policy assert "selectedEntityType" in policy - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetDataCollectorUsage(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetDataCollectorUsage(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - dataCollectorUsage = await controller.getDataCollectorUsage(APPLICATION_ID) + dataCollectorUsage = await appdService.getDataCollectorUsage(APPLICATION_ID) assert dataCollectorUsage.error is None @@ -393,44 +452,46 @@ async def testGetDataCollectorUsage(controller): assert ("HTTP Parameter", "foo", True) in dataCollectorUsage.data["allDataCollectors"] assert ( - "Business Data", - "in_snapshot_not_analytics", - False, - ) in dataCollectorUsage.data["allDataCollectors"] + "Business Data", + "in_snapshot_not_analytics", + False, + ) in dataCollectorUsage.data["allDataCollectors"] assert ( - "Business Data", - "in_snapshot_and_analytics", - True, - ) in dataCollectorUsage.data["allDataCollectors"] + "Business Data", + "in_snapshot_and_analytics", + True, + ) in dataCollectorUsage.data["allDataCollectors"] - assert not ("HTTP Parameter", "bar", True) in dataCollectorUsage.data["dataCollectorsPresentInSnapshots"] + assert not ("HTTP Parameter", "foo", True) in dataCollectorUsage.data["dataCollectorsPresentInSnapshots"] assert ( - "Business Data", - "in_snapshot_not_analytics", - False, - ) in dataCollectorUsage.data["dataCollectorsPresentInSnapshots"] + "Business Data", + "in_snapshot_not_analytics", + False, + ) in dataCollectorUsage.data["dataCollectorsPresentInSnapshots"] assert ( - "Business Data", - "in_snapshot_and_analytics", - True, - ) in dataCollectorUsage.data["dataCollectorsPresentInSnapshots"] + "Business Data", + "in_snapshot_and_analytics", + True, + ) in dataCollectorUsage.data["dataCollectorsPresentInSnapshots"] assert not ("HTTP Parameter", "foo", True) in dataCollectorUsage.data["dataCollectorsPresentInAnalytics"] assert not ("Business Data", "in_snapshot_not_analytics", False) in dataCollectorUsage.data["dataCollectorsPresentInAnalytics"] assert ( - "Business Data", - "in_snapshot_and_analytics", - True, - ) in dataCollectorUsage.data["dataCollectorsPresentInAnalytics"] - - await controller.close() + "Business Data", + "in_snapshot_and_analytics", + True, + ) in dataCollectorUsage.data["dataCollectorsPresentInAnalytics"] + await appdService.close() +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) @pytest.mark.asyncio -async def testGetAnalyticsEnabledStatusForAllApplications(controller): - assert (await controller.loginToController()).error is None +async def testGetAnalyticsEnabledStatusForAllApplications(appdServiceFactory, + auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - analyticsEnabledStatusList = await controller.getAnalyticsEnabledStatusForAllApplications() + analyticsEnabledStatusList = await appdService.getAnalyticsEnabledStatusForAllApplications() assert analyticsEnabledStatusList.error is None assert len(analyticsEnabledStatusList.data) > 0 @@ -440,14 +501,16 @@ async def testGetAnalyticsEnabledStatusForAllApplications(controller): assert "applicationId" in status assert "enabled" in status - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetDashboards(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetDashboards(appdServiceFactory, auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - dashboards = await controller.getDashboards() + dashboards = await appdService.getDashboards() assert dashboards.error is None assert len(dashboards.data) > 0 @@ -458,53 +521,46 @@ async def testGetDashboards(controller): assert "createdOn" in dashboard assert "modifiedOn" in dashboard - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetUserPermissions(controller): - assert (await controller.loginToController()).error is None - - userPermissions = await controller.getUserPermissions(USERNAME) +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetUserPermissions(appdServiceFactory,auth_method): + '''authenticate validates required permissions as well''' + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None + await appdService.close() - assert userPermissions.error is None - assert len(userPermissions.data) > 0 - - assert "name" in userPermissions.data - assert "id" in userPermissions.data - assert "version" in userPermissions.data - - assert "roles" in userPermissions.data - for role in userPermissions.data["roles"]: - assert "name" in role - assert "id" in role - assert "version" in role - - adminRole = next( - (role for role in userPermissions.data["roles"] if role["name"] == "super-admin"), - None, - ) - assert adminRole is not None + appdService = appdServiceFactory(auth_method="secret") + assert (await appdService.getAuthMethod().authenticate()).error is None + await appdService.close() - await controller.close() + appdService = appdServiceFactory(auth_method="token") + assert (await appdService.getAuthMethod().authenticate()).error is None + await appdService.close() @pytest.mark.asyncio -async def testGetCustomMetrics(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetCustomMetrics(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - customMetrics = await controller.getCustomMetrics(APPLICATION_ID, "api-services") + customMetrics = await appdService.getCustomMetrics(APPLICATION_ID, "api-services") assert customMetrics.error is None - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetAccountUsageSummary(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetAccountUsageSummary(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - accountUsageSummary = await controller.getAccountUsageSummary() + accountUsageSummary = await appdService.getAccountUsageSummary() assert accountUsageSummary.error is None assert len(accountUsageSummary.data) > 0 @@ -516,14 +572,16 @@ async def testGetAccountUsageSummary(controller): assert "numOfProvisionedLicense" in licenseData assert "expirationDate" in licenseData - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetAppServerAgents(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetAppServerAgents(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - agents = await controller.getAppServerAgents() + agents = await appdService.getAppServerAgents() assert agents.error is None assert len(agents.data) > 0 @@ -542,14 +600,16 @@ async def testGetAppServerAgents(controller): assert "machineId" in agent assert "type" in agent - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetMachineAgents(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetMachineAgents(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - agents = await controller.getMachineAgents() + agents = await appdService.getMachineAgents() assert agents.error is None assert len(agents.data) > 0 @@ -561,7 +621,7 @@ async def testGetMachineAgents(controller): assert "machineId" in agent assert "enabled" in agent - await controller.close() + await appdService.close() # @pytest.mark.asyncio @@ -613,10 +673,12 @@ async def testGetMachineAgents(controller): @pytest.mark.asyncio -async def testGetCustomMetrics(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetCustomMetrics(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - customMetrics = await controller.getCustomMetrics(APPLICATION_ID, "customer-services") + customMetrics = await appdService.getCustomMetrics(APPLICATION_ID, "customer-services") assert customMetrics.error is None assert len(customMetrics.data) > 0 @@ -631,14 +693,16 @@ async def testGetCustomMetrics(controller): assert "iconPath" in customMetric assert "hasChildren" in customMetric - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetEumApplications(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetEumApplications(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - eumApps = await controller.getEumApplications() + eumApps = await appdService.getEumApplications() assert eumApps.error is None assert len(eumApps.data) > 0 @@ -666,14 +730,16 @@ async def testGetEumApplications(controller): for metric in metrics: assert metric in eumApp["metrics"][folder] - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetEumPageListViewData(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetEumPageListViewData(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - eumPageListViewData = await controller.getEumPageListViewData(EUM_APPLICATION_ID) + eumPageListViewData = await appdService.getEumPageListViewData(EUM_APPLICATION_ID) assert eumPageListViewData.error is None assert len(eumPageListViewData.data) > 0 @@ -683,14 +749,16 @@ async def testGetEumPageListViewData(controller): assert "id" in eumPageListViewData.data["application"] assert "name" in eumPageListViewData.data["application"] - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetPagesAndFramesConfig(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetPagesAndFramesConfig(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - pagesAndFramesConfig = await controller.getPagesAndFramesConfig(EUM_APPLICATION_ID) + pagesAndFramesConfig = await appdService.getPagesAndFramesConfig(EUM_APPLICATION_ID) assert pagesAndFramesConfig.error is None @@ -717,14 +785,16 @@ async def testGetPagesAndFramesConfig(controller): assert "matchOnUserAgentType" in config assert "matchOnUserAgent" in config - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetAJAXConfig(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetAJAXConfig(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - ajaxConfig = await controller.getAJAXConfig(EUM_APPLICATION_ID) + ajaxConfig = await appdService.getAJAXConfig(EUM_APPLICATION_ID) assert ajaxConfig.error is None @@ -772,14 +842,16 @@ async def testGetAJAXConfig(controller): assert "matchOnURL" in config assert "matchBy" in config - await controller.close() + await appdService.close() @pytest.mark.asyncio -async def testGetVirtualPagesConfig(controller): - assert (await controller.loginToController()).error is None +@pytest.mark.parametrize("auth_method", ["basic", "secret", "token"]) +async def testGetVirtualPagesConfig(appdServiceFactory,auth_method): + appdService = appdServiceFactory(auth_method) + assert (await appdService.getAuthMethod().authenticate()).error is None - virtualPagesConfig = await controller.getVirtualPagesConfig(EUM_APPLICATION_ID) + virtualPagesConfig = await appdService.getVirtualPagesConfig(EUM_APPLICATION_ID) assert virtualPagesConfig.error is None @@ -806,4 +878,4 @@ async def testGetVirtualPagesConfig(controller): assert "matchOnUserAgentType" in config assert "matchOnUserAgent" in config - await controller.close() + await appdService.close()