diff --git a/app_builder_import_file/sigsci_TA_for_splunk-1.0.36.spl b/app_builder_import_file/sigsci_TA_for_splunk-1.0.36.spl new file mode 100644 index 0000000..73e237b Binary files /dev/null and b/app_builder_import_file/sigsci_TA_for_splunk-1.0.36.spl differ diff --git a/app_builder_import_file/sigsci_TA_for_splunk-1_0_36_export.tgz b/app_builder_import_file/sigsci_TA_for_splunk-1_0_36_export.tgz new file mode 100644 index 0000000..183cb55 Binary files /dev/null and b/app_builder_import_file/sigsci_TA_for_splunk-1_0_36_export.tgz differ diff --git a/sigsci_TA_for_splunk_1_0_35.tgz b/app_builder_import_file/sigsci_TA_for_splunk_1_0_35.tgz similarity index 100% rename from sigsci_TA_for_splunk_1_0_35.tgz rename to app_builder_import_file/sigsci_TA_for_splunk_1_0_35.tgz diff --git a/sigsci_TA_for_splunk-1.0.36.tgz b/sigsci_TA_for_splunk-1.0.36.tgz new file mode 100644 index 0000000..183cb55 Binary files /dev/null and b/sigsci_TA_for_splunk-1.0.36.tgz differ diff --git a/sigsci_TA_for_splunk/README.txt b/sigsci_TA_for_splunk/README.txt index 719e6ca..5957df5 100644 --- a/sigsci_TA_for_splunk/README.txt +++ b/sigsci_TA_for_splunk/README.txt @@ -10,3 +10,25 @@ This is an add-on powered by the Splunk Add-on Builder. /opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-arm64.exe: this file does not require any source code /opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-arm64.exe: this file does not require any source code /opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code +# Binary File Declaration +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/pvectorc.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-32.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-arm64.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-arm64.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-64.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-32.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-64.exe: this file does not require any source code +# Binary File Declaration +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/pvectorc.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-32.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-arm64.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-arm64.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-64.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-32.exe: this file does not require any source code +/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-64.exe: this file does not require any source code diff --git a/sigsci_TA_for_splunk/README/inputs.conf.spec b/sigsci_TA_for_splunk/README/inputs.conf.spec index bcf99ca..24169b9 100644 --- a/sigsci_TA_for_splunk/README/inputs.conf.spec +++ b/sigsci_TA_for_splunk/README/inputs.conf.spec @@ -1,8 +1,8 @@ -[SigsciActivity://] -place_holder = It was required to have one option even if it isn't needed. You can skip this one. - [SigsciEvent://] site_api_name = This is the Site API Name. It should not be a URL. [SigsciRequests://] -site_api_name = This is the API Name of the site to pull date from. This should not be a URL. \ No newline at end of file +site_api_name = This is the API Name of the site to pull request data from. This should not be a URL. + +[SigsciActivity://] +place_holder = It was required to have one option even if it isn't needed. You can skip this one. \ No newline at end of file diff --git a/sigsci_TA_for_splunk/app.manifest b/sigsci_TA_for_splunk/app.manifest index eec123c..691f870 100644 --- a/sigsci_TA_for_splunk/app.manifest +++ b/sigsci_TA_for_splunk/app.manifest @@ -5,7 +5,7 @@ "id": { "group": null, "name": "sigsci_TA_for_splunk", - "version": "1.0.35" + "version": "1.0.36" }, "author": [ { diff --git a/sigsci_TA_for_splunk/appserver/static/js/build/globalConfig.json b/sigsci_TA_for_splunk/appserver/static/js/build/globalConfig.json index 011a111..ef3a890 100644 --- a/sigsci_TA_for_splunk/appserver/static/js/build/globalConfig.json +++ b/sigsci_TA_for_splunk/appserver/static/js/build/globalConfig.json @@ -243,13 +243,13 @@ "field": "disabled", "label": "Status" }, - { - "field": "place_holder", - "label": "Place Holder" - }, { "field": "site_api_name", "label": "Site API Name" + }, + { + "field": "place_holder", + "label": "Place Holder" } ], "actions": [ @@ -261,8 +261,8 @@ }, "services": [ { - "name": "SigsciActivity", - "title": "SigsciActivity", + "name": "SigsciEvent", + "title": "SigsciEvent", "entity": [ { "field": "name", @@ -319,10 +319,10 @@ ] }, { - "field": "place_holder", - "label": "Place Holder", - "help": "It was required to have one option even if it isn't needed. You can skip this one.", - "required": false, + "field": "site_api_name", + "label": "Site API Name", + "help": "This is the Site API Name. It should not be a URL.", + "required": true, "type": "text", "validators": [ { @@ -331,16 +331,13 @@ "maxLength": 8192, "errorMsg": "Max length of text input is 8192" } - ], - "options": { - "placeholder": "Not needed" - } + ] } ] }, { - "name": "SigsciEvent", - "title": "SigsciEvent", + "name": "SigsciRequests", + "title": "SigsciRequests", "entity": [ { "field": "name", @@ -399,7 +396,7 @@ { "field": "site_api_name", "label": "Site API Name", - "help": "This is the Site API Name. It should not be a URL.", + "help": "This is the API Name of the site to pull request data from. This should not be a URL.", "required": true, "type": "text", "validators": [ @@ -414,8 +411,8 @@ ] }, { - "name": "SigsciRequests", - "title": "SigsciRequests", + "name": "SigsciActivity", + "title": "SigsciActivity", "entity": [ { "field": "name", @@ -472,10 +469,10 @@ ] }, { - "field": "site_api_name", - "label": "Site API Name", - "help": "This is the API Name of the site to pull date from. This should not be a URL.", - "required": true, + "field": "place_holder", + "label": "Place Holder", + "help": "It was required to have one option even if it isn't needed. You can skip this one.", + "required": false, "type": "text", "validators": [ { @@ -484,7 +481,10 @@ "maxLength": 8192, "errorMsg": "Max length of text input is 8192" } - ] + ], + "options": { + "placeholder": "Not needed" + } } ] } diff --git a/sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py b/sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py index d518e51..e612c05 100644 --- a/sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py +++ b/sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py @@ -4,12 +4,12 @@ from datetime import datetime from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time -''' +""" IMPORTANT Edit only the validate_input and collect_events functions. Do not edit any other part in this file. This file is generated only once when creating the modular input. -''' +""" # def use_single_instance_mode(): @@ -30,7 +30,7 @@ def collect_events(helper, ew): global_email = helper.get_global_setting("email") global_api_token = helper.get_global_setting("api_token") global_corp_api_name = helper.get_global_setting("corp_api_name") - api_host = 'https://dashboard.signalsciences.net' + api_host = "https://dashboard.signalsciences.net" helper.log_info("email: %s" % global_email) helper.log_info("corp: %s" % global_corp_api_name) @@ -38,15 +38,13 @@ def pull_events(delta, key=None): last_run_until = helper.get_check_point("activity_last_until_time") helper.log_info(f"last_run_until: {last_run_until}") if last_run_until is None: - ( - until_time, - from_time - ) = get_from_and_until_times(delta, five_min_offset=False) + (until_time, from_time) = get_from_and_until_times( + delta, five_min_offset=False + ) else: - ( - until_time, - from_time - ) = get_until_time(last_run_until, delta, five_min_offset=False) + (until_time, from_time) = get_until_time( + helper, last_run_until, delta, five_min_offset=False + ) if from_time is None: helper.log_info(f"{last_run_until} >= current now time, skipping run") return @@ -88,13 +86,13 @@ def pull_events(delta, key=None): until_time=until_time, global_email=global_email, global_corp_api_name=global_corp_api_name, - current_site='', + current_site="", ) config.headers = { - 'Content-type': 'application/json', - 'x-api-user': global_email, - 'x-api-token': global_api_token, - 'User-Agent': config.user_agent_string + "Content-type": "application/json", + "x-api-user": global_email, + "x-api-token": global_api_token, + "User-Agent": config.user_agent_string, } helper.log_info("Pulling results from Corp Activity API") all_events = get_results("Activity Events", helper, config) @@ -122,7 +120,7 @@ def pull_events(delta, key=None): source=single_name, index=helper.get_output_index(), sourcetype=source_type, - data=current_event + data=current_event, ) else: indexes = helper.get_output_index() @@ -139,7 +137,7 @@ def pull_events(delta, key=None): source=single_name, index=current_index, sourcetype=source_type, - data=current_event + data=current_event, ) try: @@ -149,13 +147,11 @@ def pull_events(delta, key=None): write_end = timer() write_time = write_end - write_start write_time_result = round(write_time, 2) - helper.log_info( - f"Total Corp Activity Output Time: {write_time_result} seconds" - ) + helper.log_info(f"Total Corp Activity Output Time: {write_time_result} seconds") # If multiple inputs configured it creates an array of values and the # script only gets called once per Input configuration - time_deltas = helper.get_arg('interval') + time_deltas = helper.get_arg("interval") helper.log_info(f"interval: {time_deltas}") if type(time_deltas) is dict: helper.log_info("run_type: Sequential") @@ -163,16 +159,11 @@ def pull_events(delta, key=None): time_delta = time_deltas[active_input] time_delta = int(time_delta) helper.log_info("time_delta: %s" % time_delta) - pull_events( - delta=time_delta, - key=active_input - ) + pull_events(delta=time_delta, key=active_input) else: helper.log_info("Run Type: Concurrent") helper.log_info("time_delta: %s" % time_deltas) - pull_events( - delta=int(time_deltas) - ) + pull_events(delta=int(time_deltas)) helper.log_info("Finished Pulling Corp Activity") end = timer() total_time = end - start diff --git a/sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py b/sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py index 2258f78..d820ea3 100644 --- a/sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py +++ b/sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py @@ -5,12 +5,12 @@ from datetime import datetime from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time -''' +""" IMPORTANT Edit only the validate_input and collect_events functions. Do not edit any other part in this file. This file is generated only once when creating the modular input. -''' +""" # def use_single_instance_mode(): @@ -19,32 +19,23 @@ def validate_input(helper, definition): # This example accesses the modular input variable - site_name = definition.parameters.get('site_api_name', None) + site_name = definition.parameters.get("site_api_name", None) if site_name is None or site_name == "": msg = "The site_name can not be empty" - raise ValueError( - "InvalidSiteName", - msg - ) + raise ValueError("InvalidSiteName", msg) elif "http" in site_name: msg = ( "The site name is not the full URL it should be the ", - "API Name of the site like 'my_example_site'" - ) - raise ValueError( - "InvalidSiteName", - msg + "API Name of the site like 'my_example_site'", ) + raise ValueError("InvalidSiteName", msg) elif " " in site_name: msg = ( "The site name should be the API Name of the site like ", "not the Display Name. Example would be 'my_site_name' instead of ", - "My Site Name" - ) - raise ValueError( - "InvalidSiteName", - msg + "My Site Name", ) + raise ValueError("InvalidSiteName", msg) pass @@ -56,7 +47,7 @@ def collect_events(helper, ew): global_email = helper.get_global_setting("email") global_api_token = helper.get_global_setting("api_token") global_corp_api_name = helper.get_global_setting("corp_api_name") - api_host = 'https://dashboard.signalsciences.net' + api_host = "https://dashboard.signalsciences.net" helper.log_info("email: %s" % global_email) helper.log_info("corp: %s" % global_corp_api_name) @@ -66,15 +57,13 @@ def pull_events(current_site, delta, key=None): last_run_until = helper.get_check_point(last_name) helper.log_info(f"last_run_until: {last_run_until}") if last_run_until is None: - ( - until_time, - from_time - ) = get_from_and_until_times(delta, five_min_offset=False) + (until_time, from_time) = get_from_and_until_times( + delta, five_min_offset=False + ) else: - ( - until_time, - from_time - ) = get_until_time(last_run_until, delta, five_min_offset=False) + (until_time, from_time) = get_until_time( + helper, last_run_until, delta, five_min_offset=False + ) if from_time is None or from_time > until_time: helper.log_info(f"{from_time} >= current now time, skipping run") return @@ -121,20 +110,17 @@ def pull_events(current_site, delta, key=None): current_site=current_site, ) config.headers = { - 'Content-type': 'application/json', - 'x-api-user': global_email, - 'x-api-token': global_api_token, - 'User-Agent': config.user_agent_string + "Content-type": "application/json", + "x-api-user": global_email, + "x-api-token": global_api_token, + "User-Agent": config.user_agent_string, } all_events = get_results("Events", helper, config) total_requests = len(all_events) helper.log_info("Total Events Pulled: %s" % total_requests) write_start = timer() for current_event in all_events: - helper.log_info(type(current_event)) helper.log_debug(current_event) - helper.log_info(f"json: {current_event}") - helper.log_info(type(current_event)) if key is None: source_type = helper.get_sourcetype() helper.log_info("Concurrent Mode") @@ -155,7 +141,7 @@ def pull_events(current_site, delta, key=None): source=single_name, index=helper.get_output_index(), sourcetype=source_type, - data=current_event + data=current_event, ) else: indexes = helper.get_output_index() @@ -172,7 +158,7 @@ def pull_events(current_site, delta, key=None): source=single_name, index=current_index, sourcetype=source_type, - data=current_event + data=current_event, ) try: @@ -182,14 +168,13 @@ def pull_events(current_site, delta, key=None): write_end = timer() write_time = write_end - write_start write_time_result = round(write_time, 2) - helper.log_info("Total Event Output Time: %s seconds" - % write_time_result) + helper.log_info("Total Event Output Time: %s seconds" % write_time_result) # If multiple inputs configured it creates an array of values and the # script only gets called once per Input configuration - all_sites = helper.get_arg('site_api_name') - time_deltas = helper.get_arg('interval') + all_sites = helper.get_arg("site_api_name") + time_deltas = helper.get_arg("interval") helper.log_info(f"interval: {time_deltas}") if type(all_sites) is dict: helper.log_info("run_type: Sequential") @@ -197,22 +182,15 @@ def pull_events(current_site, delta, key=None): site = all_sites[active_input] current_delta = int(time_deltas[active_input]) helper.log_info("site: %s" % site) - pull_events( - key=active_input, - current_site=site, - delta=current_delta - ) + pull_events(key=active_input, current_site=site, delta=current_delta) helper.log_info("Finished Pulling Events for %s" % site) else: helper.log_info("Run Type: Concurrent") - site = helper.get_arg('site_api_name') + site = helper.get_arg("site_api_name") helper.log_info("site: %s" % site) - pull_events( - current_site=site, - delta=int(time_deltas) - ) + pull_events(current_site=site, delta=int(time_deltas)) helper.log_info("Finished Pulling Events for %s" % site) end = timer() total_time = end - start time_result = round(total_time, 2) - helper.log_info("Total Script Time: %s seconds" % time_result) \ No newline at end of file + helper.log_info("Total Script Time: %s seconds" % time_result) diff --git a/sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py b/sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py index 6381859..c66d7de 100644 --- a/sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py +++ b/sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py @@ -1,15 +1,14 @@ # encoding = utf-8 from timeit import default_timer as timer -from datetime import datetime +from datetime import datetime, timezone, timedelta from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time -''' +""" IMPORTANT Edit only the validate_input and collect_events functions. Do not edit any other part in this file. This file is generated only once when creating the modular input. -''' - +""" # def use_single_instance_mode(): # return True @@ -17,32 +16,23 @@ def validate_input(helper, definition): # This example accesses the modular input variable - site_name = definition.parameters.get('site_api_name', None) + site_name = definition.parameters.get("site_api_name", None) if site_name is None or site_name == "": msg = "The site_name can not be empty" - raise ValueError( - "InvalidSiteName", - msg - ) + raise ValueError("InvalidSiteName", msg) elif "http" in site_name: msg = ( "The site name is not the full URL it should be the ", - "API Name of the site like 'my_example_site'" - ) - raise ValueError( - "InvalidSiteName", - msg + "API Name of the site like 'my_example_site'", ) + raise ValueError("InvalidSiteName", msg) elif " " in site_name: msg = ( "The site name should be the API Name of the site like ", "not the Display Name. Example would be 'my_site_name' instead of ", - "My Site Name" - ) - raise ValueError( - "InvalidSiteName", - msg + "My Site Name", ) + raise ValueError("InvalidSiteName", msg) pass @@ -55,36 +45,41 @@ def collect_events(helper, ew): global_email = helper.get_global_setting("email") global_api_token = helper.get_global_setting("api_token") global_corp_api_name = helper.get_global_setting("corp_api_name") - api_host = 'https://dashboard.signalsciences.net' + api_host = "https://dashboard.signalsciences.net" helper.log_info("email: %s" % global_email) helper.log_info("corp: %s" % global_corp_api_name) - def pull_requests(current_site, delta, key=None): + def pull_requests(helper, current_site, delta, key=None): site_name = current_site last_name = f"requests_last_until_time_{current_site}" last_run_until = helper.get_check_point(last_name) - helper.log_info(f"last_run_until: {last_run_until}") - if last_run_until is None: - helper.log_info("get_from_until") - until_time, from_time = get_from_and_until_times(delta, five_min_offset=True) + helper.log_info("no last_run_time found in checkpoint state") + helper.log_debug("get_from_until") + until_time, from_time = get_from_and_until_times( + delta, five_min_offset=True + ) else: - helper.log_info("get_until") - until_time, from_time = get_until_time(last_run_until, delta, five_min_offset=True) + helper.log_info(f"last_run_until found in state: {last_run_until}") + helper.log_debug("get_until") + until_time, from_time = get_until_time( + helper, last_run_until, delta, five_min_offset=True + ) if from_time is None: helper.log_info(f"{last_run_until} >= current now time, skipping run") return - + if from_time >= until_time: helper.save_check_point(last_name, from_time) - helper.log_info(f"from_time {from_time} >= until_time {until_time}, skipping run") + helper.log_info( + f"from_time {from_time} >= until_time {until_time}, skipping run" + ) return - helper.log_info("SiteName: %s" % site_name) - helper.log_info(f"Start Period: {datetime.fromtimestamp(from_time)} UTC") - helper.log_info(f"End Period: {datetime.fromtimestamp(until_time)} UTC") + helper.log_info(f"Start Period: {datetime.utcfromtimestamp(from_time)} UTC") + helper.log_info(f"End Period: {datetime.utcfromtimestamp(until_time)} UTC") input_name = helper.get_input_stanza_names() single_name = "" @@ -106,7 +101,7 @@ def pull_requests(current_site, delta, key=None): f"/sites/{site_name}/feed/requests?" f"from={from_time}&until={until_time}" ) - helper.log_info("Pulling requests from requests API") + helper.log_info("Pulling requests from requests API") config = Config( url=url, api_host=api_host, @@ -117,16 +112,21 @@ def pull_requests(current_site, delta, key=None): current_site=current_site, ) config.headers = { - 'Content-type': 'application/json', - 'x-api-user': global_email, - 'x-api-token': global_api_token, - 'User-Agent': config.user_agent_string + "Content-type": "application/json", + "x-api-user": global_email, + "x-api-token": global_api_token, + "User-Agent": config.user_agent_string, } all_requests = get_results("Requests", helper, config) total_requests = len(all_requests) helper.log_info("Total Requests Pulled: %s" % total_requests) + if total_requests == 0: + helper.save_check_point(last_name, until_time) + helper.log_info( + f"No events to write, saving checkpoint to value:{until_time}" + ) write_start = timer() for current_event in all_requests: if key is None: @@ -135,7 +135,7 @@ def pull_requests(current_site, delta, key=None): source=single_name, index=helper.get_output_index(), sourcetype=source_type, - data=current_event + data=current_event, ) else: indexes = helper.get_output_index() @@ -147,16 +147,18 @@ def pull_requests(current_site, delta, key=None): source=single_name, index=current_index, sourcetype=source_type, - data=current_event + data=current_event, ) try: ew.write_event(event) helper.save_check_point(last_name, until_time) + helper.log_info(f"Event written, saving checkpoint:{until_time}") except Exception as e: + helper.log_error(f"error writing event: {e}") helper.log_error(event) raise e - + write_end = timer() write_time = write_end - write_start write_time_result = round(write_time, 2) @@ -164,8 +166,8 @@ def pull_requests(current_site, delta, key=None): # If multiple inputs configured it creates an array of values and the # script only gets called once per Input configuration - all_sites = helper.get_arg('site_api_name') - time_deltas = helper.get_arg('interval') + all_sites = helper.get_arg("site_api_name") + time_deltas = helper.get_arg("interval") helper.log_info(f"interval: {time_deltas}") if type(all_sites) is dict: @@ -173,22 +175,15 @@ def pull_requests(current_site, delta, key=None): for active_input, site in all_sites.items(): time_delta = int(time_deltas[active_input]) helper.log_info("site: %s" % site) - pull_requests( - key=active_input, - current_site=site, - delta=time_delta - ) + pull_requests(helper, key=active_input, current_site=site, delta=time_delta) helper.log_info("Finished Pulling Requests for %s" % site) else: helper.log_info("Run Type: Concurrent") - site = helper.get_arg('site_api_name') + site = helper.get_arg("site_api_name") helper.log_info("site: %s" % site) - pull_requests( - current_site=site, - delta=int(time_deltas) - ) + pull_requests(helper, current_site=site, delta=int(time_deltas)) helper.log_info("Finished Pulling Requests for %s" % site) end = timer() total_time = end - start time_result = round(total_time, 2) - helper.log_info("Total Script Time: %s seconds" % time_result) \ No newline at end of file + helper.log_info("Total Script Time: %s seconds" % time_result) diff --git a/sigsci_TA_for_splunk/bin/sigsci_helper.py b/sigsci_TA_for_splunk/bin/sigsci_helper.py index 9b74653..be059c7 100644 --- a/sigsci_TA_for_splunk/bin/sigsci_helper.py +++ b/sigsci_TA_for_splunk/bin/sigsci_helper.py @@ -1,18 +1,18 @@ import json -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from timeit import default_timer as timer import time import requests def check_response( - code, - response_text, - global_email, - global_corp_api_name, - from_time=None, - until_time=None, - current_site=None + code, + response_text, + global_email, + global_corp_api_name, + from_time=None, + until_time=None, + current_site=None, ): success = False base_msg = { @@ -35,8 +35,9 @@ def check_response( base_msg["error"] = "Internal Server Error" base_msg["msg"] = "internal-error" elif code == 401: - base_msg["error"] = "Unauthorized. Incorrect credentials or lack " \ - "of permissions" + base_msg["error"] = ( + "Unauthorized. Incorrect credentials or lack of permissions" + ) base_msg["msg"] = "unauthorized" elif 400 <= code <= 599 and code != 400 and code != 500 and code != 401: base_msg["error"] = "Unknown Error" @@ -59,23 +60,19 @@ def get_request_data(url, headers, helper): verify=True, cert=None, timeout=None, - use_proxy=True + use_proxy=True, ) response_code = response_raw.status_code response_error = response_raw.text try: data = json.loads(response_raw.text) except Exception as error: - data = { - "data": [] - } + data = {"data": []} helper.log_info("Unable to parse API Response") helper.log_error(response_error) helper.log_error(error) except Exception as error: - data = { - "data": [] - } + data = {"data": []} helper.log_info("Unable to parse API Response") helper.log_error(error) response_code = 500 @@ -84,43 +81,65 @@ def get_request_data(url, headers, helper): return data, response_code, response_error +def timestamp_sanitise(_time): + new_time = datetime.utcfromtimestamp(_time).replace(second=0) + new_time = int(new_time.timestamp()) + return new_time + + def get_from_and_until_times(delta, five_min_offset=False): if five_min_offset: - until_time = datetime.utcnow() - timedelta(minutes=5) + until_time = datetime.now(timezone.utc) - timedelta(minutes=5) else: - until_time = datetime.utcnow() + until_time = datetime.now(timezone.utc) from_time = until_time - timedelta(seconds=delta) + if five_min_offset: until_time = until_time.replace(second=0) from_time = from_time.replace(second=0) + until_time = int(until_time.timestamp()) from_time = int(from_time.timestamp()) - return until_time, from_time -def timestamp_sanitise(_time): - new_time = datetime.fromtimestamp(_time).replace(second=0) - new_time = int(new_time.timestamp()) - return new_time + return timestamp_sanitise(until_time), timestamp_sanitise(from_time) + + +def get_until_time(helper, from_time, interval, five_min_offset=False): + now = datetime.now(timezone.utc).replace(second=0) + interval_timedelta = timedelta(seconds=interval) + current_time_offset = datetime.now(timezone.utc).replace(second=0) -def get_until_time(from_time, delta, five_min_offset=False): if five_min_offset: - current_time_offset = datetime.utcnow() - timedelta(minutes=5) - current_time_offset = current_time_offset.replace(second=0) - else: - current_time_offset = datetime.utcnow() - current_time_offset = int(datetime.timestamp(current_time_offset)) - - new_until_time = from_time + delta - new_until_time = timestamp_sanitise(new_until_time) - new_from_time = timestamp_sanitise(from_time) - - if from_time > current_time_offset: - return None, None - - if new_until_time > current_time_offset: - new_until_time = current_time_offset - return new_until_time, new_from_time + current_time_offset = current_time_offset - timedelta(minutes=5) + + # Get dt object with UTC timezone. + from_time_dt_obj = ( + datetime.utcfromtimestamp(from_time) + .replace(tzinfo=timezone.utc) + .replace(second=0) + ) + + # How far back is the "from time" from now + ft_diff = now - from_time_dt_obj + + # The default time to look at until, either now or five minutes ago. + _until_time = current_time_offset + _rslt_until = int(datetime.timestamp(_until_time)) + + # If we are futher back than 24 hours reset the clock. + if ft_diff > timedelta(hours=24): + helper.log_info("Adjusting from_time to 24 hours ago") + + from_time = now - timedelta(hours=24) + from_time.replace(second=0) + from_time_int = int(from_time.timestamp()) + + return _rslt_until, from_time_int + + from_time_int = int(datetime.timestamp(from_time_dt_obj)) + + return _rslt_until, from_time_int def get_results(title, helper, config): @@ -130,8 +149,9 @@ def get_results(title, helper, config): pulled_events = [] helper.log_info("Processing page %s" % counter) start_page = timer() - response_result, response_code, response_error = \ - get_request_data(config.url, config.headers, helper) + response_result, response_code, response_error = get_request_data( + config.url, config.headers, helper + ) pulled, request_details = check_response( response_code, @@ -140,7 +160,7 @@ def get_results(title, helper, config): global_corp_api_name=config.global_corp_api_name, current_site=config.current_site, from_time=config.from_time, - until_time=config.until_time + until_time=config.until_time, ) if not pulled and request_details["msg"] != "rate-limit": @@ -155,40 +175,33 @@ def get_results(title, helper, config): else: response = response_result - number_requests_per_page = len(response['data']) - helper.log_info( - f"Number of {title} for Page: {number_requests_per_page}" - ) + number_requests_per_page = len(response["data"]) + helper.log_info(f"Number of {title} for Page: {number_requests_per_page}") - for data in response['data']: - + for data in response["data"]: event_id = data["id"] if event_id not in config.event_ids: config.event_ids.append(event_id) else: continue - if 'headersOut' in data: - headers_out = data['headersOut'] + if "headersOut" in data: + headers_out = data["headersOut"] if headers_out is not None: new_header_out = [] for header in headers_out: - header_data = { - header[0]: header[1] - } + header_data = {header[0]: header[1]} new_header_out.append(header_data) - data['headersOut'] = new_header_out + data["headersOut"] = new_header_out - if 'headersIn' in data: - headers_in = data['headersIn'] + if "headersIn" in data: + headers_in = data["headersIn"] if headers_in is not None: new_header_in = [] for header in headers_in: - header_data = { - header[0]: header[1] - } + header_data = {header[0]: header[1]} new_header_in.append(header_data) - data['headersIn'] = new_header_in + data["headersIn"] = new_header_in data = json.dumps(data) config.events.append(data) @@ -198,14 +211,12 @@ def get_results(title, helper, config): next_url = next_data.get("uri") else: next_url = None - if next_url == '': + if next_url == "": helper.log_info("Finished Page %s" % counter) end_page = timer() page_time = end_page - start_page page_time_result = round(page_time, 2) - helper.log_info( - f"Total Page Time: {page_time_result} seconds" - ) + helper.log_info(f"Total Page Time: {page_time_result} seconds") loop = False elif next_url is not None: config.url = config.api_host + next_url @@ -214,9 +225,7 @@ def get_results(title, helper, config): end_page = timer() page_time = end_page - start_page page_time_result = round(page_time, 2) - helper.log_info( - f"Total Page Time: {page_time_result} seconds" - ) + helper.log_info(f"Total Page Time: {page_time_result} seconds") else: loop = False return config.events @@ -237,15 +246,15 @@ class Config: event_ids: list def __init__( - self, - api_host=None, - url=None, - headers=None, - from_time=None, - until_time=None, - global_email=None, - global_corp_api_name=None, - current_site=None, + self, + api_host=None, + url=None, + headers=None, + from_time=None, + until_time=None, + global_email=None, + global_corp_api_name=None, + current_site=None, ): self.api_host = api_host self.url = url @@ -257,8 +266,8 @@ def __init__( self.global_corp_api_name = global_corp_api_name self.current_site = current_site self.event_ids = [] - self.user_agent_version = "1.0.35" + self.user_agent_version = "1.0.36" self.user_agent_string = ( f"TA-sigsci-waf/{self.user_agent_version} " f"(PythonRequests {requests.__version__})" - ) \ No newline at end of file + ) diff --git a/sigsci_TA_for_splunk/default/app.conf b/sigsci_TA_for_splunk/default/app.conf index 56b074e..cc34841 100644 --- a/sigsci_TA_for_splunk/default/app.conf +++ b/sigsci_TA_for_splunk/default/app.conf @@ -3,11 +3,11 @@ state_change_requires_restart = false is_configured = 0 state = enabled -build = 12 +build = 14 [launcher] author = Fastly -version = 1.0.35 +version = 1.0.36 description = For users of Fastly who would like to enrich their Splunk data with information from Fastly. This app with simple configuration enabled the importing of Events, Activity, and raw request information to Splunk.This is an open source project, no support provided, public repository is available and installation documentation can be found at https://github.com/fastly/sigsci-splunk-app. The best way to report issues with the app is to create an Issue on the github page so that it can be tracked. [ui] diff --git a/sigsci_TA_for_splunk/default/restmap.conf b/sigsci_TA_for_splunk/default/restmap.conf index 47d19ba..86b94ee 100644 --- a/sigsci_TA_for_splunk/default/restmap.conf +++ b/sigsci_TA_for_splunk/default/restmap.conf @@ -1,7 +1,7 @@ [admin:sigsci_TA_for_splunk] match = / -members = sigsci_TA_for_splunk_settings, sigsci_TA_for_splunk_SigsciActivity, sigsci_TA_for_splunk_SigsciEvent, sigsci_TA_for_splunk_SigsciRequests +members = sigsci_TA_for_splunk_settings, sigsci_TA_for_splunk_SigsciEvent, sigsci_TA_for_splunk_SigsciRequests, sigsci_TA_for_splunk_SigsciActivity [admin_external:sigsci_TA_for_splunk_settings] handlertype = python @@ -10,23 +10,23 @@ handlerfile = sigsci_TA_for_splunk_rh_settings.py handleractions = edit, list handlerpersistentmode = true -[admin_external:sigsci_TA_for_splunk_SigsciActivity] +[admin_external:sigsci_TA_for_splunk_SigsciEvent] handlertype = python python.version = python3 -handlerfile = sigsci_TA_for_splunk_rh_SigsciActivity.py +handlerfile = sigsci_TA_for_splunk_rh_SigsciEvent.py handleractions = edit, list, remove, create handlerpersistentmode = true -[admin_external:sigsci_TA_for_splunk_SigsciEvent] +[admin_external:sigsci_TA_for_splunk_SigsciRequests] handlertype = python python.version = python3 -handlerfile = sigsci_TA_for_splunk_rh_SigsciEvent.py +handlerfile = sigsci_TA_for_splunk_rh_SigsciRequests.py handleractions = edit, list, remove, create handlerpersistentmode = true -[admin_external:sigsci_TA_for_splunk_SigsciRequests] +[admin_external:sigsci_TA_for_splunk_SigsciActivity] handlertype = python python.version = python3 -handlerfile = sigsci_TA_for_splunk_rh_SigsciRequests.py +handlerfile = sigsci_TA_for_splunk_rh_SigsciActivity.py handleractions = edit, list, remove, create handlerpersistentmode = true diff --git a/sigsci_TA_for_splunk/default/web.conf b/sigsci_TA_for_splunk/default/web.conf index 1938aa4..295b44c 100644 --- a/sigsci_TA_for_splunk/default/web.conf +++ b/sigsci_TA_for_splunk/default/web.conf @@ -7,14 +7,6 @@ methods = POST, GET pattern = sigsci_TA_for_splunk_settings/* methods = POST, GET, DELETE -[expose:sigsci_TA_for_splunk_SigsciActivity] -pattern = sigsci_TA_for_splunk_SigsciActivity -methods = POST, GET - -[expose:sigsci_TA_for_splunk_SigsciActivity_specified] -pattern = sigsci_TA_for_splunk_SigsciActivity/* -methods = POST, GET, DELETE - [expose:sigsci_TA_for_splunk_SigsciEvent] pattern = sigsci_TA_for_splunk_SigsciEvent methods = POST, GET @@ -31,6 +23,14 @@ methods = POST, GET pattern = sigsci_TA_for_splunk_SigsciRequests/* methods = POST, GET, DELETE +[expose:sigsci_TA_for_splunk_SigsciActivity] +pattern = sigsci_TA_for_splunk_SigsciActivity +methods = POST, GET + +[expose:sigsci_TA_for_splunk_SigsciActivity_specified] +pattern = sigsci_TA_for_splunk_SigsciActivity/* +methods = POST, GET, DELETE + [expose:_splunkd_data] pattern = data/* methods = GET diff --git a/sigsci_TA_for_splunk/sigsci_TA_for_splunk.aob_meta b/sigsci_TA_for_splunk/sigsci_TA_for_splunk.aob_meta index 43fe3c4..b296fa7 100644 --- a/sigsci_TA_for_splunk/sigsci_TA_for_splunk.aob_meta +++ b/sigsci_TA_for_splunk/sigsci_TA_for_splunk.aob_meta @@ -1 +1 @@ -{"basic_builder": {"appname": "sigsci_TA_for_splunk", "friendly_name": "Signal Sciences WAF TA", "version": "1.0.35", "author": "Fastly", "description": "For users of Fastly who would like to enrich their Splunk data with information from Fastly. This app with simple configuration enabled the importing of Events, Activity, and raw request information to Splunk.\n\nThis is an open source project, no support provided, public repository is available and installation documentation can be found at https://github.com/fastly/sigsci-splunk-app. The best way to report issues with the app is to create an Issue on the github page so that it can be tracked.", "theme": "#ea3f23", "large_icon": "iVBORw0KGgoAAAANSUhEUgAAAEgAAABICAYAAABV7bNHAAAAAXNSR0IArs4c6QAAD+5JREFUeF7tXAuMVNUZ/s69d96wCwULGoWGiFofNT6KsVEEhYqiEJAKiq0ixSpLa1EKKqAGBXV5RAlugkYXUFCK5dHEtKXl0QqrrNhoVojBRxsUUEGW3Z2dnZn7ar6zc8bLPnBn791BTU+y2dmz5/7n/P+c/z//4ztXuK7rIteSySRM04QQAvF4HOFwWP6ntrZWDZH9kUhE/m1ZFhoaGuRn9vF/bOl0Gk1NTfKzpmkoLS2FdeQIxJQpEP/+d55WZz6IK6+EePllqLWSRigUQrdu3fLk1HoNw0AikZBrsG0b9fX1+TE9e/bMf25sbEQ2m5V/e/mrq6uDKKaAtNGjoVVVdUYuXz8zfDiwefP/BdSuFIstoKNHj+ZVjNuU25Vax21HdWPzbsdUKoVMJiP7uYW7d+/eSq2i0ShisZjsdxwHcqumUkg8+CD0PXsgPvkE4tixgnaS27cv3D59YF1yCRoXLZIqxbWycZ1UOdXUetnP9XINuq6jpKSklRqyg2pIc0K+OV6pG02DKJaApD06cgTdIhGIu++G9pe/FCQg87e/hV1Whoymwendu3gCamxszO8gGjPVKE1lv9vrV0aYv73jaeT5oxq/QdVIKzRxIvT16wsT0IwZsObNa3NNaqd652hv7rb6vettyXfRjLRa2LFjx9B98uTCBTR9OuwFC0D1ZfvenWJ+BWRNmwZr4cLiC6iYNohCouFzx46FvmlTYSr20EMwZ8/OHxDfKyNdf+gQjJoaGFu3IpROQ9uwAeKzzwoSkHPJJbAvuwx2NAr78ssRGjoU4V69vrunGD3rTFUVIo8/DvHmmzyLIdJpWvOCBNNqMI0/PflwGM4558B6+GG4w4bJ47lLjvkgPekYT8GdO+Fu3gxRWQlx+LA/YXTwafeUU+BOmgQxZAgwdCiO5cKcb0+oYVmIb9qE8J/+1Lxb6uo6yFrAw2IxuIMHI33ttUjfdhuMaNR/LOY4Tn7P03tmAEq/gJ6w8lTpCavGfhXE2pkMslu2IDpzJrS9ewPm1h8559xzYS5eDGPoUGjhsAxWvd42DwvVqJ4qavDyx0C805608fHHCK9YgfDq1QWHDf5YL+DpaBSZ22+HOXEinAsvLF6ooVVWIlJeDrF/fwGrPXlD3dNOg/W738GYOTPv4XtTOCeMxZLJZF7FGNBRvehuMzxQoQaNHRsDTH3xYugVFUBj48njuDMzGwacm2+G9dRTcHv3hmXbeSqKb3aQbxUayf6OnmLawYOIP/AAQn/9KzNlnVnit+IZ68orkVqyBCWXXppfj/+E2eHDKL3sMmj//a9/P+ZbICY3kUD6wAHEcob6hAL6plDDePdddBs/vmg+TbHkR98p+dJLsAYN6nw+CG+/jfj06dDee69Y6y7qPM6PfoSmJ59EaMyY9hNmtbW1eSNNa06DTOOc3r8foenTEdq4saiLLvZk1rBhwLPPQh84UPLNYoPyiZgtbdtINzQgPncuwqtWFXu9J2U+86ab0LhwIdwePTpW1Qjv2IHE2LHf6dOqIEnrOjLl5UhNmtRaQJZl5VWM9Szr4EGUnnVWQfQLGkw/64c/lD/OqafCGj8e1uDBELW1iF9/vSTlbNuG5Omng19UaOVK6B9/DBw6BPHFF116ita/9x4iZ58tzQwbQ5DjQo1YJILwjTdC27q1IJ47NFgIOMznjBkD/YorYJ1/PtKAjJHYetTWwr3zTkAIJJ9+Gna/frKfaRP+iLffhr1jB7S//Q36li1dIijmnJL/+AfsXEqmVVUjtn49otOmsTTaIZ47Osjt3x9N8+cjO3QoSvv2hchVOul/KAH1pE+Sm7c+m80v0luxbWLJqb4e4W3bEHvoIQj6ZUE2w0Dm3nuRmj1bUpUCUqGG09CA2LhxMHbuDG5KTUP21lthLVoEN1caVtu3vXCGk1No3oqKqqp4wwAtmYQ+YwZC69fLmltQjX5R+pVXIPr0kaqWP8Wc5cshZs2CCCqXo+tofOYZWLfdJr8J1VjVIPOMc1gHVwLzBo8s8PH/bCxSqmwhUxEtqxrhTZsQv+8+acMCafE43PnzIX7/++aCJ/NBLgEIf/gDtOeeC2QONxZDauVKmMOHS+CAqr6SOAEEFBD7ybASkDfnxIS8V0A8PJQ9UgLy5nC46xO//jXE558Hsn5z7FiIpUuRoh/EUCNUVYUEkReHDvmewO3VC3Z5OXSmQGmYc6VnRbhHjx6yn2p0nA3yoC0oRGWbvDbIW/ZuWdXIrl2LGBN3AfCAWAzJFSsQHzcO4uiRI26UxMvKfAsHug7zV7+CvXgxojmGiyWgZH094i++iMicOUAOyuKHofTjjyMyYwZE/WefufGpU6H/+c9+6MlnnYEDkdy9W6ZqVVpWAQIUcdodtbNoX1TuhWGOanT3Vb+ipf3nP8hu3470iBEQJSXy6FfqyTSxAlQkBg+GHkDs6PzsZ3A3boRwDx1y3fPPh/jqK/8Cqq6G9tOftgmg8mOkrXXrEJs9G9qXX8KuqJDq217p2dq/H0b//r55QTyOut27Ier27nVLzj3XN0Hn4ouhvfOOpNMWwqwzAuIuspctQ2jmTB5nkF/k7t2yLnai2rx1110wnn/eN0/JrVshUqtXu7GJE/0Ri0TQuHo1wqNGSTo0sNz2bFQnpW78m9gbqh37qSLKx1EqwjFUK+PoUYglS6AtWiTpuOPHI11ZmVcrRtxKDSViJIcV4lhz507ER4yQQvXTzHnzIKyRI1399df90AGTT03btyNz6qmSTlsAqkJOsfTOnQjdfz/0XbvkYwQu6AsWoIlBZY7pE9XmS5JJiFGjoL37ri++7Ouug7AHDXK16mp/hMaMgblsGZpyqLJCBSSSScQ2b4axdi1QUiJjQen46TqcyZOBRYugde8uHcaOCKiHrsOeMcO3mjn9+0M4/fq5fss3ZlkZrEceAXO9Sq2U6qiQQn0D7FeVE/7P3rMHiVGjWvtgQoB03Vmz5A5l81ZaFB2pfrkqTH4Ox4FWUYHwrFm+vni3X79gBEQQgTtgAERpKbQ1awoy0jqdsTagMAxwsWcPsoZxwlBD2SwFA3aefBLOa69By2ah1dT4ExCzCEHsIIm2YKr2ggvg7NghU5YKCKkw12ql9J7VLmM4ER0xAtobb7RmJJGAXVd3HC2vf8WTUnnbkk4OeSbGjm3GPxL256l9dUZSLqGEQQjI/OUvJQTFLC2FedVV0olTC6ZaKLA5F8lAVIUadAjDS5YgMm9eM0Oelr3/fqTmzJEnoELMUijKBtHh9KJcVUAb+uc/ET98GHZVFYzKys7IJf+M27NnQAK6+24JsEznMnGFGGknk0H3sjLo69blF+Zccw3qXnlF7sqOxmLHwYCjUThPPAHtscf8CYg2yB4yxNW2b/dFyL7qKpgrV8LKIb645b15H6VunIQ7QoUa9JVU3ifU0AC7pgbagAGwiYnOrchLi+OVWnH3ePNEqhLBxyKNjdAYPhWIpG0pBPessyDS8+e7kVwGrdNSikSQfucdRM87T5IIypMmrW/KB7U00vy7rqYGJddfXzDMryX/zGWJzI4dbviKKzotG/Vg5rnnEJky5VshoMYNG5qrMj5bE1FyyX373MRPfuI7D02fIbtvn1yS1y+hOnmB6MqXUeBtBThXqsPn2/NxvLS8PpG3n88agwZB+PSiuY5UdTXEsX373BK61Cyt+GyZJUuQuuMO36FGZxJmykjHdu1C9LrrfHICuD/4AdyaGojagwfdBOvTS5f6J5pIyLKJceGFrS6zKOJdlVGkgLSvvkLJxRdD5O6w+WHIHj0a+sqVEOlUytU3boQxaZLv6Jc1Leuaa5BhAaBvX7m+YqmYQ5+KWILVq307iFy3+eKLsG6+ubmq4TKPc8cdEO+/70fozc8y4p42DU0PPgg3HM7fOFSEg65qNE+pI/7MMzDo9+SucPlhxGVWYs0a1F90EYRpmq5DZOucOQgtXuyH7tfPUki/+Q3Sc+dC79Yt71VzgLqqqfJEyoCrfo6hc6j66fsoH4e+lfKe6VtJw57JwFi6FLEFCwLZOZzfmjoV7hNPyKtXQsFfQv/6F2L33tuMIguiaRqsESPkVs3k7riSrLrUxlOIQlGnl/eym7faQcdShS30ibyhBukZEyZA27w5MMwk8QJNCxfCHDlSlquOhwFPmIAQJwuwEVLS9PLLyF56qVS5QIy0acLg5bwhQwJHvpnDhiH5xz9KCcjSs/dCHaqrEbvhBnYGKCJ55xHmz3+O7C23QKeHmwNpeS+vGakUuIsZtGYGD4aTK1Uf5/uYJuwtWxDesAGhTZuCx2fHYsi++iokqEr5Y8ehXOvrEZk/H6Hy8mAFlKPG6NghZnnkSGTvuguRM87I56vrq6vRfdw4idoQf/87cM458impVp9+isi6dQixZn7gALd9l6zPnjABdmUlwrnUSbvXwkuuvhp6AJ7oCbnQdemMOQx0eRc+k0GEhpZGcu5cWIYB4803ob/1FsSnnwZmgNtbE8Hmx95/vzWAquWdVbml9+6F8YtfBOJdd8lXHTBRAsvttWthsVjY4q5u23c1HAfmqlUI33OP7xgtYF66hJy7bJm8ie1qWsevhTd98YW8dhCuqAgUf9MlHHaWaCSC7KRJ0ObNg8E7Z23dm89ms3mMoirGUc1koqqhARp9o1df7RLIW2f5Cuo5d/hwOC+8gCxVLJe/pjOqoDd0Rr/xroZ8kMfza699f4REICmzhR98IGV9wqsI3h1EidHDVWXhPIiJ8LxZs6CvWUPoZ1Bf4MmhEwrBufVWmMuX58MZag43Qiu+M5mvQw2u1ou09yLOZRhA8MCKFdAefRTiyy9PDnN+Z00k4Dz8MMTUqajz3FhStwzbRNp/02UWrsn7chN71SqI8nJoe/b4XW5Rn3fPPhsuAVGTJ8ud0uELdel0Om+k2yvttkyH6vRmKypg8GJdwJDhwKUWjSJ7++3ysMl6cEPKfHC+9tK30tx09EKdWrh6Q5Nr28ju2oUQ3+Tis8QbuFBUaHPmmbCefVZe7HU07eS8gSr5+eeILl+O8MaN0IJIuAUgLefHP0Z29Gg4ZWWInnaav1d0NTQ05FVMFeNorJioUgAl9c4yGSd5inde4JLz0Uewt21D9KWXoL31VgBsFk6CN5vTU6YgOmwY7AED5PpVcbIlkMsL2FK+T0u+ZZGzUCPdkTdQhfbtQ+yRR6SgBN+CwDSo31dStCUvlrpjMdjnnYfUU0/BuuACOeo78wYq88gRaFu2wHrjDeiffCIzBBLs3Vlh0cHr0wd8aYAYOBC46CJg1CjYvXt33Su6Wl6HUg4Tt5eqryvICr8dL/yWYxUKnmOVKraCv0SjvBQiX9HVwBjvwAHoH34Io6pKXnGS15yYpPNAgTmXy1fp9O8vo2ybAunXD/FevUCwelM4DCsnaC/8RXnG/M1+8kFToFK8aiN6YcdUN4Wp9PJHbfkf65FaJpqTIWoAAAAASUVORK5CYII=", "small_icon": "iVBORw0KGgoAAAANSUhEUgAAACQAAAAkCAYAAADhAJiYAAAAAXNSR0IArs4c6QAABiNJREFUWEetWFtIVF0U/s6Zm859JCV8SoKgjMS0C0SUFQVJYQSlFUlJ9eBDZBFB+Wag9FJSkRQGQuRb9hDdKI2E/JUopKwISoiIUObijHN1zv75tjq/l3Eu+u+nxTlrr7X2Wmt/a62tCCGEpmnw+XxwuVyIRqMYHx+XNJfH44HVaoXBYEjQWm8vTHv2yP8Lru/f4bHb5W/KCoVCCIfDkh4bG0M8Hpe0EAJerxdOpxMTExNQAoGAUBQFer0esVgMqqpCp9NJmouGkJEbSVOQ6O2FeffulPaEhoagW7lS8lAWZVI2aeqiTtLTuqUxigLF6/UKMprNZgQCAanUZDJJmove4eloCGmekputT59CV1OT1KjIq1cIl5VJfi7KokzKJk1d1EmaRlgsFhkVGq0sJmQUHOroQG5tbXKDnjyBae9eGeKsQ+Z2u6WHHA6HFGA0GqXFM4X5/X7pFcacp6KrbY8eQV9Xl9Sg8IsXCJWXz8rD3Nxc5OTkSLk2m02GjTQ9xPxhHslQRiIROkkq5AfSDA+9MDP+IhqF6fRpqF1dQCSSOqGn/xoM0KqqELl3j8kIXp65OTlXd9qQBd6+haW2FsrPn5kZsQCXKCzEeGcnrFu3pr5lC4ZsZASOykqoAwNLMmTuZm3dOoh//oFuKnzzQhaPx2XImCd2u13mB2+Vc/nyzEOTrcmqCt/fv7Dl5cmdxCXmFVNlPg719yN31y5A07JVkzV/uLsb2saNMncTODQzZL7hYTiKirIWvJQN/k+fYCsu/u+WJXDI7YaroAAQYinys9+rKPD++QNHQcGklxJIvW8fdG/eZCxQ27AB0Y4OhJ1OOM6cQVzTEGhvhzMYhHboENT+/sxllZYi0NMzidQej0cotHDNmowERI8fR/DGDQmkLCORSETSTEiCJmmelKXAfv481Pb2jOT6BgehKyqaLB1QlIw2BZubYWhoSFR+Ii8RmIjL0/GWkmaS8taQNj18CHN9fUbyY9EoFPfoqHAtW5Z2Q+TmTZjq6xOlY25LQa8QOvh9Ji2La08PrBUVaXWMu91Q4leuCLWpKSWztn49MDAgFdIj9ABp1j1WcdL0EKu4xLMtW6C9e4fAFHTQW1plJfTPnqXUE6urgxKrqBD67u6UjBPd3dBv3z6vWZsbMjYbKvPp8GHoOjtnFejYly8wpMnTeFkZFC0vTyhu98IG6XTwezzzqv3MkDmGhoDRUahVVdBqaqB1dCTCR8HMJVnt2UFONX7JFAqHA4owmwWCwYUNMpkQ+/oVhhUrknrIeOAAdM+fy/3RixcxfunSrKTmdxof/vEDOWvXAqHQwrpyczMwSFWhbduG8cePZT/DnpttBGkmb05hIcFMKmHTpq+ulv/JRx4uwoNlxw6o79+nBV5FOBwCPt/CVhuN8I+OLhiy6K9fsNy+DaWkBGNVVUlvWSJkTmfqgm02Q5nYvFno+vpSZ7/bDYPLlTapk+FQImQ+H3I4yaQoTdrq1VAiR48K44MHKQ2KnDqFYEvLrBY2GxySo1RjIwy3bqWGl8pKKL7Pn4W9uDgtaPm+fYNj1apFAaN3eBjODLqI8IcP2ZUO7+/fsOTnZ1U6ZMjy84F4PO2hJ0uH2y2M3d2wHDyYdgNrXuD1a8RKSuZNo8lKh21kBHqCYQbGjLe1QRw5MjWXxeNQOdSFw+mNAiBsNoz19UlsYrmInjsHPWW0tsrEN4yOwlJeDmVsLCN5MBohQiFMsIWdbtACg4Owl5ZmJmCaS1EgXC4oU2AnCGwcDrNs8gIvX8Kyc+dkgzazhY00NcHU2JidUUvkJrobW1qStLBTrx/ayZNQ799foprMtov9+4GurtmvH36/X75+cKIk3HOszmlogNrWlpnURXJp1dUI3r0rd7ONmX4JmRWymbP9WH8/7Js2LVJd6m3i40dZapLO9qleP1gkY5cvw9Tc/L8YFj17FoZr1+Cfet5J+mCVyetHgFf5wgWYOjqAiYmsjYucOAG0tsJktaZ//Zgeg/gEw/aTucS2gTQX289gMCinCtIcs03Xr0O9c4djLxT2UtNTLmHAYoGwWiGOHYNy9aqUQVlsdZkrpIldbHlJM3/5sMWJhd/+BXe3XAACKxnuAAAAAElFTkSuQmCC", "visible": true, "tab_version": "4.1.3", "tab_build_no": "0", "build_no": 12}, "data_input_builder": {"datainputs": [{"index": "default", "sourcetype": "sigsci-event", "interval": "300", "use_external_validation": true, "streaming_mode_xml": true, "name": "SigsciEvent", "title": "SigsciEvent", "description": "", "type": "customized", "parameters": [{"name": "site_api_name", "label": "Site API Name", "help_string": "This is the Site API Name. It should not be a URL.", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": "splunk-demo-server"}], "data_inputs_options": [{"type": "customized_var", "name": "site_api_name", "title": "Site API Name", "description": "This is the Site API Name. It should not be a URL.", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}], "code": "# encoding = utf-8\r\nfrom timeit import default_timer as timer\r\nimport requests\r\nimport json\r\nfrom sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time\r\n\r\n'''\r\n IMPORTANT\r\n Edit only the validate_input and collect_events functions.\r\n Do not edit any other part in this file.\r\n This file is generated only once when creating the modular input.\r\n'''\r\n\r\n\r\n# def use_single_instance_mode():\r\n# return True\r\n\r\n\r\ndef validate_input(helper, definition):\r\n # This example accesses the modular input variable\r\n site_name = definition.parameters.get('site_api_name', None)\r\n if site_name is None or site_name == \"\":\r\n msg = \"The site_name can not be empty\"\r\n raise ValueError(\r\n \"InvalidSiteName\",\r\n msg\r\n )\r\n elif \"http\" in site_name:\r\n msg = (\r\n \"The site name is not the full URL it should be the \",\r\n \"API Name of the site like 'my_example_site'\"\r\n )\r\n raise ValueError(\r\n \"InvalidSiteName\",\r\n msg\r\n )\r\n elif \" \" in site_name:\r\n msg = (\r\n \"The site name should be the API Name of the site like \",\r\n \"not the Display Name. Example would be 'my_site_name' instead of \",\r\n \"My Site Name\"\r\n )\r\n raise ValueError(\r\n \"InvalidSiteName\",\r\n msg\r\n )\r\n pass\r\n\r\n\r\ndef collect_events(helper, ew):\r\n start = timer()\r\n # loglevel = helper.get_log_level()\r\n # Proxy setting configuration\r\n # proxy_settings = helper.get_proxy()\r\n global_email = helper.get_global_setting(\"email\")\r\n global_api_token = helper.get_global_setting(\"api_token\")\r\n global_corp_api_name = helper.get_global_setting(\"corp_api_name\")\r\n api_host = 'https://dashboard.signalsciences.net'\r\n helper.log_info(\"email: %s\" % global_email)\r\n helper.log_info(\"corp: %s\" % global_corp_api_name)\r\n python_requests_version = requests.__version__\r\n user_agent_version = \"1.0.33\"\r\n user_agent_string = (\r\n f\"TA-sigsci-waf/{user_agent_version} \"\r\n f\"(PythonRequests {python_requests_version})\"\r\n )\r\n\r\n def pull_events(current_site, delta, key=None):\r\n site_name = current_site\r\n last_name = f\"requests_last_until_time_{current_site}\"\r\n last_run_until = helper.get_check_point(last_name)\r\n helper.log_info(f\"last_run_until: {last_run_until}\")\r\n if last_run_until is None:\r\n (\r\n until_time,\r\n from_time,\r\n from_time_friendly,\r\n until_time_friendly\r\n ) = get_from_and_until_times(delta, five_min_offset=False)\r\n else:\r\n (\r\n until_time,\r\n from_time,\r\n from_time_friendly,\r\n until_time_friendly\r\n ) = get_until_time(last_run_until, delta, five_min_offset=False)\r\n if from_time is None or from_time > until_time:\r\n helper.log_info(f\"{from_time} >= current now time, skipping run\")\r\n return\r\n if from_time >= until_time:\r\n helper.save_check_point(last_name, from_time)\r\n helper.log_info(\r\n f\"from_time {from_time} >= until_time {until_time}, skipping run\"\r\n )\r\n return\r\n helper.save_check_point(last_name, until_time)\r\n helper.log_info(\"SiteName: %s\" % site_name)\r\n\r\n helper.log_info(f\"Start Period: {from_time_friendly}\")\r\n helper.log_info(f\"End Period: {until_time_friendly}\")\r\n\r\n input_name = helper.get_input_stanza_names()\r\n single_name = \"\"\r\n\r\n if type(input_name) is dict and input_name > 1:\r\n helper.log_info(\"Multi instance mode\")\r\n for current_name in input_name:\r\n single_name = current_name\r\n else:\r\n helper.log_info(\"Single instance mode\")\r\n helper.log_info(\"Inputs: %s\" % input_name)\r\n helper.log_info(\"Inputs Num: %s\" % len(input_name))\r\n single_name = input_name\r\n helper.log_info(f\"single_name: {single_name}\")\r\n\r\n # Loop across all the data and output it in one big JSON object\r\n url = (\r\n f\"{api_host}/api/v0/corps/{global_corp_api_name}\"\r\n f\"/sites/{site_name}/activity?\"\r\n f\"from={from_time}&until={until_time}\"\r\n )\r\n helper.log_info(\"Pulling results from Events API\")\r\n config = Config(\r\n url=url,\r\n api_host=api_host,\r\n from_time=from_time,\r\n until_time=until_time,\r\n global_email=global_email,\r\n global_corp_api_name=global_corp_api_name,\r\n current_site=current_site,\r\n )\r\n config.headers = {\r\n 'Content-type': 'application/json',\r\n 'x-api-user': global_email,\r\n 'x-api-token': global_api_token,\r\n 'User-Agent': config.user_agent_string\r\n }\r\n all_events = get_results(\"Events\", helper, config)\r\n total_requests = len(all_events)\r\n helper.log_info(\"Total Events Pulled: %s\" % total_requests)\r\n write_start = timer()\r\n for current_event in all_events:\r\n # helper.log_debug(current_event)\r\n # helper.log_info(f\"data={event_data}\")\r\n current_event = json.dumps(current_event)\r\n if key is None:\r\n source_type = helper.get_sourcetype()\r\n helper.log_info(\"Concurrent Mode\")\r\n source_type_info = type(source_type)\r\n active_index = helper.get_output_index()\r\n index_info = type(active_index)\r\n single_name_info = type(single_name)\r\n current_event_info = type(current_event)\r\n helper.log_info(f\"source_type: {source_type}\")\r\n helper.log_info(f\"source_type_info: {source_type_info}\")\r\n helper.log_info(f\"index: {active_index}\")\r\n helper.log_info(f\"index_info: {index_info}\")\r\n helper.log_info(f\"single_name: {single_name}\")\r\n helper.log_info(f\"single_name_info: {single_name_info}\")\r\n helper.log_info(f\"current_event: {current_event}\")\r\n helper.log_info(f\"current_event_info: {current_event_info}\")\r\n event = helper.new_event(\r\n source=single_name,\r\n index=helper.get_output_index(),\r\n sourcetype=source_type,\r\n data=current_event\r\n )\r\n else:\r\n indexes = helper.get_output_index()\r\n current_index = indexes[key]\r\n types = helper.get_sourcetype()\r\n source_type = types[key]\r\n single_name = single_name[0]\r\n helper.log_info(\"Sequential Mode\")\r\n helper.log_info(f\"source_type: {source_type}\")\r\n helper.log_info(f\"index: {current_index}\")\r\n helper.log_info(f\"single_name: {single_name}\")\r\n helper.log_info(f\"current_event: {current_event}\")\r\n event = helper.new_event(\r\n source=single_name,\r\n index=current_index,\r\n sourcetype=source_type,\r\n data=current_event\r\n )\r\n\r\n try:\r\n ew.write_event(event)\r\n except Exception as e:\r\n raise e\r\n write_end = timer()\r\n write_time = write_end - write_start\r\n write_time_result = round(write_time, 2)\r\n helper.log_info(\"Total Event Output Time: %s seconds\"\r\n % write_time_result)\r\n\r\n # If multiple inputs configured it creates an array of values and the\r\n # script only gets called once per Input configuration\r\n\r\n # host_test = helper.get_arg('Host')\r\n # helper.log_info(\"Host: %s\" % host_test)\r\n all_sites = helper.get_arg('site_api_name')\r\n time_deltas = helper.get_arg('interval')\r\n helper.log_info(f\"interval: {time_deltas}\")\r\n if type(all_sites) is dict:\r\n helper.log_info(\"run_type: Sequential\")\r\n for active_input in all_sites:\r\n site = all_sites[active_input]\r\n current_delta = int(time_deltas[active_input])\r\n helper.log_info(\"site: %s\" % site)\r\n pull_events(\r\n key=active_input,\r\n current_site=site,\r\n delta=current_delta\r\n )\r\n helper.log_info(\"Finished Pulling Events for %s\" % site)\r\n else:\r\n helper.log_info(\"Run Type: Concurrent\")\r\n site = helper.get_arg('site_api_name')\r\n helper.log_info(\"site: %s\" % site)\r\n pull_events(\r\n current_site=site,\r\n delta=int(time_deltas)\r\n )\r\n helper.log_info(\"Finished Pulling Events for %s\" % site)\r\n end = timer()\r\n total_time = end - start\r\n time_result = round(total_time, 2)\r\n helper.log_info(\"Total Script Time: %s seconds\" % time_result)", "customized_options": [{"name": "site_api_name", "value": "splunk-demo-server"}], "uuid": "294ad5bbdf92407b9a6785b46106152a", "sample_count": "5"}, {"index": "default", "sourcetype": "sigsci-activity", "interval": "300", "use_external_validation": true, "streaming_mode_xml": true, "name": "SigsciActivity", "title": "SigsciActivity", "description": "", "type": "customized", "parameters": [{"name": "place_holder", "label": "Place Holder", "help_string": "It was required to have one option even if it isn't needed. You can skip this one.", "required": false, "format_type": "text", "default_value": "", "placeholder": "Not needed", "type": "text", "value": ""}], "data_inputs_options": [{"type": "customized_var", "name": "place_holder", "title": "Place Holder", "description": "It was required to have one option even if it isn't needed. You can skip this one.", "required_on_edit": false, "required_on_create": false, "format_type": "text", "default_value": "", "placeholder": "Not needed"}], "code": "# encoding = utf-8\nfrom timeit import default_timer as timer\nimport json\nfrom datetime import datetime\nfrom sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time\n\n'''\n IMPORTANT\n Edit only the validate_input and collect_events functions.\n Do not edit any other part in this file.\n This file is generated only once when creating the modular input.\n'''\n\n\n# def use_single_instance_mode():\n# return True\n\n\ndef validate_input(helper, definition):\n # This example accesses the modular input variable\n pass\n\n\ndef collect_events(helper, ew):\n start = timer()\n loglevel = helper.get_log_level()\n helper.set_log_level(loglevel)\n # Proxy setting configuration\n # proxy_settings = helper.get_proxy()\n global_email = helper.get_global_setting(\"email\")\n global_api_token = helper.get_global_setting(\"api_token\")\n global_corp_api_name = helper.get_global_setting(\"corp_api_name\")\n api_host = 'https://dashboard.signalsciences.net'\n helper.log_info(\"email: %s\" % global_email)\n helper.log_info(\"corp: %s\" % global_corp_api_name)\n\n def pull_events(delta, key=None):\n last_run_until = helper.get_check_point(\"activity_last_until_time\")\n helper.log_info(f\"last_run_until: {last_run_until}\")\n if last_run_until is None:\n (\n until_time,\n from_time\n ) = get_from_and_until_times(delta, five_min_offset=False)\n else:\n (\n until_time,\n from_time\n ) = get_until_time(last_run_until, delta, five_min_offset=False)\n if from_time is None:\n helper.log_info(f\"{last_run_until} >= current now time, skipping run\")\n return\n if from_time >= until_time:\n helper.save_check_point(\"activity_last_until_time\", from_time)\n helper.log_info(\n f\"from_time {from_time} >= until_time {until_time}, skipping run\"\n )\n return\n helper.save_check_point(\"activity_last_until_time\", until_time)\n\n helper.log_info(f\"Start Period: {datetime.fromtimestamp(from_time)}\")\n helper.log_info(f\"End Period: {datetime.fromtimestamp(until_time)}\")\n\n input_name = helper.get_input_stanza_names()\n single_name = \"\"\n\n if type(input_name) is dict and input_name > 1:\n helper.log_info(\"Multi instance mode\")\n for current_name in input_name:\n single_name = current_name\n else:\n helper.log_info(\"Single instance mode\")\n helper.log_info(\"Inputs: %s\" % input_name)\n helper.log_info(\"Inputs Num: %s\" % len(input_name))\n single_name = input_name\n helper.log_info(f\"single_name: {single_name}\")\n\n # Loop across all the data and output it in one big JSON object\n url = (\n f\"{api_host}/api/v0/corps/{global_corp_api_name}\"\n f\"/activity?\"\n f\"from={from_time}&until={until_time}\"\n )\n config = Config(\n url=url,\n api_host=api_host,\n from_time=from_time,\n until_time=until_time,\n global_email=global_email,\n global_corp_api_name=global_corp_api_name,\n current_site='',\n )\n config.headers = {\n 'Content-type': 'application/json',\n 'x-api-user': global_email,\n 'x-api-token': global_api_token,\n 'User-Agent': config.user_agent_string\n }\n helper.log_info(\"Pulling results from Corp Activity API\")\n all_events = get_results(\"Activity Events\", helper, config)\n total_requests = len(all_events)\n helper.log_info(\"Total Corp Activity Pulled: %s\" % total_requests)\n write_start = timer()\n for current_event in all_events:\n if key is None:\n source_type = helper.get_sourcetype()\n helper.log_info(\"Concurrent Mode\")\n source_type_info = type(source_type)\n active_index = helper.get_output_index()\n index_info = type(active_index)\n single_name_info = type(single_name)\n current_event_info = type(current_event)\n helper.log_info(f\"source_type: {source_type}\")\n helper.log_info(f\"source_type_info: {source_type_info}\")\n helper.log_info(f\"index: {active_index}\")\n helper.log_info(f\"index_info: {index_info}\")\n helper.log_info(f\"single_name: {single_name}\")\n helper.log_info(f\"single_name_info: {single_name_info}\")\n helper.log_info(f\"current_event: {current_event}\")\n helper.log_info(f\"current_event_info: {current_event_info}\")\n event = helper.new_event(\n source=single_name,\n index=helper.get_output_index(),\n sourcetype=source_type,\n data=current_event\n )\n else:\n indexes = helper.get_output_index()\n current_index = indexes[key]\n types = helper.get_sourcetype()\n source_type = types[key]\n single_name = single_name[0]\n helper.log_info(\"Sequential Mode\")\n helper.log_info(f\"source_type: {source_type}\")\n helper.log_info(f\"index: {current_index}\")\n helper.log_info(f\"single_name: {single_name}\")\n helper.log_info(f\"current_event: {current_event}\")\n event = helper.new_event(\n source=single_name,\n index=current_index,\n sourcetype=source_type,\n data=current_event\n )\n\n try:\n ew.write_event(event)\n except Exception as e:\n raise e\n write_end = timer()\n write_time = write_end - write_start\n write_time_result = round(write_time, 2)\n helper.log_info(\n f\"Total Corp Activity Output Time: {write_time_result} seconds\"\n )\n\n # If multiple inputs configured it creates an array of values and the\n # script only gets called once per Input configuration\n time_deltas = helper.get_arg('interval')\n helper.log_info(f\"interval: {time_deltas}\")\n if type(time_deltas) is dict:\n helper.log_info(\"run_type: Sequential\")\n for active_input in time_deltas:\n time_delta = time_deltas[active_input]\n time_delta = int(time_delta)\n helper.log_info(\"time_delta: %s\" % time_delta)\n pull_events(\n delta=time_delta,\n key=active_input\n )\n else:\n helper.log_info(\"Run Type: Concurrent\")\n helper.log_info(\"time_delta: %s\" % time_deltas)\n pull_events(\n delta=int(time_deltas)\n )\n helper.log_info(\"Finished Pulling Corp Activity\")\n end = timer()\n total_time = end - start\n time_result = round(total_time, 2)\n helper.log_info(f\"Total Script Time: {time_result} seconds\")\n", "customized_options": [{"name": "place_holder", "value": ""}], "uuid": "07b37d1943e942cf831c8ee85ffcb4a2", "sample_count": 0}, {"index": "default", "sourcetype": "sigsci-requests", "interval": "300", "use_external_validation": true, "streaming_mode_xml": true, "name": "SigsciRequests", "title": "SigsciRequests", "description": "", "type": "customized", "parameters": [{"name": "site_api_name", "label": "Site API Name", "help_string": "This is the API Name of the site to pull request data from. This should not be a URL.", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": "jeremycx"}], "data_inputs_options": [{"type": "customized_var", "name": "site_api_name", "title": "Site API Name", "description": "This is the API Name of the site to pull request data from. This should not be a URL.", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}], "customized_options": [{"name": "site_api_name", "value": "jeremycx"}], "code": "# encoding = utf-8\nfrom timeit import default_timer as timer\nfrom datetime import datetime\nfrom sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time\n\n'''\n IMPORTANT\n Edit only the validate_input and collect_events functions.\n Do not edit any other part in this file.\n This file is generated only once when creating the modular input.\n'''\n\n\n# def use_single_instance_mode():\n# return True\n\n\ndef validate_input(helper, definition):\n # This example accesses the modular input variable\n site_name = definition.parameters.get('site_api_name', None)\n if site_name is None or site_name == \"\":\n msg = \"The site_name can not be empty\"\n raise ValueError(\n \"InvalidSiteName\",\n msg\n )\n elif \"http\" in site_name:\n msg = (\n \"The site name is not the full URL it should be the \",\n \"API Name of the site like 'my_example_site'\"\n )\n raise ValueError(\n \"InvalidSiteName\",\n msg\n )\n elif \" \" in site_name:\n msg = (\n \"The site name should be the API Name of the site like \",\n \"not the Display Name. Example would be 'my_site_name' instead of \",\n \"My Site Name\"\n )\n raise ValueError(\n \"InvalidSiteName\",\n msg\n )\n pass\n\n\ndef collect_events(helper, ew):\n start = timer()\n loglevel = helper.get_log_level()\n helper.set_log_level(loglevel)\n # Proxy setting configuration\n # proxy_settings = helper.get_proxy()\n global_email = helper.get_global_setting(\"email\")\n global_api_token = helper.get_global_setting(\"api_token\")\n global_corp_api_name = helper.get_global_setting(\"corp_api_name\")\n api_host = 'https://dashboard.signalsciences.net'\n helper.log_info(\"email: %s\" % global_email)\n helper.log_info(\"corp: %s\" % global_corp_api_name)\n\n def pull_requests(current_site, delta, key=None):\n site_name = current_site\n last_name = f\"requests_last_until_time_{current_site}\"\n last_run_until = helper.get_check_point(last_name)\n helper.log_info(f\"last_run_until: {last_run_until}\")\n\n\n if last_run_until is None:\n helper.log_info(\"get_from_until\")\n until_time, from_time = get_from_and_until_times(delta, five_min_offset=True)\n else:\n helper.log_info(\"get_until\")\n until_time, from_time = get_until_time(last_run_until, delta, five_min_offset=True)\n\n if from_time is None:\n helper.log_info(f\"{last_run_until} >= current now time, skipping run\")\n return\n \n if from_time >= until_time:\n helper.save_check_point(last_name, from_time)\n helper.log_info(f\"from_time {from_time} >= until_time {until_time}, skipping run\")\n return\n\n helper.log_info(\"SiteName: %s\" % site_name)\n helper.log_info(f\"Start Period: {datetime.fromtimestamp(from_time)} UTC\")\n helper.log_info(f\"End Period: {datetime.fromtimestamp(until_time)} UTC\")\n\n input_name = helper.get_input_stanza_names()\n single_name = \"\"\n\n if type(input_name) is dict and input_name > 1:\n helper.log_info(\"Multi instance mode\")\n for current_name in input_name:\n single_name = current_name\n else:\n helper.log_info(\"Single instance mode\")\n helper.log_info(\"Inputs: %s\" % input_name)\n helper.log_info(\"Inputs Num: %s\" % len(input_name))\n single_name = input_name\n helper.log_info(f\"single_name: {single_name}\")\n\n # Loop across all the data and output it in one big JSON object\n url = (\n f\"{api_host}/api/v0/corps/{global_corp_api_name}\"\n f\"/sites/{site_name}/feed/requests?\"\n f\"from={from_time}&until={until_time}\"\n )\n helper.log_info(\"Pulling requests from requests API\") \n config = Config(\n url=url,\n api_host=api_host,\n from_time=from_time,\n until_time=until_time,\n global_email=global_email,\n global_corp_api_name=global_corp_api_name,\n current_site=current_site,\n )\n config.headers = {\n 'Content-type': 'application/json',\n 'x-api-user': global_email,\n 'x-api-token': global_api_token,\n 'User-Agent': config.user_agent_string\n }\n\n all_requests = get_results(\"Requests\", helper, config)\n\n total_requests = len(all_requests)\n helper.log_info(\"Total Requests Pulled: %s\" % total_requests)\n write_start = timer()\n for current_event in all_requests:\n if key is None:\n source_type = helper.get_sourcetype()\n event = helper.new_event(\n source=single_name,\n index=helper.get_output_index(),\n sourcetype=source_type,\n data=current_event\n )\n else:\n indexes = helper.get_output_index()\n current_index = indexes[key]\n types = helper.get_sourcetype()\n source_type = types[key]\n single_name = single_name[0]\n event = helper.new_event(\n source=single_name,\n index=current_index,\n sourcetype=source_type,\n data=current_event\n )\n\n try:\n ew.write_event(event)\n helper.save_check_point(last_name, until_time)\n except Exception as e:\n helper.log_error(event)\n raise e\n \n write_end = timer()\n write_time = write_end - write_start\n write_time_result = round(write_time, 2)\n helper.log_info(\"Total Event Output Time: %s seconds\" % write_time_result)\n\n # If multiple inputs configured it creates an array of values and the\n # script only gets called once per Input configuration\n all_sites = helper.get_arg('site_api_name')\n time_deltas = helper.get_arg('interval')\n helper.log_info(f\"interval: {time_deltas}\")\n\n if type(all_sites) is dict:\n helper.log_info(\"run_type: Sequential\")\n for active_input, site in all_sites.items():\n time_delta = int(time_deltas[active_input])\n helper.log_info(\"site: %s\" % site)\n pull_requests(\n key=active_input,\n current_site=site,\n delta=time_delta\n )\n helper.log_info(\"Finished Pulling Requests for %s\" % site)\n else:\n helper.log_info(\"Run Type: Concurrent\")\n site = helper.get_arg('site_api_name')\n helper.log_info(\"site: %s\" % site)\n pull_requests(\n current_site=site,\n delta=int(time_deltas)\n )\n helper.log_info(\"Finished Pulling Requests for %s\" % site)\n end = timer()\n total_time = end - start\n time_result = round(total_time, 2)\n helper.log_info(\"Total Script Time: %s seconds\" % time_result)", "uuid": "aaaeb391da9043e1819408033d9db708", "sample_count": 0}]}, "field_extraction_builder": {"sigsci-event": {"is_parsed": true, "data_format": "json"}, "sigsci-activity": {"is_parsed": true, "data_format": "json"}, "sigsci-requests": {"is_parsed": true, "data_format": "json"}}, "global_settings_builder": {"global_settings": {"proxy_settings": {"proxy_type": "http"}, "log_settings": {"log_level": "DEBUG"}, "customized_settings": [{"required": true, "name": "email", "label": "E-mail", "default_value": "", "placeholder": "example@example.com", "help_string": "This it the e-mail username of the user who has the correct permissions for the app to pull the data.", "type": "text", "format_type": "text", "value": "dacoburn@dactbc.com"}, {"required": true, "name": "corp_api_name", "label": "Corp", "default_value": "", "placeholder": "", "help_string": "This is the API name of your corp.", "type": "text", "format_type": "text", "value": "dactbc"}, {"required": true, "name": "api_token", "label": "API Token", "placeholder": "", "default_value": "", "help_string": "This is the API Token of the user who has the correct permissions. The API Token is tied to the username.", "type": "password", "format_type": "password", "value": ""}]}}, "sourcetype_builder": {"sigsci-requests": {"metadata": {"event_count": 0, "data_input_name": "SigsciRequests", "extractions_count": 0, "cims_count": 0}}, "sigsci-event": {"metadata": {"event_count": 0, "data_input_name": "SigsciEvent", "extractions_count": 0, "cims_count": 0}}, "sigsci-activity": {"metadata": {"event_count": 0, "data_input_name": "SigsciActivity", "extractions_count": 0, "cims_count": 0}}}, "validation": {"validators": ["best_practice_validation", "data_model_mapping_validation", "field_extract_validation", "app_cert_validation"], "status": "job_started", "validation_id": "v_1689009952_31"}} \ No newline at end of file +{"basic_builder": {"appname": "sigsci_TA_for_splunk", "friendly_name": "Signal Sciences WAF TA", "version": "1.0.36", "author": "Fastly", "description": "For users of Fastly who would like to enrich their Splunk data with information from Fastly. This app with simple configuration enabled the importing of Events, Activity, and raw request information to Splunk.\n\nThis is an open source project, no support provided, public repository is available and installation documentation can be found at https://github.com/fastly/sigsci-splunk-app. The best way to report issues with the app is to create an Issue on the github page so that it can be tracked.", "theme": "#ea3f23", "large_icon": "iVBORw0KGgoAAAANSUhEUgAAAEgAAABICAYAAABV7bNHAAAAAXNSR0IArs4c6QAAD+5JREFUeF7tXAuMVNUZ/s69d96wCwULGoWGiFofNT6KsVEEhYqiEJAKiq0ixSpLa1EKKqAGBXV5RAlugkYXUFCK5dHEtKXl0QqrrNhoVojBRxsUUEGW3Z2dnZn7ar6zc8bLPnBn791BTU+y2dmz5/7n/P+c/z//4ztXuK7rIteSySRM04QQAvF4HOFwWP6ntrZWDZH9kUhE/m1ZFhoaGuRn9vF/bOl0Gk1NTfKzpmkoLS2FdeQIxJQpEP/+d55WZz6IK6+EePllqLWSRigUQrdu3fLk1HoNw0AikZBrsG0b9fX1+TE9e/bMf25sbEQ2m5V/e/mrq6uDKKaAtNGjoVVVdUYuXz8zfDiwefP/BdSuFIstoKNHj+ZVjNuU25Vax21HdWPzbsdUKoVMJiP7uYW7d+/eSq2i0ShisZjsdxwHcqumUkg8+CD0PXsgPvkE4tixgnaS27cv3D59YF1yCRoXLZIqxbWycZ1UOdXUetnP9XINuq6jpKSklRqyg2pIc0K+OV6pG02DKJaApD06cgTdIhGIu++G9pe/FCQg87e/hV1Whoymwendu3gCamxszO8gGjPVKE1lv9vrV0aYv73jaeT5oxq/QdVIKzRxIvT16wsT0IwZsObNa3NNaqd652hv7rb6vettyXfRjLRa2LFjx9B98uTCBTR9OuwFC0D1ZfvenWJ+BWRNmwZr4cLiC6iYNohCouFzx46FvmlTYSr20EMwZ8/OHxDfKyNdf+gQjJoaGFu3IpROQ9uwAeKzzwoSkHPJJbAvuwx2NAr78ssRGjoU4V69vrunGD3rTFUVIo8/DvHmmzyLIdJpWvOCBNNqMI0/PflwGM4558B6+GG4w4bJ47lLjvkgPekYT8GdO+Fu3gxRWQlx+LA/YXTwafeUU+BOmgQxZAgwdCiO5cKcb0+oYVmIb9qE8J/+1Lxb6uo6yFrAw2IxuIMHI33ttUjfdhuMaNR/LOY4Tn7P03tmAEq/gJ6w8lTpCavGfhXE2pkMslu2IDpzJrS9ewPm1h8559xzYS5eDGPoUGjhsAxWvd42DwvVqJ4qavDyx0C805608fHHCK9YgfDq1QWHDf5YL+DpaBSZ22+HOXEinAsvLF6ooVVWIlJeDrF/fwGrPXlD3dNOg/W738GYOTPv4XtTOCeMxZLJZF7FGNBRvehuMzxQoQaNHRsDTH3xYugVFUBj48njuDMzGwacm2+G9dRTcHv3hmXbeSqKb3aQbxUayf6OnmLawYOIP/AAQn/9KzNlnVnit+IZ68orkVqyBCWXXppfj/+E2eHDKL3sMmj//a9/P+ZbICY3kUD6wAHEcob6hAL6plDDePdddBs/vmg+TbHkR98p+dJLsAYN6nw+CG+/jfj06dDee69Y6y7qPM6PfoSmJ59EaMyY9hNmtbW1eSNNa06DTOOc3r8foenTEdq4saiLLvZk1rBhwLPPQh84UPLNYoPyiZgtbdtINzQgPncuwqtWFXu9J2U+86ab0LhwIdwePTpW1Qjv2IHE2LHf6dOqIEnrOjLl5UhNmtRaQJZl5VWM9Szr4EGUnnVWQfQLGkw/64c/lD/OqafCGj8e1uDBELW1iF9/vSTlbNuG5Omng19UaOVK6B9/DBw6BPHFF116ita/9x4iZ58tzQwbQ5DjQo1YJILwjTdC27q1IJ47NFgIOMznjBkD/YorYJ1/PtKAjJHYetTWwr3zTkAIJJ9+Gna/frKfaRP+iLffhr1jB7S//Q36li1dIijmnJL/+AfsXEqmVVUjtn49otOmsTTaIZ47Osjt3x9N8+cjO3QoSvv2hchVOul/KAH1pE+Sm7c+m80v0luxbWLJqb4e4W3bEHvoIQj6ZUE2w0Dm3nuRmj1bUpUCUqGG09CA2LhxMHbuDG5KTUP21lthLVoEN1caVtu3vXCGk1No3oqKqqp4wwAtmYQ+YwZC69fLmltQjX5R+pVXIPr0kaqWP8Wc5cshZs2CCCqXo+tofOYZWLfdJr8J1VjVIPOMc1gHVwLzBo8s8PH/bCxSqmwhUxEtqxrhTZsQv+8+acMCafE43PnzIX7/++aCJ/NBLgEIf/gDtOeeC2QONxZDauVKmMOHS+CAqr6SOAEEFBD7ybASkDfnxIS8V0A8PJQ9UgLy5nC46xO//jXE558Hsn5z7FiIpUuRoh/EUCNUVYUEkReHDvmewO3VC3Z5OXSmQGmYc6VnRbhHjx6yn2p0nA3yoC0oRGWbvDbIW/ZuWdXIrl2LGBN3AfCAWAzJFSsQHzcO4uiRI26UxMvKfAsHug7zV7+CvXgxojmGiyWgZH094i++iMicOUAOyuKHofTjjyMyYwZE/WefufGpU6H/+c9+6MlnnYEDkdy9W6ZqVVpWAQIUcdodtbNoX1TuhWGOanT3Vb+ipf3nP8hu3470iBEQJSXy6FfqyTSxAlQkBg+GHkDs6PzsZ3A3boRwDx1y3fPPh/jqK/8Cqq6G9tOftgmg8mOkrXXrEJs9G9qXX8KuqJDq217p2dq/H0b//r55QTyOut27Ier27nVLzj3XN0Hn4ouhvfOOpNMWwqwzAuIuspctQ2jmTB5nkF/k7t2yLnai2rx1110wnn/eN0/JrVshUqtXu7GJE/0Ri0TQuHo1wqNGSTo0sNz2bFQnpW78m9gbqh37qSLKx1EqwjFUK+PoUYglS6AtWiTpuOPHI11ZmVcrRtxKDSViJIcV4lhz507ER4yQQvXTzHnzIKyRI1399df90AGTT03btyNz6qmSTlsAqkJOsfTOnQjdfz/0XbvkYwQu6AsWoIlBZY7pE9XmS5JJiFGjoL37ri++7Ouug7AHDXK16mp/hMaMgblsGZpyqLJCBSSSScQ2b4axdi1QUiJjQen46TqcyZOBRYugde8uHcaOCKiHrsOeMcO3mjn9+0M4/fq5fss3ZlkZrEceAXO9Sq2U6qiQQn0D7FeVE/7P3rMHiVGjWvtgQoB03Vmz5A5l81ZaFB2pfrkqTH4Ox4FWUYHwrFm+vni3X79gBEQQgTtgAERpKbQ1awoy0jqdsTagMAxwsWcPsoZxwlBD2SwFA3aefBLOa69By2ah1dT4ExCzCEHsIIm2YKr2ggvg7NghU5YKCKkw12ql9J7VLmM4ER0xAtobb7RmJJGAXVd3HC2vf8WTUnnbkk4OeSbGjm3GPxL256l9dUZSLqGEQQjI/OUvJQTFLC2FedVV0olTC6ZaKLA5F8lAVIUadAjDS5YgMm9eM0Oelr3/fqTmzJEnoELMUijKBtHh9KJcVUAb+uc/ET98GHZVFYzKys7IJf+M27NnQAK6+24JsEznMnGFGGknk0H3sjLo69blF+Zccw3qXnlF7sqOxmLHwYCjUThPPAHtscf8CYg2yB4yxNW2b/dFyL7qKpgrV8LKIb645b15H6VunIQ7QoUa9JVU3ifU0AC7pgbagAGwiYnOrchLi+OVWnH3ePNEqhLBxyKNjdAYPhWIpG0pBPessyDS8+e7kVwGrdNSikSQfucdRM87T5IIypMmrW/KB7U00vy7rqYGJddfXzDMryX/zGWJzI4dbviKKzotG/Vg5rnnEJky5VshoMYNG5qrMj5bE1FyyX373MRPfuI7D02fIbtvn1yS1y+hOnmB6MqXUeBtBThXqsPn2/NxvLS8PpG3n88agwZB+PSiuY5UdTXEsX373BK61Cyt+GyZJUuQuuMO36FGZxJmykjHdu1C9LrrfHICuD/4AdyaGojagwfdBOvTS5f6J5pIyLKJceGFrS6zKOJdlVGkgLSvvkLJxRdD5O6w+WHIHj0a+sqVEOlUytU3boQxaZLv6Jc1Leuaa5BhAaBvX7m+YqmYQ5+KWILVq307iFy3+eKLsG6+ubmq4TKPc8cdEO+/70fozc8y4p42DU0PPgg3HM7fOFSEg65qNE+pI/7MMzDo9+SucPlhxGVWYs0a1F90EYRpmq5DZOucOQgtXuyH7tfPUki/+Q3Sc+dC79Yt71VzgLqqqfJEyoCrfo6hc6j66fsoH4e+lfKe6VtJw57JwFi6FLEFCwLZOZzfmjoV7hNPyKtXQsFfQv/6F2L33tuMIguiaRqsESPkVs3k7riSrLrUxlOIQlGnl/eym7faQcdShS30ibyhBukZEyZA27w5MMwk8QJNCxfCHDlSlquOhwFPmIAQJwuwEVLS9PLLyF56qVS5QIy0acLg5bwhQwJHvpnDhiH5xz9KCcjSs/dCHaqrEbvhBnYGKCJ55xHmz3+O7C23QKeHmwNpeS+vGakUuIsZtGYGD4aTK1Uf5/uYJuwtWxDesAGhTZuCx2fHYsi++iokqEr5Y8ehXOvrEZk/H6Hy8mAFlKPG6NghZnnkSGTvuguRM87I56vrq6vRfdw4idoQf/87cM458impVp9+isi6dQixZn7gALd9l6zPnjABdmUlwrnUSbvXwkuuvhp6AJ7oCbnQdemMOQx0eRc+k0GEhpZGcu5cWIYB4803ob/1FsSnnwZmgNtbE8Hmx95/vzWAquWdVbml9+6F8YtfBOJdd8lXHTBRAsvttWthsVjY4q5u23c1HAfmqlUI33OP7xgtYF66hJy7bJm8ie1qWsevhTd98YW8dhCuqAgUf9MlHHaWaCSC7KRJ0ObNg8E7Z23dm89ms3mMoirGUc1koqqhARp9o1df7RLIW2f5Cuo5d/hwOC+8gCxVLJe/pjOqoDd0Rr/xroZ8kMfza699f4REICmzhR98IGV9wqsI3h1EidHDVWXhPIiJ8LxZs6CvWUPoZ1Bf4MmhEwrBufVWmMuX58MZag43Qiu+M5mvQw2u1ou09yLOZRhA8MCKFdAefRTiyy9PDnN+Z00k4Dz8MMTUqajz3FhStwzbRNp/02UWrsn7chN71SqI8nJoe/b4XW5Rn3fPPhsuAVGTJ8ud0uELdel0Om+k2yvttkyH6vRmKypg8GJdwJDhwKUWjSJ7++3ysMl6cEPKfHC+9tK30tx09EKdWrh6Q5Nr28ju2oUQ3+Tis8QbuFBUaHPmmbCefVZe7HU07eS8gSr5+eeILl+O8MaN0IJIuAUgLefHP0Z29Gg4ZWWInnaav1d0NTQ05FVMFeNorJioUgAl9c4yGSd5inde4JLz0Uewt21D9KWXoL31VgBsFk6CN5vTU6YgOmwY7AED5PpVcbIlkMsL2FK+T0u+ZZGzUCPdkTdQhfbtQ+yRR6SgBN+CwDSo31dStCUvlrpjMdjnnYfUU0/BuuACOeo78wYq88gRaFu2wHrjDeiffCIzBBLs3Vlh0cHr0wd8aYAYOBC46CJg1CjYvXt33Su6Wl6HUg4Tt5eqryvICr8dL/yWYxUKnmOVKraCv0SjvBQiX9HVwBjvwAHoH34Io6pKXnGS15yYpPNAgTmXy1fp9O8vo2ybAunXD/FevUCwelM4DCsnaC/8RXnG/M1+8kFToFK8aiN6YcdUN4Wp9PJHbfkf65FaJpqTIWoAAAAASUVORK5CYII=", "small_icon": "iVBORw0KGgoAAAANSUhEUgAAACQAAAAkCAYAAADhAJiYAAAAAXNSR0IArs4c6QAABiNJREFUWEetWFtIVF0U/s6Zm859JCV8SoKgjMS0C0SUFQVJYQSlFUlJ9eBDZBFB+Wag9FJSkRQGQuRb9hDdKI2E/JUopKwISoiIUObijHN1zv75tjq/l3Eu+u+nxTlrr7X2Wmt/a62tCCGEpmnw+XxwuVyIRqMYHx+XNJfH44HVaoXBYEjQWm8vTHv2yP8Lru/f4bHb5W/KCoVCCIfDkh4bG0M8Hpe0EAJerxdOpxMTExNQAoGAUBQFer0esVgMqqpCp9NJmouGkJEbSVOQ6O2FeffulPaEhoagW7lS8lAWZVI2aeqiTtLTuqUxigLF6/UKMprNZgQCAanUZDJJmove4eloCGmekputT59CV1OT1KjIq1cIl5VJfi7KokzKJk1d1EmaRlgsFhkVGq0sJmQUHOroQG5tbXKDnjyBae9eGeKsQ+Z2u6WHHA6HFGA0GqXFM4X5/X7pFcacp6KrbY8eQV9Xl9Sg8IsXCJWXz8rD3Nxc5OTkSLk2m02GjTQ9xPxhHslQRiIROkkq5AfSDA+9MDP+IhqF6fRpqF1dQCSSOqGn/xoM0KqqELl3j8kIXp65OTlXd9qQBd6+haW2FsrPn5kZsQCXKCzEeGcnrFu3pr5lC4ZsZASOykqoAwNLMmTuZm3dOoh//oFuKnzzQhaPx2XImCd2u13mB2+Vc/nyzEOTrcmqCt/fv7Dl5cmdxCXmFVNlPg719yN31y5A07JVkzV/uLsb2saNMncTODQzZL7hYTiKirIWvJQN/k+fYCsu/u+WJXDI7YaroAAQYinys9+rKPD++QNHQcGklxJIvW8fdG/eZCxQ27AB0Y4OhJ1OOM6cQVzTEGhvhzMYhHboENT+/sxllZYi0NMzidQej0cotHDNmowERI8fR/DGDQmkLCORSETSTEiCJmmelKXAfv481Pb2jOT6BgehKyqaLB1QlIw2BZubYWhoSFR+Ii8RmIjL0/GWkmaS8taQNj18CHN9fUbyY9EoFPfoqHAtW5Z2Q+TmTZjq6xOlY25LQa8QOvh9Ji2La08PrBUVaXWMu91Q4leuCLWpKSWztn49MDAgFdIj9ABp1j1WcdL0EKu4xLMtW6C9e4fAFHTQW1plJfTPnqXUE6urgxKrqBD67u6UjBPd3dBv3z6vWZsbMjYbKvPp8GHoOjtnFejYly8wpMnTeFkZFC0vTyhu98IG6XTwezzzqv3MkDmGhoDRUahVVdBqaqB1dCTCR8HMJVnt2UFONX7JFAqHA4owmwWCwYUNMpkQ+/oVhhUrknrIeOAAdM+fy/3RixcxfunSrKTmdxof/vEDOWvXAqHQwrpyczMwSFWhbduG8cePZT/DnpttBGkmb05hIcFMKmHTpq+ulv/JRx4uwoNlxw6o79+nBV5FOBwCPt/CVhuN8I+OLhiy6K9fsNy+DaWkBGNVVUlvWSJkTmfqgm02Q5nYvFno+vpSZ7/bDYPLlTapk+FQImQ+H3I4yaQoTdrq1VAiR48K44MHKQ2KnDqFYEvLrBY2GxySo1RjIwy3bqWGl8pKKL7Pn4W9uDgtaPm+fYNj1apFAaN3eBjODLqI8IcP2ZUO7+/fsOTnZ1U6ZMjy84F4PO2hJ0uH2y2M3d2wHDyYdgNrXuD1a8RKSuZNo8lKh21kBHqCYQbGjLe1QRw5MjWXxeNQOdSFw+mNAiBsNoz19UlsYrmInjsHPWW0tsrEN4yOwlJeDmVsLCN5MBohQiFMsIWdbtACg4Owl5ZmJmCaS1EgXC4oU2AnCGwcDrNs8gIvX8Kyc+dkgzazhY00NcHU2JidUUvkJrobW1qStLBTrx/ayZNQ799foprMtov9+4GurtmvH36/X75+cKIk3HOszmlogNrWlpnURXJp1dUI3r0rd7ONmX4JmRWymbP9WH8/7Js2LVJd6m3i40dZapLO9qleP1gkY5cvw9Tc/L8YFj17FoZr1+Cfet5J+mCVyetHgFf5wgWYOjqAiYmsjYucOAG0tsJktaZ//Zgeg/gEw/aTucS2gTQX289gMCinCtIcs03Xr0O9c4djLxT2UtNTLmHAYoGwWiGOHYNy9aqUQVlsdZkrpIldbHlJM3/5sMWJhd/+BXe3XAACKxnuAAAAAElFTkSuQmCC", "visible": true, "tab_version": "4.1.3", "tab_build_no": "0", "build_no": 14}, "data_input_builder": {"datainputs": [{"index": "default", "sourcetype": "sigsci-event", "interval": "300", "use_external_validation": true, "streaming_mode_xml": true, "name": "SigsciEvent", "title": "SigsciEvent", "description": "", "type": "customized", "parameters": [{"name": "site_api_name", "label": "Site API Name", "help_string": "This is the Site API Name. It should not be a URL.", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": "jeremycx"}], "data_inputs_options": [{"type": "customized_var", "name": "site_api_name", "title": "Site API Name", "description": "This is the Site API Name. It should not be a URL.", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}], "code": "# encoding = utf-8\nfrom timeit import default_timer as timer\nimport requests\nimport json\nfrom datetime import datetime\nfrom sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time\n\n\"\"\"\n IMPORTANT\n Edit only the validate_input and collect_events functions.\n Do not edit any other part in this file.\n This file is generated only once when creating the modular input.\n\"\"\"\n\n\n# def use_single_instance_mode():\n# return True\n\n\ndef validate_input(helper, definition):\n # This example accesses the modular input variable\n site_name = definition.parameters.get(\"site_api_name\", None)\n if site_name is None or site_name == \"\":\n msg = \"The site_name can not be empty\"\n raise ValueError(\"InvalidSiteName\", msg)\n elif \"http\" in site_name:\n msg = (\n \"The site name is not the full URL it should be the \",\n \"API Name of the site like 'my_example_site'\",\n )\n raise ValueError(\"InvalidSiteName\", msg)\n elif \" \" in site_name:\n msg = (\n \"The site name should be the API Name of the site like \",\n \"not the Display Name. Example would be 'my_site_name' instead of \",\n \"My Site Name\",\n )\n raise ValueError(\"InvalidSiteName\", msg)\n pass\n\n\ndef collect_events(helper, ew):\n start = timer()\n loglevel = helper.get_log_level()\n # Proxy setting configuration\n # proxy_settings = helper.get_proxy()\n global_email = helper.get_global_setting(\"email\")\n global_api_token = helper.get_global_setting(\"api_token\")\n global_corp_api_name = helper.get_global_setting(\"corp_api_name\")\n api_host = \"https://dashboard.signalsciences.net\"\n helper.log_info(\"email: %s\" % global_email)\n helper.log_info(\"corp: %s\" % global_corp_api_name)\n\n def pull_events(current_site, delta, key=None):\n site_name = current_site\n last_name = f\"events_last_until_time_{current_site}\"\n last_run_until = helper.get_check_point(last_name)\n helper.log_info(f\"last_run_until: {last_run_until}\")\n if last_run_until is None:\n (until_time, from_time) = get_from_and_until_times(\n delta, five_min_offset=False\n )\n else:\n (until_time, from_time) = get_until_time(\n helper, last_run_until, delta, five_min_offset=False\n )\n if from_time is None or from_time > until_time:\n helper.log_info(f\"{from_time} >= current now time, skipping run\")\n return\n if from_time >= until_time:\n helper.save_check_point(last_name, from_time)\n helper.log_info(\n f\"from_time {from_time} >= until_time {until_time}, skipping run\"\n )\n return\n helper.save_check_point(last_name, until_time)\n helper.log_info(\"SiteName: %s\" % site_name)\n\n helper.log_info(f\"Start Period: {datetime.fromtimestamp(from_time)}\")\n helper.log_info(f\"End Period: {datetime.fromtimestamp(until_time)}\")\n\n input_name = helper.get_input_stanza_names()\n single_name = \"\"\n\n if type(input_name) is dict and input_name > 1:\n helper.log_info(\"Multi instance mode\")\n for current_name in input_name:\n single_name = current_name\n else:\n helper.log_info(\"Single instance mode\")\n helper.log_info(\"Inputs: %s\" % input_name)\n helper.log_info(\"Inputs Num: %s\" % len(input_name))\n single_name = input_name\n helper.log_info(f\"single_name: {single_name}\")\n\n # Loop across all the data and output it in one big JSON object\n url = (\n f\"{api_host}/api/v0/corps/{global_corp_api_name}\"\n f\"/sites/{site_name}/activity?\"\n f\"from={from_time}&until={until_time}\"\n )\n helper.log_info(\"Pulling results from Events API\")\n config = Config(\n url=url,\n api_host=api_host,\n from_time=from_time,\n until_time=until_time,\n global_email=global_email,\n global_corp_api_name=global_corp_api_name,\n current_site=current_site,\n )\n config.headers = {\n \"Content-type\": \"application/json\",\n \"x-api-user\": global_email,\n \"x-api-token\": global_api_token,\n \"User-Agent\": config.user_agent_string,\n }\n all_events = get_results(\"Events\", helper, config)\n total_requests = len(all_events)\n helper.log_info(\"Total Events Pulled: %s\" % total_requests)\n write_start = timer()\n for current_event in all_events:\n helper.log_debug(current_event)\n if key is None:\n source_type = helper.get_sourcetype()\n helper.log_info(\"Concurrent Mode\")\n source_type_info = type(source_type)\n active_index = helper.get_output_index()\n index_info = type(active_index)\n single_name_info = type(single_name)\n current_event_info = type(current_event)\n helper.log_info(f\"source_type: {source_type}\")\n helper.log_info(f\"source_type_info: {source_type_info}\")\n helper.log_info(f\"index: {active_index}\")\n helper.log_info(f\"index_info: {index_info}\")\n helper.log_info(f\"single_name: {single_name}\")\n helper.log_info(f\"single_name_info: {single_name_info}\")\n helper.log_info(f\"current_event: {current_event}\")\n helper.log_info(f\"current_event_info: {current_event_info}\")\n event = helper.new_event(\n source=single_name,\n index=helper.get_output_index(),\n sourcetype=source_type,\n data=current_event,\n )\n else:\n indexes = helper.get_output_index()\n current_index = indexes[key]\n types = helper.get_sourcetype()\n source_type = types[key]\n single_name = single_name[0]\n helper.log_info(\"Sequential Mode\")\n helper.log_info(f\"source_type: {source_type}\")\n helper.log_info(f\"index: {current_index}\")\n helper.log_info(f\"single_name: {single_name}\")\n helper.log_info(f\"current_event: {current_event}\")\n event = helper.new_event(\n source=single_name,\n index=current_index,\n sourcetype=source_type,\n data=current_event,\n )\n\n try:\n ew.write_event(event)\n except Exception as e:\n raise e\n write_end = timer()\n write_time = write_end - write_start\n write_time_result = round(write_time, 2)\n helper.log_info(\"Total Event Output Time: %s seconds\" % write_time_result)\n\n # If multiple inputs configured it creates an array of values and the\n # script only gets called once per Input configuration\n\n all_sites = helper.get_arg(\"site_api_name\")\n time_deltas = helper.get_arg(\"interval\")\n helper.log_info(f\"interval: {time_deltas}\")\n if type(all_sites) is dict:\n helper.log_info(\"run_type: Sequential\")\n for active_input in all_sites:\n site = all_sites[active_input]\n current_delta = int(time_deltas[active_input])\n helper.log_info(\"site: %s\" % site)\n pull_events(key=active_input, current_site=site, delta=current_delta)\n helper.log_info(\"Finished Pulling Events for %s\" % site)\n else:\n helper.log_info(\"Run Type: Concurrent\")\n site = helper.get_arg(\"site_api_name\")\n helper.log_info(\"site: %s\" % site)\n pull_events(current_site=site, delta=int(time_deltas))\n helper.log_info(\"Finished Pulling Events for %s\" % site)\n end = timer()\n total_time = end - start\n time_result = round(total_time, 2)\n helper.log_info(\"Total Script Time: %s seconds\" % time_result)\n", "customized_options": [{"name": "site_api_name", "value": "yoursite"}], "uuid": "294ad5bbdf92407b9a6785b46106152a", "sample_count": 0}, {"index": "default", "sourcetype": "sigsci-requests", "interval": "300", "use_external_validation": true, "streaming_mode_xml": true, "name": "SigsciRequests", "title": "SigsciRequests", "description": "", "type": "customized", "parameters": [{"name": "site_api_name", "label": "Site API Name", "help_string": "This is the API Name of the site to pull request data from. This should not be a URL.", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": "jeremycx"}], "data_inputs_options": [{"type": "customized_var", "name": "site_api_name", "title": "Site API Name", "description": "This is the API Name of the site to pull request data from. This should not be a URL.", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}], "customized_options": [{"name": "site_api_name", "value": "yoursite"}], "code": "# encoding = utf-8\nfrom timeit import default_timer as timer\nfrom datetime import datetime, timezone, timedelta\nfrom sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time\n\n\"\"\"\n IMPORTANT\n Edit only the validate_input and collect_events functions.\n Do not edit any other part in this file.\n This file is generated only once when creating the modular input.\n\"\"\"\n\n# def use_single_instance_mode():\n# return True\n\n\ndef validate_input(helper, definition):\n # This example accesses the modular input variable\n site_name = definition.parameters.get(\"site_api_name\", None)\n if site_name is None or site_name == \"\":\n msg = \"The site_name can not be empty\"\n raise ValueError(\"InvalidSiteName\", msg)\n elif \"http\" in site_name:\n msg = (\n \"The site name is not the full URL it should be the \",\n \"API Name of the site like 'my_example_site'\",\n )\n raise ValueError(\"InvalidSiteName\", msg)\n elif \" \" in site_name:\n msg = (\n \"The site name should be the API Name of the site like \",\n \"not the Display Name. Example would be 'my_site_name' instead of \",\n \"My Site Name\",\n )\n raise ValueError(\"InvalidSiteName\", msg)\n pass\n\n\ndef collect_events(helper, ew):\n start = timer()\n loglevel = helper.get_log_level()\n helper.set_log_level(loglevel)\n # Proxy setting configuration\n # proxy_settings = helper.get_proxy()\n global_email = helper.get_global_setting(\"email\")\n global_api_token = helper.get_global_setting(\"api_token\")\n global_corp_api_name = helper.get_global_setting(\"corp_api_name\")\n api_host = \"https://dashboard.signalsciences.net\"\n helper.log_info(\"email: %s\" % global_email)\n helper.log_info(\"corp: %s\" % global_corp_api_name)\n\n def pull_requests(helper, current_site, delta, key=None):\n site_name = current_site\n last_name = f\"requests_last_until_time_{current_site}\"\n last_run_until = helper.get_check_point(last_name)\n\n if last_run_until is None:\n helper.log_info(\"no last_run_time found in checkpoint state\")\n helper.log_debug(\"get_from_until\")\n until_time, from_time = get_from_and_until_times(\n delta, five_min_offset=True\n )\n else:\n helper.log_info(f\"last_run_until found in state: {last_run_until}\")\n helper.log_debug(\"get_until\")\n until_time, from_time = get_until_time(\n helper, last_run_until, delta, five_min_offset=True\n )\n\n if from_time is None:\n helper.log_info(f\"{last_run_until} >= current now time, skipping run\")\n return\n\n if from_time >= until_time:\n helper.save_check_point(last_name, from_time)\n helper.log_info(\n f\"from_time {from_time} >= until_time {until_time}, skipping run\"\n )\n return\n helper.log_info(\"SiteName: %s\" % site_name)\n helper.log_info(f\"Start Period: {datetime.utcfromtimestamp(from_time)} UTC\")\n helper.log_info(f\"End Period: {datetime.utcfromtimestamp(until_time)} UTC\")\n\n input_name = helper.get_input_stanza_names()\n single_name = \"\"\n\n if type(input_name) is dict and input_name > 1:\n helper.log_info(\"Multi instance mode\")\n for current_name in input_name:\n single_name = current_name\n else:\n helper.log_info(\"Single instance mode\")\n helper.log_info(\"Inputs: %s\" % input_name)\n helper.log_info(\"Inputs Num: %s\" % len(input_name))\n single_name = input_name\n helper.log_info(f\"single_name: {single_name}\")\n\n # Loop across all the data and output it in one big JSON object\n url = (\n f\"{api_host}/api/v0/corps/{global_corp_api_name}\"\n f\"/sites/{site_name}/feed/requests?\"\n f\"from={from_time}&until={until_time}\"\n )\n helper.log_info(\"Pulling requests from requests API\")\n config = Config(\n url=url,\n api_host=api_host,\n from_time=from_time,\n until_time=until_time,\n global_email=global_email,\n global_corp_api_name=global_corp_api_name,\n current_site=current_site,\n )\n config.headers = {\n \"Content-type\": \"application/json\",\n \"x-api-user\": global_email,\n \"x-api-token\": global_api_token,\n \"User-Agent\": config.user_agent_string,\n }\n\n all_requests = get_results(\"Requests\", helper, config)\n\n total_requests = len(all_requests)\n helper.log_info(\"Total Requests Pulled: %s\" % total_requests)\n if total_requests == 0:\n helper.save_check_point(last_name, until_time)\n helper.log_info(\n f\"No events to write, saving checkpoint to value:{until_time}\"\n )\n write_start = timer()\n for current_event in all_requests:\n if key is None:\n source_type = helper.get_sourcetype()\n event = helper.new_event(\n source=single_name,\n index=helper.get_output_index(),\n sourcetype=source_type,\n data=current_event,\n )\n else:\n indexes = helper.get_output_index()\n current_index = indexes[key]\n types = helper.get_sourcetype()\n source_type = types[key]\n single_name = single_name[0]\n event = helper.new_event(\n source=single_name,\n index=current_index,\n sourcetype=source_type,\n data=current_event,\n )\n\n try:\n ew.write_event(event)\n helper.save_check_point(last_name, until_time)\n helper.log_info(f\"Event written, saving checkpoint:{until_time}\")\n except Exception as e:\n helper.log_error(f\"error writing event: {e}\")\n helper.log_error(event)\n raise e\n\n write_end = timer()\n write_time = write_end - write_start\n write_time_result = round(write_time, 2)\n helper.log_info(\"Total Event Output Time: %s seconds\" % write_time_result)\n\n # If multiple inputs configured it creates an array of values and the\n # script only gets called once per Input configuration\n all_sites = helper.get_arg(\"site_api_name\")\n time_deltas = helper.get_arg(\"interval\")\n helper.log_info(f\"interval: {time_deltas}\")\n\n if type(all_sites) is dict:\n helper.log_info(\"run_type: Sequential\")\n for active_input, site in all_sites.items():\n time_delta = int(time_deltas[active_input])\n helper.log_info(\"site: %s\" % site)\n pull_requests(helper, key=active_input, current_site=site, delta=time_delta)\n helper.log_info(\"Finished Pulling Requests for %s\" % site)\n else:\n helper.log_info(\"Run Type: Concurrent\")\n site = helper.get_arg(\"site_api_name\")\n helper.log_info(\"site: %s\" % site)\n pull_requests(helper, current_site=site, delta=int(time_deltas))\n helper.log_info(\"Finished Pulling Requests for %s\" % site)\n end = timer()\n total_time = end - start\n time_result = round(total_time, 2)\n helper.log_info(\"Total Script Time: %s seconds\" % time_result)\n", "uuid": "aaaeb391da9043e1819408033d9db708", "sample_count": 0}, {"index": "default", "sourcetype": "sigsci-activity", "interval": "300", "use_external_validation": true, "streaming_mode_xml": true, "name": "SigsciActivity", "title": "SigsciActivity", "description": "", "type": "customized", "parameters": [{"name": "place_holder", "label": "Place Holder", "help_string": "It was required to have one option even if it isn't needed. You can skip this one.", "required": false, "format_type": "text", "default_value": "", "placeholder": "Not needed", "type": "text", "value": ""}], "data_inputs_options": [{"type": "customized_var", "name": "place_holder", "title": "Place Holder", "description": "It was required to have one option even if it isn't needed. You can skip this one.", "required_on_edit": false, "required_on_create": false, "format_type": "text", "default_value": "", "placeholder": "Not needed"}], "code": "# encoding = utf-8\nfrom timeit import default_timer as timer\nimport json\nfrom datetime import datetime\nfrom sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time\n\n\"\"\"\n IMPORTANT\n Edit only the validate_input and collect_events functions.\n Do not edit any other part in this file.\n This file is generated only once when creating the modular input.\n\"\"\"\n\n\n# def use_single_instance_mode():\n# return True\n\n\ndef validate_input(helper, definition):\n # This example accesses the modular input variable\n pass\n\n\ndef collect_events(helper, ew):\n start = timer()\n loglevel = helper.get_log_level()\n helper.set_log_level(loglevel)\n # Proxy setting configuration\n # proxy_settings = helper.get_proxy()\n global_email = helper.get_global_setting(\"email\")\n global_api_token = helper.get_global_setting(\"api_token\")\n global_corp_api_name = helper.get_global_setting(\"corp_api_name\")\n api_host = \"https://dashboard.signalsciences.net\"\n helper.log_info(\"email: %s\" % global_email)\n helper.log_info(\"corp: %s\" % global_corp_api_name)\n\n def pull_events(delta, key=None):\n last_run_until = helper.get_check_point(\"activity_last_until_time\")\n helper.log_info(f\"last_run_until: {last_run_until}\")\n if last_run_until is None:\n (until_time, from_time) = get_from_and_until_times(\n delta, five_min_offset=False\n )\n else:\n (until_time, from_time) = get_until_time(\n helper, last_run_until, delta, five_min_offset=False\n )\n if from_time is None:\n helper.log_info(f\"{last_run_until} >= current now time, skipping run\")\n return\n if from_time >= until_time:\n helper.save_check_point(\"activity_last_until_time\", from_time)\n helper.log_info(\n f\"from_time {from_time} >= until_time {until_time}, skipping run\"\n )\n return\n helper.save_check_point(\"activity_last_until_time\", until_time)\n\n helper.log_info(f\"Start Period: {datetime.fromtimestamp(from_time)}\")\n helper.log_info(f\"End Period: {datetime.fromtimestamp(until_time)}\")\n\n input_name = helper.get_input_stanza_names()\n single_name = \"\"\n\n if type(input_name) is dict and input_name > 1:\n helper.log_info(\"Multi instance mode\")\n for current_name in input_name:\n single_name = current_name\n else:\n helper.log_info(\"Single instance mode\")\n helper.log_info(\"Inputs: %s\" % input_name)\n helper.log_info(\"Inputs Num: %s\" % len(input_name))\n single_name = input_name\n helper.log_info(f\"single_name: {single_name}\")\n\n # Loop across all the data and output it in one big JSON object\n url = (\n f\"{api_host}/api/v0/corps/{global_corp_api_name}\"\n f\"/activity?\"\n f\"from={from_time}&until={until_time}\"\n )\n config = Config(\n url=url,\n api_host=api_host,\n from_time=from_time,\n until_time=until_time,\n global_email=global_email,\n global_corp_api_name=global_corp_api_name,\n current_site=\"\",\n )\n config.headers = {\n \"Content-type\": \"application/json\",\n \"x-api-user\": global_email,\n \"x-api-token\": global_api_token,\n \"User-Agent\": config.user_agent_string,\n }\n helper.log_info(\"Pulling results from Corp Activity API\")\n all_events = get_results(\"Activity Events\", helper, config)\n total_requests = len(all_events)\n helper.log_info(\"Total Corp Activity Pulled: %s\" % total_requests)\n write_start = timer()\n for current_event in all_events:\n if key is None:\n source_type = helper.get_sourcetype()\n helper.log_info(\"Concurrent Mode\")\n source_type_info = type(source_type)\n active_index = helper.get_output_index()\n index_info = type(active_index)\n single_name_info = type(single_name)\n current_event_info = type(current_event)\n helper.log_info(f\"source_type: {source_type}\")\n helper.log_info(f\"source_type_info: {source_type_info}\")\n helper.log_info(f\"index: {active_index}\")\n helper.log_info(f\"index_info: {index_info}\")\n helper.log_info(f\"single_name: {single_name}\")\n helper.log_info(f\"single_name_info: {single_name_info}\")\n helper.log_info(f\"current_event: {current_event}\")\n helper.log_info(f\"current_event_info: {current_event_info}\")\n event = helper.new_event(\n source=single_name,\n index=helper.get_output_index(),\n sourcetype=source_type,\n data=current_event,\n )\n else:\n indexes = helper.get_output_index()\n current_index = indexes[key]\n types = helper.get_sourcetype()\n source_type = types[key]\n single_name = single_name[0]\n helper.log_info(\"Sequential Mode\")\n helper.log_info(f\"source_type: {source_type}\")\n helper.log_info(f\"index: {current_index}\")\n helper.log_info(f\"single_name: {single_name}\")\n helper.log_info(f\"current_event: {current_event}\")\n event = helper.new_event(\n source=single_name,\n index=current_index,\n sourcetype=source_type,\n data=current_event,\n )\n\n try:\n ew.write_event(event)\n except Exception as e:\n raise e\n write_end = timer()\n write_time = write_end - write_start\n write_time_result = round(write_time, 2)\n helper.log_info(f\"Total Corp Activity Output Time: {write_time_result} seconds\")\n\n # If multiple inputs configured it creates an array of values and the\n # script only gets called once per Input configuration\n time_deltas = helper.get_arg(\"interval\")\n helper.log_info(f\"interval: {time_deltas}\")\n if type(time_deltas) is dict:\n helper.log_info(\"run_type: Sequential\")\n for active_input in time_deltas:\n time_delta = time_deltas[active_input]\n time_delta = int(time_delta)\n helper.log_info(\"time_delta: %s\" % time_delta)\n pull_events(delta=time_delta, key=active_input)\n else:\n helper.log_info(\"Run Type: Concurrent\")\n helper.log_info(\"time_delta: %s\" % time_deltas)\n pull_events(delta=int(time_deltas))\n helper.log_info(\"Finished Pulling Corp Activity\")\n end = timer()\n total_time = end - start\n time_result = round(total_time, 2)\n helper.log_info(f\"Total Script Time: {time_result} seconds\")\n", "customized_options": [{"name": "place_holder", "value": ""}], "uuid": "07b37d1943e942cf831c8ee85ffcb4a2", "sample_count": 0}]}, "field_extraction_builder": {"sigsci-event": {"is_parsed": true, "data_format": "json"}, "sigsci-activity": {"is_parsed": true, "data_format": "json"}, "sigsci-requests": {"is_parsed": true, "data_format": "json"}}, "global_settings_builder": {"global_settings": {"proxy_settings": {"proxy_type": "http"}, "log_settings": {"log_level": "DEBUG"}, "customized_settings": [{"required": true, "name": "email", "label": "E-mail", "default_value": "", "placeholder": "example@example.com", "help_string": "This it the e-mail username of the user who has the correct permissions for the app to pull the data.", "type": "text", "format_type": "text", "value": "youremail@yourdomain.com"}, {"required": true, "name": "corp_api_name", "label": "Corp", "default_value": "", "placeholder": "", "help_string": "This is the API name of your corp.", "type": "text", "format_type": "text", "value": "corpname"}, {"required": true, "name": "api_token", "label": "API Token", "placeholder": "", "default_value": "", "help_string": "This is the API Token of the user who has the correct permissions. The API Token is tied to the username.", "type": "password", "format_type": "password", "value": ""}]}}, "sourcetype_builder": {"sigsci-requests": {"metadata": {"event_count": 0, "data_input_name": "SigsciRequests", "extractions_count": 0, "cims_count": 0}}, "sigsci-event": {"metadata": {"event_count": 0, "data_input_name": "SigsciEvent", "extractions_count": 0, "cims_count": 0}}, "sigsci-activity": {"metadata": {"event_count": 0, "data_input_name": "SigsciActivity", "extractions_count": 0, "cims_count": 0}}}, "validation": {"validators": ["best_practice_validation", "data_model_mapping_validation", "field_extract_validation", "app_cert_validation"], "status": "job_started", "validation_id": "v_1692022983_67"}} \ No newline at end of file diff --git a/splunkbase_prepped_file/sigsci_TA_for_splunk-1.0.36.spl b/splunkbase_prepped_file/sigsci_TA_for_splunk-1.0.36.spl new file mode 100644 index 0000000..183cb55 Binary files /dev/null and b/splunkbase_prepped_file/sigsci_TA_for_splunk-1.0.36.spl differ