Merge pull request #38 from fastly/refactor-time-epoch

Release - 1.0.37

.gitignore

@@ -2,4 +2,7 @@
 *.tar
 .DS_Store
 *.log
-.idea
+.idea
+*jupyter-env
+*.ipynb
+dev/

sigsci_TA_for_splunk/README.txt

@@ -9,26 +9,4 @@ This is an add-on powered by the Splunk Add-on Builder.
 /opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui.exe: this file does not require any source code
 /opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-arm64.exe: this file does not require any source code
 /opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-arm64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code
-# Binary File Declaration
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/pvectorc.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-32.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-arm64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-arm64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-32.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-64.exe: this file does not require any source code
-# Binary File Declaration
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/pvectorc.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-32.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-arm64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-arm64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-64.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/gui-32.exe: this file does not require any source code
-/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/setuptools/cli-64.exe: this file does not require any source code
+/opt/splunk/var/data/tabuilder/package/sigsci_TA_for_splunk/bin/sigsci_ta_for_splunk/aob_py3/markupsafe/_speedups.cpython-37m-x86_64-linux-gnu.so: this file does not require any source code

sigsci_TA_for_splunk/app.manifest

@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "sigsci_TA_for_splunk",
-      "version": "1.0.36"
+      "version": "1.0.37"
     },
     "author": [
       {

sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py

@@ -1,6 +1,7 @@
 # encoding = utf-8
 from timeit import default_timer as timer
 import json
+import time
 from datetime import datetime
 from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time
 
@@ -39,7 +40,7 @@ def pull_events(delta, key=None):
         helper.log_info(f"last_run_until: {last_run_until}")
         if last_run_until is None:
             (until_time, from_time) = get_from_and_until_times(
-                delta, five_min_offset=False
+                helper, delta, five_min_offset=False
             )
         else:
             (until_time, from_time) = get_until_time(
@@ -56,8 +57,8 @@ def pull_events(delta, key=None):
             return
         helper.save_check_point("activity_last_until_time", until_time)
 
-        helper.log_info(f"Start Period: {datetime.fromtimestamp(from_time)}")
-        helper.log_info(f"End Period: {datetime.fromtimestamp(until_time)}")
+        helper.log_info(f"Start Period: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(from_time))}")
+        helper.log_info(f"End Period: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(until_time))}")
 
         input_name = helper.get_input_stanza_names()
         single_name = ""

sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py

@@ -2,6 +2,7 @@
 from timeit import default_timer as timer
 import requests
 import json
+import time
 from datetime import datetime
 from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time
 
@@ -58,7 +59,7 @@ def pull_events(current_site, delta, key=None):
         helper.log_info(f"last_run_until: {last_run_until}")
         if last_run_until is None:
             (until_time, from_time) = get_from_and_until_times(
-                delta, five_min_offset=False
+                helper, delta, five_min_offset=False
             )
         else:
             (until_time, from_time) = get_until_time(
@@ -76,8 +77,8 @@ def pull_events(current_site, delta, key=None):
         helper.save_check_point(last_name, until_time)
         helper.log_info("SiteName: %s" % site_name)
 
-        helper.log_info(f"Start Period: {datetime.fromtimestamp(from_time)}")
-        helper.log_info(f"End Period: {datetime.fromtimestamp(until_time)}")
+        helper.log_info(f"Start Period: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(from_time))}")
+        helper.log_info(f"End Period: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(until_time))}")
 
         input_name = helper.get_input_stanza_names()
         single_name = ""

sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py

@@ -1,5 +1,6 @@
 # encoding = utf-8
 from timeit import default_timer as timer
+import time
 from datetime import datetime, timezone, timedelta
 from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time
 
@@ -53,12 +54,13 @@ def pull_requests(helper, current_site, delta, key=None):
         site_name = current_site
         last_name = f"requests_last_until_time_{current_site}"
         last_run_until = helper.get_check_point(last_name)
-
+
+
         if last_run_until is None:
             helper.log_info("no last_run_time found in checkpoint state")
             helper.log_debug("get_from_until")
             until_time, from_time = get_from_and_until_times(
-                delta, five_min_offset=True
+                helper, delta, five_min_offset=True
             )
         else:
             helper.log_info(f"last_run_until found in state: {last_run_until}")
@@ -77,9 +79,10 @@ def pull_requests(helper, current_site, delta, key=None):
                 f"from_time {from_time} >= until_time {until_time}, skipping run"
             )
             return
+
         helper.log_info("SiteName: %s" % site_name)
-        helper.log_info(f"Start Period: {datetime.utcfromtimestamp(from_time)} UTC")
-        helper.log_info(f"End Period: {datetime.utcfromtimestamp(until_time)} UTC")
+        helper.log_info(f"Start Period: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(from_time))}")
+        helper.log_info(f"End Period: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(until_time))}")
 
         input_name = helper.get_input_stanza_names()
         single_name = ""
@@ -128,6 +131,7 @@ def pull_requests(helper, current_site, delta, key=None):
                 f"No events to write, saving checkpoint to value:{until_time}"
             )
         write_start = timer()
+        event_count = 0 
         for current_event in all_requests:
             if key is None:
                 source_type = helper.get_sourcetype()
@@ -152,13 +156,14 @@ def pull_requests(helper, current_site, delta, key=None):
 
             try:
                 ew.write_event(event)
+                event_count += 1  # increment the count for successful events to not spam debug.
                 helper.save_check_point(last_name, until_time)
-                helper.log_info(f"Event written, saving checkpoint:{until_time}")
             except Exception as e:
                 helper.log_error(f"error writing event: {e}")
                 helper.log_error(event)
                 raise e
-
+        if event_count != 0: # We save the checkpoint earlier on 0 events.
+            helper.log_info(f"{event_count} events written, saving checkpoint: {until_time}")        
         write_end = timer()
         write_time = write_end - write_start
         write_time_result = round(write_time, 2)

sigsci_TA_for_splunk/bin/sigsci_helper.py

@@ -4,7 +4,6 @@
 import time
 import requests
 
-
 def check_response(
     code,
     response_text,
@@ -80,67 +79,56 @@ def get_request_data(url, headers, helper):
 
     return data, response_code, response_error
 
-
 def timestamp_sanitise(_time):
-    new_time = datetime.utcfromtimestamp(_time).replace(second=0)
-    new_time = int(new_time.timestamp())
-    return new_time
+    return _time - _time % 60
 
+def get_from_and_until_times(helper, delta, five_min_offset=False):
+    # Get the current epoch time
+    until_time = int(time.time())
+    helper.log_info(f"Time Now: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(int(time.time())))}")
 
-def get_from_and_until_times(delta, five_min_offset=False):
+
+    # Check if five_min_offset is needed.
     if five_min_offset:
-        until_time = datetime.now(timezone.utc) - timedelta(minutes=5)
-    else:
-        until_time = datetime.now(timezone.utc)
-    from_time = until_time - timedelta(seconds=delta)
+        until_time -= 5 * 60  # Subtract 5 minutes in seconds
 
-    if five_min_offset:
-        until_time = until_time.replace(second=0)
-        from_time = from_time.replace(second=0)
+    # Always sanitize the until_time irrespective of five_min_offset, 
+    # because it makes sure the timestamp is always aligned to a whole minute boundary.
+    until_time = timestamp_sanitise(until_time)
 
-    until_time = int(until_time.timestamp())
-    from_time = int(from_time.timestamp())
+    # Get the starting time.
+    from_time = until_time - delta
+
+    # If five_min_offset, then sanitize from_time as well
+    if five_min_offset:
+        from_time = timestamp_sanitise(from_time)
 
-    return timestamp_sanitise(until_time), timestamp_sanitise(from_time)
+    return until_time, from_time
 
+SECONDS_IN_DAY= 24 * 60 * 60
 
 def get_until_time(helper, from_time, interval, five_min_offset=False):
-    now = datetime.now(timezone.utc).replace(second=0)
-    interval_timedelta = timedelta(seconds=interval)
-
-    current_time_offset = datetime.now(timezone.utc).replace(second=0)
+    # Get current epoch time rounded down to nearest minute
+    now = timestamp_sanitise(int(time.time()))
+    helper.log_info(f"Time Now: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(now))}")
 
     if five_min_offset:
-        current_time_offset = current_time_offset - timedelta(minutes=5)
-
-    # Get dt object with UTC timezone.
-    from_time_dt_obj = (
-        datetime.utcfromtimestamp(from_time)
-        .replace(tzinfo=timezone.utc)
-        .replace(second=0)
-    )
-
-    # How far back is the "from time" from now
-    ft_diff = now - from_time_dt_obj
+        until_time = now - 5 * 60  # Subtract 5 minutes in seconds
+    else:
+        until_time = now
 
-    # The default time to look at until, either now or five minutes ago.
-    _until_time = current_time_offset
-    _rslt_until = int(datetime.timestamp(_until_time))
+    # Calculate the difference between now and the from_time
+    time_difference = now - from_time
 
-    # If we are futher back than 24 hours reset the clock.
-    if ft_diff > timedelta(hours=24):
+    # If the difference is more than 24 hours (in seconds), reset the from_time
+    if time_difference > SECONDS_IN_DAY:  # 24 hours in seconds
         helper.log_info("Adjusting from_time to 24 hours ago")
+        adjusted_from_time = now - SECONDS_IN_DAY  # Subtract 24 hours in seconds
+        helper.log_info(f"Previous Run: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(from_time))}")
+        helper.log_info(f"Adjusted from_time: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(adjusted_from_time))}")
+        return until_time, adjusted_from_time
 
-        from_time = now - timedelta(hours=24)
-        from_time.replace(second=0)
-        from_time_int = int(from_time.timestamp())
-
-        return _rslt_until, from_time_int
-
-    from_time_int = int(datetime.timestamp(from_time_dt_obj))
-
-    return _rslt_until, from_time_int
-
+    return until_time, from_time
 
 def get_results(title, helper, config):
     loop = True
@@ -236,8 +224,8 @@ class Config:
     url: str
     headers: dict
     events: dict
-    from_time: str
-    until_time: str
+    from_time: int
+    until_time: int
     global_email: str
     global_corp_api_name: str
     current_site: str
@@ -266,7 +254,7 @@ def __init__(
         self.global_corp_api_name = global_corp_api_name
         self.current_site = current_site
         self.event_ids = []
-        self.user_agent_version = "1.0.36"
+        self.user_agent_version = "1.0.37"
         self.user_agent_string = (
             f"TA-sigsci-waf/{self.user_agent_version} "
             f"(PythonRequests {requests.__version__})"

sigsci_TA_for_splunk/default/app.conf

@@ -3,11 +3,11 @@
 state_change_requires_restart = false
 is_configured = 0
 state = enabled
-build = 14
+build = 15
 
 [launcher]
 author = Fastly
-version = 1.0.36
+version = 1.0.37
 description = For users of Fastly who would like to enrich their Splunk data with information from Fastly. This app with simple configuration enabled the importing of Events, Activity, and raw request information to Splunk.This is an open source project, no support provided, public repository is available and installation documentation can be found at https://github.com/fastly/sigsci-splunk-app. The best way to report issues with the app is to create an Issue on the github page so that it can be tracked.
 
 [ui]

sigsci_TA_for_splunk/default/inputs.conf

@@ -19,3 +19,14 @@ sourcetype = sigsci-activity
 interval = 300
 disabled = 0
 
+[SigsciRequests://Demo_SigSciRequests]
+interval = 300
+site_api_name = YourSite
+
+[SigsciEvent://Demo_SigsciEvents]
+interval = 60
+site_api_name = YourSite
+
+[SigsciActivity://Demo_SigSciActivity]
+interval = 60
+