Add bulk logging of function invocations. Tracing tips for deluge cra…

…sh bot output

docs/CONTRIBUTING.md

@@ -219,6 +219,47 @@ The only build able to send debug messages via sysex is the "debug" build, so yo
 To make debug log prints in your code, which will be sent to the console, here is a code example:
 `D_PRINTLN("my log message which prints an integer value %d", theIntegerValue);`
 
+Similar functionality is available for popup windows
+
+`D_POPUP("%d bottles of beer on the wall.", 100/2);`
+
+### Automatically logging all function calls. 
+
+A task exists that will traverse through the source files in a path and annotate each function with a `D_PRINT` statement, revealing the line and the calling function call on each instance. 
+
+`dbt annotate add -p src/deluge/gui/views`
+
+This will add a logging line to every function invocation unless the function is in the blacklist in `scripts/tasks/task-annotate.py`
+
+```
+ArrangerView::ArrangerView() {
+	D_PRINTLN("ArrangerView"); // ArrangerView DBT:ANNOTATE
+	doingAutoScrollNow = false;
+
+	lastInteractedOutputIndex = 0;
+	lastInteractedPos = -1;
+	lastInteractedSection = 0;
+	lastInteractedClipInstance = nullptr;
+}
+
+void ArrangerView::renderOLED(deluge::hid::display::oled_canvas::Canvas& canvas) {
+	D_PRINTLN("renderOLED"); // renderOLED DBT:ANNOTATE
+	sessionView.renderOLED(canvas);
+}
+
+void ArrangerView::moveClipToSession() {
+	D_PRINTLN("moveClipToSession"); // moveClipToSession DBT:ANNOTATE
+	Output* output = outputsOnScreen[yPressedEffective];
+```
+
+To remove the potentially hundreds of added log invocations. simply call
+
+`dbt annotate rm -p src/deluge/ui/views`
+
+each logging call added during `dbt annotate add` has a string marker used to later remove the annotations
+
+
+
 ### Deluge Crash Reader Discord Bot
 
 If deluge crashes, there is a colorful pixelated image that gets displayed across the main pads and sidebar. In case 
@@ -247,6 +288,8 @@ I couldn't find a recent matching commit for `0x714a`
 Then run the command that Deluge Crash Reader Bot outputs. This outputs the latest stack trace before hard crashing.
 (If you're on Windows and using VS Code, try running `.\dbt.cmd shell` first in your terminal.)
 
+Alternatively, you can run `dbt trace`, choose the firmware binary the crash originated from, and paste in the output of the discord bot. this will provide line numbers and file reeferences as well.
+
 [Black Box PR](https://github.com/SynthstromAudible/DelugeFirmware/pull/660)
 [Discord Bot Repo](https://github.com/0beron/delugeqr/)
 

scripts/tasks/task-annotate.py

@@ -0,0 +1,165 @@
+import os
+import clang.cindex
+import argparse
+import subprocess
+from iterfzf import iterfzf
+
+
+FN_BLACKLIST = [
+    "graphicsRoutine"
+]
+tags_file = "tags"  # Replace with the actual path to your tags file
+
+def argparser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="annotate",
+        description="Adds print statements everywhere"
+    )
+    parser.add_argument('action', choices=['add', 'rm'], help="The action to perform")
+    parser.add_argument(
+        "-p", "--path", help="the directory to process"
+    )
+    parser.add_argument(
+        "-c", "--comment", help="A comment added to the annotation to help with search/removal", default="DBT:ANNOTATE"
+    )
+    parser.add_argument(
+        "-m", "--max", help="Maximum number of files to annotate"
+    )
+    parser.add_argument(
+        "-t", "--template", help="A template for the log statement", default='D_PRINTLN("{function}");'
+    )
+
+    return parser
+
+def remove_specific_lines(file_path, text_to_remove):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+        filtered_lines = [line for line in lines if text_to_remove not in line]
+        if len(filtered_lines) < len(lines):
+            with open(file_path, 'w') as file:
+                file.writelines(filtered_lines)
+            print(f"Removed {len(lines) - len(filtered_lines)} lines from {file_path}")
+            return
+
+def remove_annotations(args):
+    for root, dirs, files in os.walk(args.path):
+        for file in files:
+            if file.endswith('.cpp'):
+                remove_specific_lines(os.path.join(root, file), args.comment)
+
+def is_function_definition(node):
+    """
+    Check if the node is a function definition.
+    """
+    if node.kind != clang.cindex.CursorKind.CXX_METHOD:
+        return False
+    for child in node.get_children():
+        if child.kind == clang.cindex.CursorKind.COMPOUND_STMT:
+            return True
+    return False
+
+
+
+def ensure_tags_exist():
+    result = subprocess.run(["ctags", "--version"])
+    ctags_binary = "ctags"
+    if result.returncode != 0:
+        def iter_ctags_binaries():
+            which_ctags_result = subprocess.run(["which", "-a", "ctags"], capture_output=True)
+            for line in which_ctags_result.stdout.decode('utf-8').splitlines():
+                yield line
+        print("Default ctags is erroring out on `ctags --version`. GNU standard ctags required. (brew install universal-ctags on MacOSX).")
+        ctags_binary = iterfzf(iter_ctags_binaries(), prompt = 'Please choose the appropriate ctags path: ')
+
+    ctags_invocation = [ctags_binary, "-R", "-x", "--c++-types=f", "--extras=q", "--format=1", 
+                    "src/deluge/gui/views" ,
+                    "src/deluge/deluge.cpp",
+                    "src/deluge/model/action",
+                    "src/deluge/model/clip", 
+                    "src/deluge/model/consequence",
+                    "src/deluge/model/drum",
+                    "src/deluge/model/instrument"]
+    print(f"invoking {' '.join(ctags_invocation)}")
+    result = subprocess.run(ctags_invocation, capture_output=True)
+    if result.returncode == 0:
+        with open(tags_file, "w+") as tags_file_pointer:
+            tags_file_pointer.write(result.stdout.decode('utf-8'))
+    else:
+        print(f"Failed to invoke {" ".join(ctags_invocation)}")
+        exit(1)
+
+def parse_ctags_output(tags_file):
+    function_map = {}
+    ensure_tags_exist()
+    with open(tags_file, 'r') as file:
+        for line in file:
+            parts = line.split()
+            if len(parts) >= 3:
+                function_name, line_number, file_path = parts[0], int(parts[1]), parts[2]
+
+                # Check if the function (namespaced or not) is already in the map
+                already_included = any(fn in function_name or function_name in fn for fn, _ in function_map.get(file_path, []))
+
+                if file_path not in function_map:
+                    function_map[file_path] = []
+
+                blacklisted = False
+                for blacklisted_fn in FN_BLACKLIST:
+                    if blacklisted_fn in function_name:
+                        blacklisted = True
+                        break
+                # Add the function if it's not a non-namespaced version of an existing namespaced function
+                if not already_included and not blacklisted:
+                    function_map[file_path].append((function_name, line_number))
+    return function_map
+
+def add_annotations(args):
+    function_map = parse_ctags_output(tags_file)
+
+    processed_files = 0
+    for root, dirs, files in os.walk(args.path, followlinks=False):
+        for file in files:
+            if file.endswith('.cpp'):
+                file_path = os.path.join(root, file)
+                if file_path in function_map:
+                    functions = sorted(function_map[file_path], key=lambda x: x[1], reverse=True)
+                    if args.max and processed_files > int(args.max):
+                        break
+                    annotate_file(file_path, functions, args)
+                    processed_files += 1
+
+def annotate_file(file_path, functions, args):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    for function_name, line_number in functions:
+        # Search for the opening curly brace from the line number
+        insertion_point = line_number
+        for i in range(line_number - 1, min(line_number + 5, len(lines))):  # Check next few lines for the opening brace
+            if '{' in lines[i]:
+                insertion_point = i + 1  # Insert after the opening brace
+                break
+
+        # basename without extension
+        basename = os.path.basename(file_path)
+        debug_line = args.template.replace('{function}', f"{function_name}").replace('{file}', basename)
+        debug_line += f" // {function_name} {args.comment}\n"
+
+        if debug_line.strip() not in lines[insertion_point].strip():  # Avoid duplicates
+            lines.insert(insertion_point, debug_line)
+
+        print("".join(lines))
+    # Uncomment to write changes
+    with open(file_path, 'w') as file:
+        file.writelines(lines)
+
+# Rest of the script remains the same...
+
+
+def main() -> int:
+    args = argparser().parse_args()
+    if (args.action == 'rm'):
+        remove_annotations(args)
+    else:
+        add_annotations(args)
+    return 0

scripts/tasks/task-trace.py

@@ -0,0 +1,70 @@
+#! /usr/bin/env python3
+import argparse
+from pathlib import Path
+from iterfzf import iterfzf
+import datetime
+import os
+
+def argparser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="trace",
+        description="Parse crash reader output into stack trace. e.g. ``",
+    )
+    parser.group = "Debugging"
+    return parser
+
+
+def parse_nmdump(file_path):
+    with open(file_path, 'r') as file:
+        nmdump_data = []
+        for line in file:
+            parts = line.split()
+            if len(parts) >= 3:
+                address = int(parts[0])  # Convert hex to int
+                label = ' '.join(parts[2:])
+                nmdump_data.append((address, label))
+        nmdump_data.sort()  # Sort by address
+    return nmdump_data
+
+def find_closest_match(build_basename, nmdump_data, addresses):
+    print("addr_hex\taddr_dec\taddr_src\tfile:line\tlabel")
+    for addr in addresses:
+        target = int(addr, 16)  # Convert hex to int
+        closest_label = None
+        for index, (nmdump_addr, label) in enumerate(nmdump_data):
+            if nmdump_addr <= target:
+                closest_label = f"{target}\t{nmdump_addr}\t{build_basename}:{index}\t{label}"
+            else:
+                break
+        print(f"{addr}\t{closest_label}")
+
+def input_addresses():
+    print("Enter the addresses separated by newlines (press Enter twice to finish):")
+    input_lines = []
+    while True:
+        line = input()
+        if line == "":
+            break
+        input_lines.append(line)
+    return input_lines
+
+
+def main() -> int:
+    source_build_dir = Path(f"{os.path.dirname(os.path.abspath(__file__))}/../../build/Debug")
+    source_bins = [f for f in source_build_dir.glob('*.nmdump')]
+    bins = sorted(source_bins, key=lambda b: -os.path.getmtime(b)) 
+
+    def iter_bins():
+        for bin in bins:
+            yield "\t".join([str(datetime.datetime.fromtimestamp(os.path.getmtime(bin))), os.path.basename(bin)]) 
+
+    chosen_display_option = iterfzf(iter_bins(), prompt=f"please confirm your target build file.")
+    [tiem, build_basename] = chosen_display_option.split("\t")
+    chosen_build = Path(source_build_dir, build_basename)
+    data = parse_nmdump(chosen_build)
+    addresses = input_addresses()
+    find_closest_match(build_basename, data, addresses)
+    return 0
+
+if __name__ == "__main__":
+    main()

scripts/toolchain/requirements.txt

@@ -2,4 +2,6 @@ certifi==2024.6.2
 setuptools==70.1.1
 pyserial==3.5
 ansi==0.3.7
-python-rtmidi==1.5.8
+python-rtmidi==1.5.8
+iterfzf
+libclang