Fix/window event race condition

openadapt/alembic/versions/54b42e4fd1a4_add_a11y_event_remove_state_from_window_.py

@@ -0,0 +1,49 @@
+"""add_a11y_event_remove_state_from_window_event
+
+Revision ID: 54b42e4fd1a4
+Revises: bb25e889ad71
+Create Date: 2024-07-23 16:36:45.883878
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite
+import openadapt
+
+# revision identifiers, used by Alembic.
+revision = '54b42e4fd1a4'
+down_revision = 'bb25e889ad71'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('a11y_event',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True),
+    sa.Column('handle', sa.Integer(), nullable=True),
+    sa.Column('data', sa.JSON(), nullable=True),
+    sa.Column('counter', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['counter'], ['window_event.counter'], name=op.f('fk_a11y_event_counter_window_event')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_a11y_event'))
+    )
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('handle', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('counter', sa.Integer(), nullable=True))
+        batch_op.drop_column('state')
+        batch_op.drop_column('window_id')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True))
+        batch_op.add_column(sa.Column('state', sqlite.JSON(), nullable=True))
+        batch_op.drop_column('counter')
+        batch_op.drop_column('handle')
+
+    op.drop_table('a11y_event')
+    # ### end Alembic commands ###

openadapt/db/crud.py

@@ -27,6 +27,7 @@
     Screenshot,
     ScrubbedRecording,
     WindowEvent,
+    A11yEvent,
     copy_sa_instance,
 )
 from openadapt.privacy.base import ScrubbingProvider
@@ -262,6 +263,25 @@ def insert_recording(session: SaSession, recording_data: dict) -> Recording:
     return db_obj
 
 
+def insert_a11y_event(
+    db: SaSession,
+    data: dict,
+) -> None:
+    """Insert an a11y event into the database.
+
+    Args:
+        db: The database session.
+        data: The data associated with the a11y event.
+    """
+    handle = data["handle"]
+    a11y_data = data["state"]
+    counter = data["counter"]
+
+    a11y_event = A11yEvent(handle=handle, data=a11y_data, counter=counter)
+    db.add(a11y_event)
+    db.commit()
+
+
 def delete_recording(session: SaSession, recording: Recording) -> None:
     """Remove the recording from the db.
 
@@ -590,6 +610,31 @@ def get_window_events(
     )
 
 
+def get_a11y_events(
+    session: SaSession,
+    recording: Recording,
+) -> list[A11yEvent]:
+    """Get accessibility events for a given recording.
+
+    Args:
+        session (SaSession): The SQLAlchemy session.
+        recording (Recording): The recording object.
+
+    Returns:
+        list[A11yEvent]: A list of accessibility events for the recording.
+    """
+    return (
+        session.query(A11yEvent)
+        .join(WindowEvent, A11yEvent.handle == WindowEvent.handle)
+        .filter(WindowEvent.recording_id == recording.id)
+        .options(
+            joinedload(A11yEvent.window_event).joinedload(WindowEvent.recording),
+        )
+        .order_by(A11yEvent.counter)
+        .all()
+    )
+
+
 def disable_action_event(session: SaSession, event_id: int) -> None:
     """Disable an action event.
 

openadapt/events.py

@@ -46,6 +46,7 @@ def get_events(
     action_events = crud.get_action_events(db, recording)
     window_events = crud.get_window_events(db, recording)
     screenshots = crud.get_screenshots(db, recording)
+    a11y_events = crud.get_a11y_events(db, recording)
 
     if recording.original_recording_id:
         # if recording is a copy, it already has its events processed when it
@@ -62,10 +63,12 @@ def get_events(
     assert num_action_events > 0, "No action events found."
     num_window_events = len(window_events)
     num_screenshots = len(screenshots)
+    num_a11y_events = len(a11y_events)
 
     num_action_events_raw = num_action_events
     num_window_events_raw = num_window_events
     num_screenshots_raw = num_screenshots
+    num_a11y_events_raw = num_a11y_events
     duration_raw = action_events[-1].timestamp - action_events[0].timestamp
 
     num_process_iters = 0
@@ -76,26 +79,31 @@ def get_events(
                 f"{num_action_events=} "
                 f"{num_window_events=} "
                 f"{num_screenshots=}"
+                f"{num_a11y_events=}"
             )
             (
                 action_events,
                 window_events,
                 screenshots,
+                a11y_events,
             ) = process_events(
                 action_events,
                 window_events,
                 screenshots,
+                a11y_events,
             )
             if (
                 len(action_events) == num_action_events
                 and len(window_events) == num_window_events
                 and len(screenshots) == num_screenshots
+                and len(a11y_events) == num_a11y_events
             ):
                 break
             num_process_iters += 1
             num_action_events = len(action_events)
             num_window_events = len(window_events)
             num_screenshots = len(screenshots)
+            num_a11y_events = len(a11y_events)
             if num_process_iters == MAX_PROCESS_ITERS:
                 break
 
@@ -116,6 +124,10 @@ def get_events(
             num_screenshots,
             num_screenshots_raw,
         )
+        meta["num_a11y_events"] = format_num(
+            num_a11y_events,
+            num_a11y_events_raw,
+        )
 
         duration = action_events[-1].timestamp - action_events[0].timestamp
         if len(action_events) > 1:
@@ -797,17 +809,20 @@ def process_events(
     action_events: list[models.ActionEvent],
     window_events: list[models.WindowEvent],
     screenshots: list[models.Screenshot],
+    a11y_events: list[models.A11yEvent],
 ) -> tuple[
     list[models.ActionEvent],
     list[models.WindowEvent],
     list[models.Screenshot],
+    list[models.A11yEvent],
 ]:
     """Process action events, window events, and screenshots.
 
     Args:
         action_events (list): The list of action events.
         window_events (list): The list of window events.
         screenshots (list): The list of screenshots.
+        a11y_events (list): The list of a11y events.
 
     Returns:
         tuple: A tuple containing the processed action events, window events,
@@ -821,9 +836,16 @@ def process_events(
     num_action_events = len(action_events)
     num_window_events = len(window_events)
     num_screenshots = len(screenshots)
-    num_total = num_action_events + num_window_events + num_screenshots
+    num_a11y_events = len(a11y_events)
+    num_total = (
+        num_action_events + num_window_events + num_screenshots + num_a11y_events
+    )
     logger.info(
-        f"before {num_action_events=} {num_window_events=} {num_screenshots=} "
+        "before "
+        f"{num_action_events=} "
+        f"{num_window_events=} "
+        f"{num_screenshots=} "
+        f"{num_a11y_events=} "
         f"{num_total=}"
     )
     process_fns = [
@@ -862,19 +884,36 @@ def process_events(
             action_events,
             "screenshot_timestamp",
         )
+        a11y_events = discard_unused_events(
+            a11y_events,
+            action_events,
+            "window_event_timestamp",
+        )
     num_action_events_ = len(action_events)
     num_window_events_ = len(window_events)
     num_screenshots_ = len(screenshots)
-    num_total_ = num_action_events_ + num_window_events_ + num_screenshots_
+    num_a11y_events_ = len(a11y_events)
+    num_total_ = (
+        num_action_events_ + num_window_events_ + num_screenshots_ + num_a11y_events_
+    )
     pct_action_events = num_action_events_ / num_action_events
     pct_window_events = num_window_events_ / num_window_events
+    pct_a11y_events = num_a11y_events_ / num_a11y_events
     pct_screenshots = num_screenshots_ / num_screenshots
     pct_total = num_total_ / num_total
     logger.info(
-        f"after {num_action_events_=} {num_window_events_=} {num_screenshots_=} "
+        "after "
+        f"{num_action_events_=}, "
+        f"{num_window_events_=}, "
+        f"{num_screenshots_=}, "
+        f"{num_a11y_events_=}, "
         f"{num_total_=}"
     )
     logger.info(
-        f"{pct_action_events=} {pct_window_events=} {pct_screenshots=} {pct_total=}"
+        f"{pct_action_events=} "
+        f"{pct_window_events=} "
+        f"{pct_screenshots=} "
+        f"{pct_a11y_events=} "
+        f"{pct_total=}"
     )
-    return action_events, window_events, screenshots
+    return action_events, window_events, screenshots, a11y_events

openadapt/models.py

@@ -491,6 +491,34 @@ def to_prompt_dict(self) -> dict[str, Any]:
         return action_dict
 
 
+class A11yEvent(db.Base):
+    """Class representing an accessibility (a11y) event in the database."""
+
+    __tablename__ = "a11y_event"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    timestamp = sa.Column(ForceFloat)
+    handle = sa.Column(sa.Integer)
+    data = sa.Column(sa.JSON)
+    counter = sa.Column(
+        sa.Integer, sa.ForeignKey("window_event.counter"), nullable=True
+    )
+
+    window_event = sa.orm.relationship("WindowEvent", back_populates="a11y_event")
+
+    def __init__(self, handle: int, data: dict, counter: int = None) -> None:
+        """Initialize an A11yEvent instance.
+
+        Args:
+            handle (int): The handle associated with the a11y event.
+            data (dict): The data related to the a11y event.
+            counter (int, optional): The counter value related to the window event.
+        """
+        self.handle = handle
+        self.data = data
+        self.counter = counter
+
+
 class WindowEvent(db.Base):
     """Class representing a window event in the database."""
 
@@ -500,16 +528,19 @@ class WindowEvent(db.Base):
     recording_timestamp = sa.Column(ForceFloat)
     recording_id = sa.Column(sa.ForeignKey("recording.id"))
     timestamp = sa.Column(ForceFloat)
-    state = sa.Column(sa.JSON)
     title = sa.Column(sa.String)
     left = sa.Column(sa.Integer)
     top = sa.Column(sa.Integer)
     width = sa.Column(sa.Integer)
     height = sa.Column(sa.Integer)
-    window_id = sa.Column(sa.String)
+    handle = sa.Column(sa.Integer)
+    counter = sa.Column(sa.Integer)
 
     recording = sa.orm.relationship("Recording", back_populates="window_events")
     action_events = sa.orm.relationship("ActionEvent", back_populates="window_event")
+    a11y_event = sa.orm.relationship(
+        "A11yEvent", uselist=False, back_populates="window_event"
+    )
 
     @classmethod
     def get_active_window_event(
@@ -525,7 +556,16 @@ def get_active_window_event(
         Returns:
             (WindowEvent) the active window event.
         """
-        return WindowEvent(**window.get_active_window_data(include_window_data))
+        window_event_data = window.get_active_window_data(include_window_data)
+
+        a11y_event_data = window_event_data.get("state")
+        window_event_data.pop("state", None)
+        a11y_event_handle = window_event_data.get("handle")
+        a11y_event = A11yEvent(data=a11y_event_data, handle=a11y_event_handle)
+        window_event = WindowEvent(**window_event_data)
+        window_event.a11y_event = a11y_event
+
+        return window_event
 
     def scrub(self, scrubber: ScrubbingProvider | TextScrubbingMixin) -> None:
         """Scrub the window event."""
@@ -556,7 +596,14 @@ def to_prompt_dict(self, include_data: bool = True) -> dict[str, Any]:
         if "state" in window_dict:
             if include_data:
                 key_suffixes = [
-                    "value", "h", "w", "x", "y", "description", "title", "help",
+                    "value",
+                    "h",
+                    "w",
+                    "x",
+                    "y",
+                    "description",
+                    "title",
+                    "help",
                 ]
                 if sys.platform == "win32":
                     logger.warning(