diff --git a/assets/message-chatgpt.png b/assets/message-chatgpt.png new file mode 100644 index 0000000..419b13d Binary files /dev/null and b/assets/message-chatgpt.png differ diff --git a/assets/send-button.png b/assets/send-button.png new file mode 100644 index 0000000..2cbed2e Binary files /dev/null and b/assets/send-button.png differ diff --git a/assets/stop-button.png b/assets/stop-button.png new file mode 100644 index 0000000..ee99a7e Binary files /dev/null and b/assets/stop-button.png differ diff --git a/docs/_data/sidebars/home_sidebar.yml b/docs/_data/sidebars/home_sidebar.yml index d009cdb..08a7bfa 100644 --- a/docs/_data/sidebars/home_sidebar.yml +++ b/docs/_data/sidebars/home_sidebar.yml @@ -13,7 +13,10 @@ entries: title: iPyTest url: core.html - output: web,pdf - title: Mock OpenAI API + title: Browser Automation + url: automation.html + - output: web,pdf + title: browser url: browser.html - output: web,pdf title: LLM diff --git a/docs/automation.html b/docs/automation.html new file mode 100644 index 0000000..1809bca --- /dev/null +++ b/docs/automation.html @@ -0,0 +1,810 @@ +--- + +title: Browser Automation + + +keywords: fastai +sidebar: home_sidebar + +summary: "Using Selenium for Browser Automation" +description: "Using Selenium for Browser Automation" +nb_path: "nbs\2_automation.ipynb" +--- + +
+ + {% raw %} + +
+
+ {% endraw %} + +
+
+
pip install undetected_chromedriver webdriver_manager
+
+
+
+
+ {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

init[source]

+
+

init(*arguments)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

quit[source]

+
+

quit()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
# def init(*arguments):
+#     chrome_options = webdriver.ChromeOptions()
+#     for argument in arguments:
+#         if isinstance(argument, str):
+#             chrome_options.add_argument(argument)
+#     global driver
+#     driver = webdriver.Chrome(options = chrome_options)
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
init('--user-data-dir=C:\\Users\\seii-saintway\\Downloads\\chrome-profile')
+
+
+
+
+
+ {% endraw %} + +
+
+
+

Logging

+
+
+
+
+
+
+

Using Text-based Search for Browser Automation

+
+
+
+ {% raw %} + +
+
+
+
+
+

ok[source]

+
+

ok()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

last[source]

+
+

last()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

new[source]

+
+

new(url)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

close[source]

+
+

close()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

find_elements[source]

+
+

find_elements(prompt, exactly=True)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

find_elements[source]

+
+

find_elements(prompt, exactly=True)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

find_element[source]

+
+

find_element(prompt, closest_prompt=None)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

click[source]

+
+

click(prompt, closest_prompt=None)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

input[source]

+
+

input(prompt, text, closest_prompt=None)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + +
+
+
+

Using Selenium for Static Inspection of Page Appearance

+
+
+
+ {% raw %} + +
+
+
+
+
+

image_hash[source]

+
+

image_hash(image_path)

+
+

Calculate the hash value of the image

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

screen_hash[source]

+
+

screen_hash()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

wait[source]

+
+

wait(timeout=1.7976931348623157e+308, stability_duration=1.0, check_interval=0.5)

+
+

Wait until the screenshot does not change.

+

Args: +timeout: how long to wait for stabilization (seconds) +stability_duration: duration for stabilization (seconds) +check_interval: check interval (seconds)

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + +
+
+
+

Using Airtest for Browser Automation

+
+
+
+
+
+
pip install airtest
+
+
+
+
+ {% raw %} + +
+
+
+
+
+

find_position[source]

+
+

find_position(image, timeout=1.0, threshold=0.9, interval=0.5, intervalfunc=None)

+
+

Search for image template in the screen until timeout

+

Args: +path: image file path to be found in screenshot +timeout: time interval how long to look for the image template +threshold: default is None +interval: sleep interval before next attempt to find the image template +intervalfunc: function that is executed after unsuccessful attempt to find the image template

+

Raises: +TargetNotFoundError: when image template is not found in screenshot

+

Returns: +TargetNotFoundError if image template not found, otherwise returns the position where the image template has +been found in screenshot

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

inject[source]

+
+

inject()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

get_mouse_position[source]

+
+

get_mouse_position()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

move_to_center[source]

+
+

move_to_center()

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

move_and_click[source]

+
+

move_and_click(x, y)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

exists[source]

+
+

exists(image)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

touch[source]

+
+

touch(image, text=None)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

fill[source]

+
+

fill(text)

+
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + +
+
+
+

Using Selenium for Printing to PNG and PDF

+
+
+
+
+
+
pip install markdown
+
+
+
+
+ {% raw %} + +
+
+
+
+
+

convert_md_with_ruby_to_html[source]

+
+

convert_md_with_ruby_to_html(md_file_path)

+
+

Convert Markdown files containing tags into HTML file

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

convert_html_with_ruby_to_png[source]

+
+

convert_html_with_ruby_to_png(html_file_path)

+
+

Use Selenium to print the HTML file as PNG

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

convert_md_with_ruby_to_png[source]

+
+

convert_md_with_ruby_to_png(md_file_path)

+
+

Convert the Markdown file containing tags into PNG

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

dialog_for_printing[source]

+
+

dialog_for_printing(timeout=inf)

+
+

Dialog for printing to PDF

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

convert_html_with_ruby_to_pdf[source]

+
+

convert_html_with_ruby_to_pdf(html_file_path)

+
+

Use Selenium to print the HTML file as PDF

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
+

convert_md_with_ruby_to_pdf[source]

+
+

convert_md_with_ruby_to_pdf(md_file_path)

+
+

Convert the Markdown file containing tags into PDF

+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+ {% endraw %} + +
+
+
+
+
+
+ {% raw %} + +
+
+
+
+
init('--lang=en')
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
convert_md_with_ruby_to_png('2024-11-21.md')
+
+
+
+
+
+ {% endraw %} + + {% raw %} + +
+
+
+
+
convert_md_with_ruby_to_pdf('2024-11-21.md')
+
+
+
+
+
+ {% endraw %} + +
diff --git a/ipymock/__init__.py b/ipymock/__init__.py index ae2b423..be0bee7 100644 --- a/ipymock/__init__.py +++ b/ipymock/__init__.py @@ -4,7 +4,6 @@ # Cell import py -import sys import traceback import _pytest.config diff --git a/ipymock/_nbdev.py b/ipymock/_nbdev.py index 8e7ceac..3818c48 100644 --- a/ipymock/_nbdev.py +++ b/ipymock/_nbdev.py @@ -5,6 +5,65 @@ index = {"get_test_funcs": "1_core.ipynb", "print_result": "1_core.ipynb", "do": "1_core.ipynb", + "driver": "2_automation.ipynb", + "device_pixel_ratio": "2_automation.ipynb", + "init": "2_browser.ipynb", + "quit": "2_automation.ipynb", + "logger": "2_automation.ipynb", + "handler": "2_automation.ipynb", + "ok": "2_automation.ipynb", + "last": "2_automation.ipynb", + "new": "2_automation.ipynb", + "close": "2_automation.ipynb", + "find_elements": "2_automation.ipynb", + "find_element": "2_automation.ipynb", + "click": "2_automation.ipynb", + "input": "2_automation.ipynb", + "get_html_hash": "2_automation.ipynb", + "wait": "2_automation.ipynb", + "screen_hash": "2_automation.ipynb", + "watch": "2_automation.ipynb", + "try_log_screen": "2_automation.ipynb", + "find_position": "2_automation.ipynb", + "inject": "2_automation.ipynb", + "get_mouse_position": "2_automation.ipynb", + "move_to_center": "2_automation.ipynb", + "move_and_click": "2_automation.ipynb", + "exists": "2_automation.ipynb", + "touch": "2_automation.ipynb", + "fill": "2_automation.ipynb", + "mouseMarkRD.id": "2_automation.ipynb", + "mouseMarkRD.style.position": "2_automation.ipynb", + "mouseMarkRD.style.width": "2_automation.ipynb", + "mouseMarkRD.style.height": "2_automation.ipynb", + "mouseMarkRD.style.backgroundColor": "2_automation.ipynb", + "mouseMarkRD.style.zIndex": "2_automation.ipynb", + "mouseMarkLU.id": "2_automation.ipynb", + "mouseMarkLU.style.position": "2_automation.ipynb", + "mouseMarkLU.style.width": "2_automation.ipynb", + "mouseMarkLU.style.height": "2_automation.ipynb", + "mouseMarkLU.style.backgroundColor": "2_automation.ipynb", + "mouseMarkLU.style.zIndex": "2_automation.ipynb", + "mouseMarkRU.id": "2_automation.ipynb", + "mouseMarkRU.style.position": "2_automation.ipynb", + "mouseMarkRU.style.width": "2_automation.ipynb", + "mouseMarkRU.style.height": "2_automation.ipynb", + "mouseMarkRU.style.backgroundColor": "2_automation.ipynb", + "mouseMarkRU.style.zIndex": "2_automation.ipynb", + "mouseMarkLD.id": "2_automation.ipynb", + "mouseMarkLD.style.position": "2_automation.ipynb", + "mouseMarkLD.style.width": "2_automation.ipynb", + "mouseMarkLD.style.height": "2_automation.ipynb", + "mouseMarkLD.style.backgroundColor": "2_automation.ipynb", + "mouseMarkLD.style.zIndex": "2_automation.ipynb", + "convert_md_with_ruby_to_html": "2_automation.ipynb", + "convert_md_content_to_html": "2_automation.ipynb", + "write_html_file": "2_automation.ipynb", + "convert_html_with_ruby_to_png": "2_automation.ipynb", + "convert_md_with_ruby_to_png": "2_automation.ipynb", + "dialog_for_printing": "2_automation.ipynb", + "convert_html_with_ruby_to_pdf": "2_automation.ipynb", + "convert_md_with_ruby_to_pdf": "2_automation.ipynb", "Common": "2_browser.ipynb", "common": "4_agi.ipynb", "get_conversations": "2_browser.ipynb", @@ -18,7 +77,6 @@ "clear_conversations": "2_browser.ipynb", "ChatGPTConverter": "2_browser.ipynb", "markdownize": "2_browser.ipynb", - "init": "2_browser.ipynb", "login": "2_browser.ipynb", "open_chat": "2_browser.ipynb", "remove_portal": "2_browser.ipynb", @@ -74,6 +132,7 @@ "DuckDuckGoSearchAPIWrapper": "4_tools_duckduckgo.ipynb"} modules = ["__init__.py", + "automation.py", "browser.py", "llm.py", "agi.py", diff --git a/ipymock/automation.py b/ipymock/automation.py new file mode 100644 index 0000000..868028e --- /dev/null +++ b/ipymock/automation.py @@ -0,0 +1,520 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/2_automation.ipynb (unless otherwise specified). + +__all__ = ['driver', 'device_pixel_ratio', 'init', 'quit', 'ok', 'last', 'new', 'close', 'find_elements', + 'find_elements', 'find_element', 'click', 'input', 'get_html_hash', 'wait', 'screen_hash', 'watch', + 'try_log_screen', 'find_position', 'inject', 'get_mouse_position', 'move_to_center', 'move_and_click', + 'exists', 'touch', 'fill', 'convert_md_with_ruby_to_html', 'convert_md_content_to_html', 'write_html_file', + 'convert_html_with_ruby_to_png', 'convert_md_with_ruby_to_png', 'dialog_for_printing', + 'convert_html_with_ruby_to_pdf', 'convert_md_with_ruby_to_pdf'] + +# Internal Cell +import os +import undetected_chromedriver +from webdriver_manager.chrome import ChromeDriverManager + +# Cell +driver = None +device_pixel_ratio = 1 + +# Cell +def init(*arguments): + chrome_options = undetected_chromedriver.ChromeOptions() + for argument in arguments: + if isinstance(argument, str): + chrome_options.add_argument(argument) + global driver + driver = undetected_chromedriver.Chrome( + options = chrome_options, + driver_executable_path = os.path.join( + os.path.dirname(ChromeDriverManager().install()), 'chromedriver.exe' if os.name == 'nt' else 'chromedriver' + ) + ) + global device_pixel_ratio + device_pixel_ratio = driver.execute_script('return window.devicePixelRatio;') + +# Cell +def quit(): + global driver + global device_pixel_ratio + driver.quit() + driver = None + device_pixel_ratio = 1 + +# Internal Cell +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter( + fmt = '[%(asctime)s][%(levelname)s]<%(name)s> %(message)s', + datefmt = '%H:%M:%S' +)) +logger.addHandler(handler) + +# Internal Cell +from selenium.common.exceptions import WebDriverException + +# Cell +def ok(): + if driver is None: + return False + try: + if driver.window_handles == []: + return False + except WebDriverException: + return False + return True + +# Cell +def last(): + if not ok(): + return + driver.switch_to.window(driver.window_handles[-1]) + +# Cell +def new(url): + last() + if not ok(): + init() + if 'data:,' not in driver.current_url and 'chrome://new-tab-page/' not in driver.current_url: + driver.switch_to.new_window('tab') + driver.get(url) + +# Cell +def close(): + global driver + global device_pixel_ratio + driver.close() + if not ok(): + driver = None + device_pixel_ratio = 1 + +# Internal Cell +from selenium.webdriver.common.by import By + +# Internal Cell +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions + +# Cell +def find_elements(prompt, exactly = True): + return [elem for elem in WebDriverWait(driver, 10).until( + expected_conditions.presence_of_all_elements_located((By.XPATH, f'//*[not(contains(text(), "\n")) and contains(., "{prompt}")]')) + ) if elem.is_displayed() and (elem.text == prompt or not exactly)] + +# Internal Cell +from selenium.common.exceptions import NoSuchElementException + +# Cell +def find_elements(prompt, exactly = True): + for scope in range(1, 5): + elements = driver.find_elements(By.XPATH, f'//*[.{"/*" * (scope - 1)} and not(.{"/*" * scope}) and contains(., "{prompt}")]') + elements = [elem for elem in elements if elem.is_displayed() and (elem.text == prompt or not exactly)] + logger.info(f'Search for {prompt} in scope {scope}: found {len(elements)} element(s)') + if elements: + return elements + raise NoSuchElementException + +# Cell +def find_element(prompt, closest_prompt = None): + if closest_prompt is None: + return find_elements(prompt)[-1] + if isinstance(closest_prompt, str): + closest_prompt = find_elements(closest_prompt, False)[-1] + closest_location = closest_prompt.location + return min( + find_elements(prompt), + key = lambda elem: (elem.location['x'] - closest_location['x']) ** 2 + (elem.location['y'] - closest_location['y']) ** 2 + ) + +# Internal Cell +from selenium.webdriver.common.action_chains import ActionChains + +# Cell +def click(prompt, closest_prompt = None): + if isinstance(prompt, str): + prompt = find_element(prompt, closest_prompt) + ActionChains(driver).move_to_element(prompt).click().perform() + return prompt + +# Cell +def input(prompt, text, closest_prompt = None): + prompt = click(prompt, closest_prompt) + ActionChains(driver).send_keys(text).perform() + return prompt + +# Internal Cell +import hashlib, time + +# Cell +def get_html_hash(xpath = '//body'): + """Get the hash of the element's outerHTML.""" + # driver is the Selenium WebDriver global instance. + elements = driver.find_elements(By.XPATH, xpath) + html = elements[-1].get_attribute('outerHTML') if elements else '' + return hashlib.md5(html.encode('utf-8')).hexdigest(), time.time() + +# Cell +def wait(timeout=float('inf'), stability_duration=1.0, check_interval=0.5, xpath='//body'): + """ + Wait until the HTML of the specified element does not change. + + Args: + timeout: Maximum wait time for stabilization (seconds). + stability_duration: Duration for stabilization (seconds). + check_interval: Interval to check for changes (seconds). + xpath: XPATH of the element to monitor for HTML changes. + """ + # Get the initial hash value + previous_hash, previous_time = get_html_hash(xpath) + + # Wait until the HTML does not change + start_time = time.time() + while True: + time.sleep(check_interval) + + # Get the current hash value + current_hash, current_time = get_html_hash(xpath) + + # Check if the hash value has stabilized + if current_hash == previous_hash: + if current_time - previous_time >= stability_duration: + logger.info('HTML content has stabilized.') + break + else: + # Update hash and time if the content changes + previous_hash, previous_time = current_hash, current_time + + # Check for timeout + if current_time - start_time >= timeout: + logger.info('Wait for HTML stabilization timed out.') + break + +# Cell +def screen_hash(): + """Calculate the hash value of the screenshot.""" + return hashlib.md5(driver.get_screenshot_as_base64().encode('utf-8')).hexdigest(), time.time() + +# Internal Cell +from sys import float_info + +# Cell +def watch(timeout = float_info.max, stability_duration = 1.0, check_interval = 0.5): + """ + Wait until the screenshot does not change. + + Args: + timeout: how long to wait for stabilization (seconds) + stability_duration: duration for stabilization (seconds) + check_interval: check interval (seconds) + """ + # Get the initial hash value + previous_hash, previous_time = screen_hash() + + # Wait until the image does not change + start_time = previous_time + while True: + time.sleep(check_interval) + + # Take another screenshot and calculate the new hash value + current_hash, current_time = screen_hash() + + # Check if the hash value has changed + if current_hash == previous_hash: + if current_time - previous_time >= stability_duration: + logger.info('Screenshot has stabilized.') + break + else: + if current_time - start_time >= timeout: + logger.warning('Wait for screenshot stabilization timeout.') + break + previous_hash, previous_time = current_hash, current_time + +# Cell +def try_log_screen(xpath = None): + screenshot_path = 'screen.png' + if isinstance(xpath, str): + driver.find_element(By.XPATH, xpath).screenshot(screenshot_path) + return + driver.save_screenshot(screenshot_path) + +# Internal Cell +from selenium.webdriver.common.by import By +from selenium.webdriver.common.action_chains import ActionChains + +import os, time +from airtest.aircv import get_resolution, imread +from airtest.core.api import Template +from airtest.core.error import TargetNotFoundError + +# Cell +def find_position(image, timeout=1.0, threshold=0.9, interval=0.5, intervalfunc=None): + """ + Search for image template in the screen until timeout + + Args: + image: image file path to be found in screenshot + timeout: time interval how long to look for the image template + threshold: default is None + interval: sleep interval before next attempt to find the image template + intervalfunc: function that is executed after unsuccessful attempt to find the image template + + Raises: + TargetNotFoundError: when image template is not found in screenshot + + Returns: + TargetNotFoundError if image template not found, otherwise returns the position where the image template has + been found in screenshot + """ + logger.info(f'Try to find {image}') + query = Template(image, rgb = True) + start_time = time.time() + while True: + LOG_DIR = '.' + file_path = os.path.join(LOG_DIR, 'screen.png') + driver.save_screenshot(file_path) + screen = imread(file_path) + query.resolution = get_resolution(screen) + # query.resolution = (1920, 1080) + if screen is None: + logger.warning('Screen is None: may be locked') + else: + if threshold: + query.threshold = threshold + match_pos = query.match_in(screen) + if match_pos: + # logger.info(f'match_pos == {match_pos}') + # try_log_screen(screen) + return match_pos[0] / device_pixel_ratio, match_pos[1] / device_pixel_ratio + + if intervalfunc is not None: + intervalfunc() + + # Raise an exception if timeout occurs, otherwise proceed to the next loop. + if (time.time() - start_time) > timeout: + # try_log_screen(screen) + raise TargetNotFoundError(f'Picture {query} not found in screen') + else: + # ActionChains(driver).move_by_offset(0, 0).perform() + time.sleep(interval) + +def inject(): + # Inject JavaScript code to get the mouse coordinates. + driver.execute_script(""" +var mouseMarkRD = document.createElement('div'); +mouseMarkRD.id = 'mouse_mark_rd'; +mouseMarkRD.style.position = 'absolute'; +mouseMarkRD.style.width = '8px'; +mouseMarkRD.style.height = '8px'; +mouseMarkRD.style.backgroundColor = 'red'; +mouseMarkRD.style.zIndex = '9999'; +document.body.appendChild(mouseMarkRD); + +var mouseMarkLU = document.createElement('div'); +mouseMarkLU.id = 'mouse_mark_lu'; +mouseMarkLU.style.position = 'absolute'; +mouseMarkLU.style.width = '8px'; +mouseMarkLU.style.height = '8px'; +mouseMarkLU.style.backgroundColor = 'red'; +mouseMarkLU.style.zIndex = '9999'; +document.body.appendChild(mouseMarkLU); + +var mouseMarkRU = document.createElement('div'); +mouseMarkRU.id = 'mouse_mark_ru'; +mouseMarkRU.style.position = 'absolute'; +mouseMarkRU.style.width = '8px'; +mouseMarkRU.style.height = '8px'; +mouseMarkRU.style.backgroundColor = 'red'; +mouseMarkRU.style.zIndex = '9999'; +document.body.appendChild(mouseMarkRU); + +var mouseMarkLD = document.createElement('div'); +mouseMarkLD.id = 'mouse_mark_ld'; +mouseMarkLD.style.position = 'absolute'; +mouseMarkLD.style.width = '8px'; +mouseMarkLD.style.height = '8px'; +mouseMarkLD.style.backgroundColor = 'red'; +mouseMarkLD.style.zIndex = '9999'; +document.body.appendChild(mouseMarkLD); + +document.addEventListener('mousemove', function(e) { + mouseMarkRD.style.left = e.pageX + 2 + 'px'; + mouseMarkRD.style.top = e.pageY + 2 + 'px'; + + mouseMarkLU.style.left = e.pageX - 10 + 'px'; + mouseMarkLU.style.top = e.pageY - 10 + 'px'; + + mouseMarkRU.style.left = e.pageX + 2 + 'px'; + mouseMarkRU.style.top = e.pageY - 10 + 'px'; + + mouseMarkLD.style.left = e.pageX - 10 + 'px'; + mouseMarkLD.style.top = e.pageY + 2 + 'px'; + + window.mouseX = e.clientX; + window.mouseY = e.clientY; +}); +""") + +def get_mouse_position(): + return driver.execute_script('return window.mouseX'), driver.execute_script('return window.mouseY') + +def move_to_center(): + body = driver.find_element(By.TAG_NAME, 'body') + ActionChains(driver).move_to_element(body).perform() + return body.rect['x'] + body.rect['width'] >> 1, body.rect['y'] + body.rect['height'] >> 1 + +def move_and_click(x, y): + # Retrieve the mouse coordinates. + center_x, center_y = move_to_center() + logger.debug(f'Get center position: {center_x}, {center_y}') + # Move to the specified coordinates (x, y) and click. + ActionChains(driver).move_by_offset(x - center_x, y - center_y).click().perform() + +def exists(image): + try: + find_position(image) + return True + except: + return False + +def touch(image, text = None): + try: + x, y = find_position(image) + move_and_click(x, y) + if isinstance(text, str): + fill(text) + except: + pass + +def fill(text): + ActionChains(driver).send_keys(text).perform() + +# Internal Cell +import markdown +import os, re + +# Cell +def convert_md_with_ruby_to_html(md_file_path): + ''' Convert Markdown file containing tags into HTML file ''' + # Get the filename without extension + base_name = os.path.splitext(md_file_path)[0] + html_file_path = f'{base_name}.html' + + # Read the Markdown file and convert it to HTML + with open(md_file_path, 'r', encoding='utf-8') as md_file: + md_content = md_file.read() + + # Create an HTML document while preserving the tags + html_document = convert_md_content_to_html(md_content) + write_html_file(html_file_path, html_document) + return html_file_path, html_document + +def convert_md_content_to_html(md_content): + ''' Convert Markdown content containing tags into HTML content ''' + html_content = markdown.markdown(md_content) + + # Replace the newline characters within

tags with
+ html_content = re.sub(r'

(.*?)

', lambda m: '

' + m.group(1).replace('\n', '
') + '

', html_content, flags=re.DOTALL) + + return f""" + + + + + + +{html_content} + + +""" + +def write_html_file(html_file_path, html_content): + ''' Write the HTML content to a file ''' + with open(html_file_path, 'w', encoding='utf-8') as html_file: + html_file.write(html_content) + +# Cell +def convert_html_with_ruby_to_png(html_file_path): + ''' Use Selenium to print the HTML file as PNG ''' + # Get the filename without extension + base_name = os.path.splitext(html_file_path)[0] + png_file_path = f'{base_name}.png' + + # Set up ChromeDriver + global driver + if driver is None: + init('--headless', '--disable-gpu') + + # Open the local HTML file + new('file://' + os.path.abspath(html_file_path)) + # Wait for the page to fully load + wait(1.0) + + # Get the current window size + current_window_size = driver.get_window_size() + # Set the window size to the resolution of iPhone 16 Pro Max + driver.set_window_size(642, 1389) + # Capture the page and save as PNG + driver.save_screenshot(png_file_path) + # Reset the window size + driver.set_window_size(current_window_size['width'], current_window_size['height']) + # Wait for the page to fully load + wait(1.0) + + close() + return png_file_path + +def convert_md_with_ruby_to_png(md_file_path): + ''' Convert the Markdown file containing tags into PNG ''' + html_file_path, _ = convert_md_with_ruby_to_html(md_file_path) + return convert_html_with_ruby_to_png(html_file_path) + +# Cell +def dialog_for_printing(timeout = float('inf')): + ''' Dialog for printing to PDF ''' + # Print to PDF + driver.set_script_timeout(60 * 60 * 24) + driver.execute_script('window.print();') + # Wait until the element becomes clickable + WebDriverWait(driver, timeout).until(expected_conditions.element_to_be_clickable((By.TAG_NAME, 'body'))) + +# Internal Cell +import base64 +from selenium.webdriver.common.print_page_options import PrintOptions + +# Cell +def convert_html_with_ruby_to_pdf(html_file_path): + ''' Use Selenium to print the HTML file as PDF ''' + # Get the filename without extension + base_name = os.path.splitext(html_file_path)[0] + pdf_file_path = f'{base_name}.pdf' + + # Set up ChromeDriver + global driver + if driver is None: + init('--disable-gpu', f'--print-to-pdf="{os.path.abspath(pdf_file_path)}"') + + # Open the local HTML file + new('file://' + os.path.abspath(html_file_path)) + # Wait for the page to fully load + wait(1.0) + + # Write the decoded data to a PDF file + print_options = PrintOptions() + print_options.scale = 1.3 + with open(pdf_file_path, 'wb') as pdf_file: + pdf_file.write(base64.b64decode(driver.print_page(print_options))) + + close() + return pdf_file_path + +def convert_md_with_ruby_to_pdf(md_file_path): + ''' Convert the Markdown file containing tags into PDF ''' + html_file_path, _ = convert_md_with_ruby_to_html(md_file_path) + return convert_html_with_ruby_to_pdf(html_file_path) \ No newline at end of file diff --git a/ipymock/browser.py b/ipymock/browser.py index b33a627..6962359 100644 --- a/ipymock/browser.py +++ b/ipymock/browser.py @@ -244,8 +244,6 @@ def clear_conversations(): from selenium.webdriver.support import expected_conditions from selenium.webdriver.support.ui import WebDriverWait -import undetected_chromedriver as uc - # Internal Cell from markdownify import MarkdownConverter @@ -259,19 +257,25 @@ def convert_pre(self, node, text, convert_as_inline): node_code = node.find('code') return ( f"```{' '.join([c[len('language-'):] for c in node_code.get('class') if c.startswith('language-')])}\n" - f'{node_code.text.strip()}\n' + f'{node_code.text.rstrip()}\n' '```\n' ) markdownize = ChatGPTConverter().convert +# Internal Cell +from collections.abc import Iterable +from .automation import init as init_browser + # Cell -def init(chrome_args = None): - options = uc.ChromeOptions() - if isinstance(chrome_args, list): - for arg in chrome_args: - options.add_argument(arg) - common.driver = uc.Chrome(options = options) +def init(chrome_args = set()): + if isinstance(chrome_args, Iterable): + chrome_args = set(chrome_args) + chrome_args.add('--lang=en') + chrome_args.add('--force-dark-mode') + init_browser(*chrome_args) + from .automation import driver + common.driver = driver login() open_chat(common.conversation_id) @@ -279,112 +283,148 @@ def init(chrome_args = None): global start_conversation start_conversation = ask -def login(): - common.driver.get('https://chat.openai.com/auth/login') - - WebDriverWait(common.driver, 5).until( - expected_conditions.presence_of_element_located((By.XPATH, '//*[text()="Log in"]')) - ) - - common.driver.execute_script(''' - document.evaluate( - '//*[text()="Log in"]', - document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null - ).snapshotItem(0).dispatchEvent( - new MouseEvent('click', { - view: window, - bubbles: true, - cancelable: true - }) - ); - ''') - - WebDriverWait(common.driver, 5).until( - expected_conditions.presence_of_element_located((By.XPATH, '//button[@data-provider="google"]')) - ) - - common.driver.execute_script(''' - document.evaluate( - '//button[@data-provider="google"]', - document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null - ).snapshotItem(0).dispatchEvent( - new MouseEvent('click', { - view: window, - bubbles: true, - cancelable: true - }) - ); - ''') - - WebDriverWait(common.driver, 5).until( - expected_conditions.presence_of_element_located((By.XPATH, '//input[@type="email"]')) - ) +# Internal Cell +from .automation import new, wait, click, input, fill - common.driver.execute_script(f''' - const google_email_input = document.evaluate('//input[@type="email"]', document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotItem(0); - google_email_input.value = '{common.config['email']}'; - google_email_input.dispatchEvent( - new Event('input', {{ - view: window, - bubbles: true, - cancelable: true - }}) - ); - ''') - - WebDriverWait(common.driver, 5).until( - expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="identifierNext"]')) - ) +# Cell +def login(): + new('https://chatgpt.com/auth/login') - common.driver.execute_script(''' - document.evaluate( - '//*[@id="identifierNext"]', - document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null - ).snapshotItem(0).dispatchEvent( - new MouseEvent('click', { - view: window, - bubbles: true, - cancelable: true - }) - ); - ''') - - WebDriverWait(common.driver, 10).until( - expected_conditions.element_to_be_clickable((By.XPATH, '//input[@type="password"]')) - ).click() - - ActionChains(common.driver).send_keys(common.config['password']).send_keys(Keys.ENTER).perform() + # WebDriverWait(common.driver, 5).until( + # expected_conditions.presence_of_element_located((By.XPATH, '//*[text()="Log in"]')) + # ) + wait(5.0) + + # common.driver.execute_script(''' + # document.evaluate( + # '//*[text()="Log in"]', + # document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null + # ).snapshotItem(0).dispatchEvent( + # new MouseEvent('click', { + # view: window, + # bubbles: true, + # cancelable: true + # }) + # ); + # ''') + click('Log in') + + # WebDriverWait(common.driver, 5).until( + # expected_conditions.presence_of_element_located((By.XPATH, '//button[@data-provider="google"]')) + # ) + wait(5.0) + + # common.driver.execute_script(''' + # document.evaluate( + # '//button[@data-provider="google"]', + # document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null + # ).snapshotItem(0).dispatchEvent( + # new MouseEvent('click', { + # view: window, + # bubbles: true, + # cancelable: true + # }) + # ); + # ''') + click('Continue with Google') + + # WebDriverWait(common.driver, 5).until( + # expected_conditions.presence_of_element_located((By.XPATH, '//input[@type="email"]')) + # ) + wait(5.0) + + # common.driver.execute_script(f''' + # const google_email_input = document.evaluate('//input[@type="email"]', document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotItem(0); + # google_email_input.value = '{common.config['email']}'; + # google_email_input.dispatchEvent( + # new Event('input', {{ + # view: window, + # bubbles: true, + # cancelable: true + # }}) + # ); + # ''') + input('Email or phone', common.config['email']) + + # WebDriverWait(common.driver, 5).until( + # expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="identifierNext"]')) + # ) + wait(5.0) + + # common.driver.execute_script(''' + # document.evaluate( + # '//*[@id="identifierNext"]', + # document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null + # ).snapshotItem(0).dispatchEvent( + # new MouseEvent('click', { + # view: window, + # bubbles: true, + # cancelable: true + # }) + # ); + # ''') + click('Next') + + # WebDriverWait(common.driver, 10).until( + # expected_conditions.element_to_be_clickable((By.XPATH, '//input[@type="password"]')) + # ).click() + wait(10.0) + + # ActionChains(common.driver).send_keys(common.config['password']).send_keys(Keys.ENTER).perform() + fill(common.config['password']) + fill(Keys.ENTER) + wait(stability_duration = 5.0) + + common.driver.maximize_window() + wait(1.0) remove_portal() +# Internal Cell +from selenium.common.exceptions import NoSuchElementException + +# Cell def open_chat(conversation_id = ''): + from .automation import driver + common.driver = driver if conversation_id == '': - common.driver.get('https://chat.openai.com') + common.driver.get('https://chatgpt.com/') else: - common.driver.get(f'https://chat.openai.com/c/{conversation_id}') + common.driver.get(f'https://chatgpt.com/c/{conversation_id}') if common.conversation_id != conversation_id: common.conversation_id = conversation_id common.parent_message_id = '' - WebDriverWait(common.driver, 30).until( - lambda driver: driver.execute_script('return document.readyState') == 'complete' - ) + # WebDriverWait(common.driver, 30).until( + # lambda driver: driver.execute_script('return document.readyState') == 'complete' + # ) + wait(5.0) remove_portal() def remove_portal(): while True: + # try: + # WebDriverWait(common.driver, 5).until( + # expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()="Next"]')) + # ).click() + # except TimeoutException: + # break try: - WebDriverWait(common.driver, 5).until( - expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()="Next"]')) - ).click() - except TimeoutException: + click('Next') + wait(5.0) + except NoSuchElementException: break + # try: + # WebDriverWait(common.driver, 5).until( + # expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()="Done"]')) + # ).click() + # except TimeoutException: + # pass try: - WebDriverWait(common.driver, 5).until( - expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()="Done"]')) - ).click() - except TimeoutException: + click('Done') + wait(5.0) + except NoSuchElementException: pass # Internal Cell @@ -392,37 +432,60 @@ def remove_portal(): chatgpt_disabled_button = (By.XPATH, '//textarea/following-sibling::button[@disabled]') chatgpt_enabled_button = (By.XPATH, '//textarea/following-sibling::button[not(@disabled)]') chatgpt_streaming = (By.CLASS_NAME, 'result-streaming') -chatgpt_response = (By.XPATH, '//div[starts-with(@class, "flex flex-grow flex-col gap-3")]') +chatgpt_response = (By.XPATH, '//div[starts-with(@class, "markdown prose w-full break-words")]') chatgpt_red_500 = (By.XPATH, '//div[contains(@class, "border-red-500 bg-red-500/10")]') -chatgpt_big_response = (By.XPATH, '//div[@class="flex-1 overflow-hidden"]//div[p]') -chatgpt_small_response = (By.XPATH, '//div[starts-with(@class, "markdown prose w-full break-words")]') +chatgpt_big_response = (By.XPATH, '//div[@class="flex-1 overflow-hidden"]//div[p or pre]') +chatgpt_small_response = (By.XPATH, '//div[@class="flex-1 overflow-hidden"]//code') + +# Internal Cell +from typing import Generator + +# Internal Cell +from .automation import exists, touch # Cell def request(prompt: str) -> None: - try: - textbox = WebDriverWait(common.driver, 5).until( - expected_conditions.element_to_be_clickable(chatgpt_textbox) - ) - except TimeoutException: + # try: + # textbox = WebDriverWait(common.driver, 5).until( + # expected_conditions.element_to_be_clickable(chatgpt_textbox) + # ) + # except TimeoutException: + # open_chat(common.conversation_id) + # textbox = WebDriverWait(common.driver, 5).until( + # expected_conditions.element_to_be_clickable(chatgpt_textbox) + # ) + # textbox.click() + click('ChatGPT can make mistakes. Check important info.') + textbox = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../assets/message-chatgpt.png')) + textbox = 'assets/message-chatgpt.png' + if not exists(textbox): open_chat(common.conversation_id) - textbox = WebDriverWait(common.driver, 5).until( - expected_conditions.element_to_be_clickable(chatgpt_textbox) - ) - textbox.click() + click('ChatGPT can make mistakes. Check important info.') + touch(textbox) # textbox.send_keys(prompt.strip()) - common.driver.execute_script(''' - var element = arguments[0], txt = arguments[1]; - element.value += txt; - element.dispatchEvent(new Event("change")); - ''', - textbox, - prompt.strip(), - ) + # common.driver.execute_script(''' + # var element = arguments[0], txt = arguments[1]; + # element.value += txt; + # element.dispatchEvent(new Event("change")); + # ''', + # textbox, + # prompt.strip(), + # ) + for line in prompt.strip().split('\n'): + fill(line) + ActionChains(common.driver).key_down(Keys.SHIFT).send_keys(Keys.ENTER).key_up(Keys.SHIFT).perform() # WebDriverWait(common.driver, 3).until_not( # expected_conditions.presence_of_element_located(chatgpt_disabled_button) # ) - textbox.send_keys('\n') - textbox.send_keys(Keys.ENTER) + click('ChatGPT can make mistakes. Check important info.') + wait(1.0) + # textbox.send_keys('\n') + # textbox.send_keys(Keys.ENTER) + send_button = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../assets/send-button.png')) + send_button = 'assets/send-button.png' + touch(send_button) + click('ChatGPT can make mistakes. Check important info.') + wait(1.0) # try: # WebDriverWait(common.driver, 5).until( # expected_conditions.element_to_be_clickable(chatgpt_enabled_button) @@ -431,31 +494,37 @@ def request(prompt: str) -> None: # pass def get_last_response(): + responses = common.driver.find_elements(*chatgpt_small_response) + if responses != []: + return responses[-1] responses = common.driver.find_elements(*chatgpt_big_response) if responses != []: return responses[-1] - responses = common.driver.find_elements(*chatgpt_small_response) + responses = common.driver.find_elements(*chatgpt_response) if responses != []: return responses[-1] -def get_response() -> str: +def get_response() -> Generator[str, None, None]: try: result_streaming = WebDriverWait(common.driver, 30).until( expected_conditions.presence_of_element_located(chatgpt_streaming) ) except TimeoutException: - response = common.driver.find_elements(*chatgpt_response)[-1] - is_error = common.driver.find_elements(*chatgpt_red_500) != [] + response = get_last_response() + error = common.driver.find_elements(*chatgpt_red_500) != [] sys.stderr.write( 'TimeoutException: having waited 30 seconds for result-streaming\n' f'response.text = {response.text}\n' - f'is_error = {is_error}\n' + f'error = {error}\n' ) - if not is_error: + if not error: yield markdownize(response.get_attribute('innerHTML')) - return + result_streaming = common.driver.find_elements(*chatgpt_streaming) while result_streaming: response = get_last_response() + if response is None: + result_streaming = common.driver.find_elements(*chatgpt_streaming) + continue try: if 'text-red' in response.get_attribute('class'): sys.stderr.write(f'Error Responding: response.text = {response.text}\n') diff --git a/nbs/1_core.ipynb b/nbs/1_core.ipynb index 1a51949..695bbd9 100644 --- a/nbs/1_core.ipynb +++ b/nbs/1_core.ipynb @@ -26,7 +26,6 @@ "source": [ "# export\n", "import py\n", - "import sys\n", "import traceback\n", "\n", "import _pytest.config\n", diff --git a/nbs/2_automation.ipynb b/nbs/2_automation.ipynb new file mode 100644 index 0000000..0e9541f --- /dev/null +++ b/nbs/2_automation.ipynb @@ -0,0 +1,916 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# default_exp automation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Browser Automation\n", + "\n", + "> Using Selenium for Browser Automation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "pip install undetected_chromedriver webdriver_manager\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "import os\n", + "import undetected_chromedriver\n", + "from webdriver_manager.chrome import ChromeDriverManager" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "driver = None\n", + "device_pixel_ratio = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def init(*arguments):\n", + " chrome_options = undetected_chromedriver.ChromeOptions()\n", + " for argument in arguments:\n", + " if isinstance(argument, str):\n", + " chrome_options.add_argument(argument)\n", + " global driver\n", + " driver = undetected_chromedriver.Chrome(\n", + " options = chrome_options,\n", + " driver_executable_path = os.path.join(\n", + " os.path.dirname(ChromeDriverManager().install()), 'chromedriver.exe' if os.name == 'nt' else 'chromedriver'\n", + " )\n", + " )\n", + " global device_pixel_ratio\n", + " device_pixel_ratio = driver.execute_script('return window.devicePixelRatio;')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def quit():\n", + " global driver\n", + " global device_pixel_ratio\n", + " driver.quit()\n", + " driver = None\n", + " device_pixel_ratio = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# from selenium import webdriver\n", + "\n", + "# def init(*arguments):\n", + "# chrome_options = webdriver.ChromeOptions()\n", + "# for argument in arguments:\n", + "# if isinstance(argument, str):\n", + "# chrome_options.add_argument(argument)\n", + "# global driver\n", + "# driver = webdriver.Chrome(options = chrome_options)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "init('--user-data-dir=C:\\\\Users\\\\seii-saintway\\\\Downloads\\\\chrome-profile')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "Logging" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "import logging\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "logger.setLevel(logging.DEBUG)\n", + "\n", + "handler = logging.StreamHandler()\n", + "handler.setFormatter(logging.Formatter(\n", + " fmt = '[%(asctime)s][%(levelname)s]<%(name)s> %(message)s',\n", + " datefmt = '%H:%M:%S'\n", + "))\n", + "logger.addHandler(handler)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "Using Text-based Search for Browser Automation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.common.exceptions import WebDriverException" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def ok():\n", + " if driver is None:\n", + " return False\n", + " try:\n", + " if driver.window_handles == []:\n", + " return False\n", + " except WebDriverException:\n", + " return False\n", + " return True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def last():\n", + " if not ok():\n", + " return\n", + " driver.switch_to.window(driver.window_handles[-1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def new(url):\n", + " last()\n", + " if not ok():\n", + " init()\n", + " if 'data:,' not in driver.current_url and 'chrome://new-tab-page/' not in driver.current_url:\n", + " driver.switch_to.new_window('tab')\n", + " driver.get(url)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def close():\n", + " global driver\n", + " global device_pixel_ratio\n", + " driver.close()\n", + " if not ok():\n", + " driver = None\n", + " device_pixel_ratio = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.webdriver.common.by import By" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.webdriver.support.ui import WebDriverWait\n", + "from selenium.webdriver.support import expected_conditions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def find_elements(prompt, exactly = True):\n", + " return [elem for elem in WebDriverWait(driver, 10).until(\n", + " expected_conditions.presence_of_all_elements_located((By.XPATH, f'//*[not(contains(text(), \"\\n\")) and contains(., \"{prompt}\")]'))\n", + " ) if elem.is_displayed() and (elem.text == prompt or not exactly)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.common.exceptions import NoSuchElementException" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def find_elements(prompt, exactly = True):\n", + " for scope in range(1, 5):\n", + " elements = driver.find_elements(By.XPATH, f'//*[.{\"/*\" * (scope - 1)} and not(.{\"/*\" * scope}) and contains(., \"{prompt}\")]')\n", + " elements = [elem for elem in elements if elem.is_displayed() and (elem.text == prompt or not exactly)]\n", + " logger.info(f'Search for {prompt} in scope {scope}: found {len(elements)} element(s)')\n", + " if elements:\n", + " return elements\n", + " raise NoSuchElementException" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def find_element(prompt, closest_prompt = None):\n", + " if closest_prompt is None:\n", + " return find_elements(prompt)[-1]\n", + " if isinstance(closest_prompt, str):\n", + " closest_prompt = find_elements(closest_prompt, False)[-1]\n", + " closest_location = closest_prompt.location\n", + " return min(\n", + " find_elements(prompt),\n", + " key = lambda elem: (elem.location['x'] - closest_location['x']) ** 2 + (elem.location['y'] - closest_location['y']) ** 2\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.webdriver.common.action_chains import ActionChains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def click(prompt, closest_prompt = None):\n", + " if isinstance(prompt, str):\n", + " prompt = find_element(prompt, closest_prompt)\n", + " ActionChains(driver).move_to_element(prompt).click().perform()\n", + " return prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def input(prompt, text, closest_prompt = None):\n", + " prompt = click(prompt, closest_prompt)\n", + " ActionChains(driver).send_keys(text).perform()\n", + " return prompt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "Using Selenium for Static Inspection of Page Appearance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "import hashlib, time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def get_html_hash(xpath = '//body'):\n", + " \"\"\"Get the hash of the element's outerHTML.\"\"\"\n", + " # driver is the Selenium WebDriver global instance.\n", + " elements = driver.find_elements(By.XPATH, xpath)\n", + " html = elements[-1].get_attribute('outerHTML') if elements else ''\n", + " return hashlib.md5(html.encode('utf-8')).hexdigest(), time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def wait(timeout=float('inf'), stability_duration=1.0, check_interval=0.5, xpath='//body'):\n", + " \"\"\"\n", + " Wait until the HTML of the specified element does not change.\n", + "\n", + " Args:\n", + " timeout: Maximum wait time for stabilization (seconds).\n", + " stability_duration: Duration for stabilization (seconds).\n", + " check_interval: Interval to check for changes (seconds).\n", + " xpath: XPATH of the element to monitor for HTML changes.\n", + " \"\"\"\n", + " # Get the initial hash value\n", + " previous_hash, previous_time = get_html_hash(xpath)\n", + "\n", + " # Wait until the HTML does not change\n", + " start_time = time.time()\n", + " while True:\n", + " time.sleep(check_interval)\n", + "\n", + " # Get the current hash value\n", + " current_hash, current_time = get_html_hash(xpath)\n", + "\n", + " # Check if the hash value has stabilized\n", + " if current_hash == previous_hash:\n", + " if current_time - previous_time >= stability_duration:\n", + " logger.info('HTML content has stabilized.')\n", + " break\n", + " else:\n", + " # Update hash and time if the content changes\n", + " previous_hash, previous_time = current_hash, current_time\n", + "\n", + " # Check for timeout\n", + " if current_time - start_time >= timeout:\n", + " logger.info('Wait for HTML stabilization timed out.')\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def screen_hash():\n", + " \"\"\"Calculate the hash value of the screenshot.\"\"\"\n", + " return hashlib.md5(driver.get_screenshot_as_base64().encode('utf-8')).hexdigest(), time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from sys import float_info" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def watch(timeout = float_info.max, stability_duration = 1.0, check_interval = 0.5):\n", + " \"\"\"\n", + " Wait until the screenshot does not change.\n", + "\n", + " Args:\n", + " timeout: how long to wait for stabilization (seconds)\n", + " stability_duration: duration for stabilization (seconds)\n", + " check_interval: check interval (seconds)\n", + " \"\"\"\n", + " # Get the initial hash value\n", + " previous_hash, previous_time = screen_hash()\n", + "\n", + " # Wait until the image does not change\n", + " start_time = previous_time\n", + " while True:\n", + " time.sleep(check_interval)\n", + "\n", + " # Take another screenshot and calculate the new hash value\n", + " current_hash, current_time = screen_hash()\n", + "\n", + " # Check if the hash value has changed\n", + " if current_hash == previous_hash:\n", + " if current_time - previous_time >= stability_duration:\n", + " logger.info('Screenshot has stabilized.')\n", + " break\n", + " else:\n", + " if current_time - start_time >= timeout:\n", + " logger.warning('Wait for screenshot stabilization timeout.')\n", + " break\n", + " previous_hash, previous_time = current_hash, current_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "Using Airtest for Browser Automation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "pip install airtest\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def try_log_screen(xpath = None):\n", + " screenshot_path = 'screen.png'\n", + " if isinstance(xpath, str):\n", + " driver.find_element(By.XPATH, xpath).screenshot(screenshot_path)\n", + " return\n", + " driver.save_screenshot(screenshot_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.webdriver.common.by import By\n", + "from selenium.webdriver.common.action_chains import ActionChains\n", + "\n", + "import os, time\n", + "from airtest.aircv import get_resolution, imread\n", + "from airtest.core.api import Template\n", + "from airtest.core.error import TargetNotFoundError" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def find_position(image, timeout=1.0, threshold=0.9, interval=0.5, intervalfunc=None):\n", + " \"\"\"\n", + " Search for image template in the screen until timeout\n", + "\n", + " Args:\n", + " image: image file path to be found in screenshot\n", + " timeout: time interval how long to look for the image template\n", + " threshold: default is None\n", + " interval: sleep interval before next attempt to find the image template\n", + " intervalfunc: function that is executed after unsuccessful attempt to find the image template\n", + "\n", + " Raises:\n", + " TargetNotFoundError: when image template is not found in screenshot\n", + "\n", + " Returns:\n", + " TargetNotFoundError if image template not found, otherwise returns the position where the image template has\n", + " been found in screenshot\n", + " \"\"\"\n", + " logger.info(f'Try to find {image}')\n", + " query = Template(image, rgb = True)\n", + " start_time = time.time()\n", + " while True:\n", + " LOG_DIR = '.'\n", + " file_path = os.path.join(LOG_DIR, 'screen.png')\n", + " driver.save_screenshot(file_path)\n", + " screen = imread(file_path)\n", + " query.resolution = get_resolution(screen)\n", + " # query.resolution = (1920, 1080)\n", + " if screen is None:\n", + " logger.warning('Screen is None: may be locked')\n", + " else:\n", + " if threshold:\n", + " query.threshold = threshold\n", + " match_pos = query.match_in(screen)\n", + " if match_pos:\n", + " # logger.info(f'match_pos == {match_pos}')\n", + " # try_log_screen(screen)\n", + " return match_pos[0] / device_pixel_ratio, match_pos[1] / device_pixel_ratio\n", + "\n", + " if intervalfunc is not None:\n", + " intervalfunc()\n", + "\n", + " # Raise an exception if timeout occurs, otherwise proceed to the next loop.\n", + " if (time.time() - start_time) > timeout:\n", + " # try_log_screen(screen)\n", + " raise TargetNotFoundError(f'Picture {query} not found in screen')\n", + " else:\n", + " # ActionChains(driver).move_by_offset(0, 0).perform()\n", + " time.sleep(interval)\n", + "\n", + "def inject():\n", + " # Inject JavaScript code to get the mouse coordinates.\n", + " driver.execute_script(\"\"\"\n", + "var mouseMarkRD = document.createElement('div');\n", + "mouseMarkRD.id = 'mouse_mark_rd';\n", + "mouseMarkRD.style.position = 'absolute';\n", + "mouseMarkRD.style.width = '8px';\n", + "mouseMarkRD.style.height = '8px';\n", + "mouseMarkRD.style.backgroundColor = 'red';\n", + "mouseMarkRD.style.zIndex = '9999';\n", + "document.body.appendChild(mouseMarkRD);\n", + "\n", + "var mouseMarkLU = document.createElement('div');\n", + "mouseMarkLU.id = 'mouse_mark_lu';\n", + "mouseMarkLU.style.position = 'absolute';\n", + "mouseMarkLU.style.width = '8px';\n", + "mouseMarkLU.style.height = '8px';\n", + "mouseMarkLU.style.backgroundColor = 'red';\n", + "mouseMarkLU.style.zIndex = '9999';\n", + "document.body.appendChild(mouseMarkLU);\n", + "\n", + "var mouseMarkRU = document.createElement('div');\n", + "mouseMarkRU.id = 'mouse_mark_ru';\n", + "mouseMarkRU.style.position = 'absolute';\n", + "mouseMarkRU.style.width = '8px';\n", + "mouseMarkRU.style.height = '8px';\n", + "mouseMarkRU.style.backgroundColor = 'red';\n", + "mouseMarkRU.style.zIndex = '9999';\n", + "document.body.appendChild(mouseMarkRU);\n", + "\n", + "var mouseMarkLD = document.createElement('div');\n", + "mouseMarkLD.id = 'mouse_mark_ld';\n", + "mouseMarkLD.style.position = 'absolute';\n", + "mouseMarkLD.style.width = '8px';\n", + "mouseMarkLD.style.height = '8px';\n", + "mouseMarkLD.style.backgroundColor = 'red';\n", + "mouseMarkLD.style.zIndex = '9999';\n", + "document.body.appendChild(mouseMarkLD);\n", + "\n", + "document.addEventListener('mousemove', function(e) {\n", + " mouseMarkRD.style.left = e.pageX + 2 + 'px';\n", + " mouseMarkRD.style.top = e.pageY + 2 + 'px';\n", + "\n", + " mouseMarkLU.style.left = e.pageX - 10 + 'px';\n", + " mouseMarkLU.style.top = e.pageY - 10 + 'px';\n", + "\n", + " mouseMarkRU.style.left = e.pageX + 2 + 'px';\n", + " mouseMarkRU.style.top = e.pageY - 10 + 'px';\n", + "\n", + " mouseMarkLD.style.left = e.pageX - 10 + 'px';\n", + " mouseMarkLD.style.top = e.pageY + 2 + 'px';\n", + "\n", + " window.mouseX = e.clientX;\n", + " window.mouseY = e.clientY;\n", + "});\n", + "\"\"\")\n", + "\n", + "def get_mouse_position():\n", + " return driver.execute_script('return window.mouseX'), driver.execute_script('return window.mouseY')\n", + "\n", + "def move_to_center():\n", + " body = driver.find_element(By.TAG_NAME, 'body')\n", + " ActionChains(driver).move_to_element(body).perform()\n", + " return body.rect['x'] + body.rect['width'] >> 1, body.rect['y'] + body.rect['height'] >> 1\n", + "\n", + "def move_and_click(x, y):\n", + " # Retrieve the mouse coordinates.\n", + " center_x, center_y = move_to_center()\n", + " logger.debug(f'Get center position: {center_x}, {center_y}')\n", + " # Move to the specified coordinates (x, y) and click.\n", + " ActionChains(driver).move_by_offset(x - center_x, y - center_y).click().perform()\n", + "\n", + "def exists(image):\n", + " try:\n", + " find_position(image)\n", + " return True\n", + " except:\n", + " return False\n", + "\n", + "def touch(image, text = None):\n", + " try:\n", + " x, y = find_position(image)\n", + " move_and_click(x, y)\n", + " if isinstance(text, str):\n", + " fill(text)\n", + " except:\n", + " pass\n", + "\n", + "def fill(text):\n", + " ActionChains(driver).send_keys(text).perform()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "Using Selenium for Printing to PNG and PDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "pip install markdown\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "import markdown\n", + "import os, re" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def convert_md_with_ruby_to_html(md_file_path):\n", + " ''' Convert Markdown file containing tags into HTML file '''\n", + " # Get the filename without extension\n", + " base_name = os.path.splitext(md_file_path)[0]\n", + " html_file_path = f'{base_name}.html'\n", + "\n", + " # Read the Markdown file and convert it to HTML\n", + " with open(md_file_path, 'r', encoding='utf-8') as md_file:\n", + " md_content = md_file.read()\n", + "\n", + " # Create an HTML document while preserving the tags\n", + " html_document = convert_md_content_to_html(md_content)\n", + " write_html_file(html_file_path, html_document)\n", + " return html_file_path, html_document\n", + "\n", + "def convert_md_content_to_html(md_content):\n", + " ''' Convert Markdown content containing tags into HTML content '''\n", + " html_content = markdown.markdown(md_content)\n", + "\n", + " # Replace the newline characters within

tags with
\n", + " html_content = re.sub(r'

(.*?)

', lambda m: '

' + m.group(1).replace('\\n', '
') + '

', html_content, flags=re.DOTALL)\n", + "\n", + " return f\"\"\"\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "{html_content}\n", + " \n", + "\n", + "\"\"\"\n", + "\n", + "def write_html_file(html_file_path, html_content):\n", + " ''' Write the HTML content to a file '''\n", + " with open(html_file_path, 'w', encoding='utf-8') as html_file:\n", + " html_file.write(html_content)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def convert_html_with_ruby_to_png(html_file_path):\n", + " ''' Use Selenium to print the HTML file as PNG '''\n", + " # Get the filename without extension\n", + " base_name = os.path.splitext(html_file_path)[0]\n", + " png_file_path = f'{base_name}.png'\n", + "\n", + " # Set up ChromeDriver\n", + " global driver\n", + " if driver is None:\n", + " init('--headless', '--disable-gpu')\n", + "\n", + " # Open the local HTML file\n", + " new('file://' + os.path.abspath(html_file_path))\n", + " # Wait for the page to fully load\n", + " wait(1.0)\n", + "\n", + " # Get the current window size\n", + " current_window_size = driver.get_window_size()\n", + " # Set the window size to the resolution of iPhone 16 Pro Max\n", + " driver.set_window_size(642, 1389)\n", + " # Capture the page and save as PNG\n", + " driver.save_screenshot(png_file_path)\n", + " # Reset the window size\n", + " driver.set_window_size(current_window_size['width'], current_window_size['height'])\n", + " # Wait for the page to fully load\n", + " wait(1.0)\n", + "\n", + " close()\n", + " return png_file_path\n", + "\n", + "def convert_md_with_ruby_to_png(md_file_path):\n", + " ''' Convert the Markdown file containing tags into PNG '''\n", + " html_file_path, _ = convert_md_with_ruby_to_html(md_file_path)\n", + " return convert_html_with_ruby_to_png(html_file_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def dialog_for_printing(timeout = float('inf')):\n", + " ''' Dialog for printing to PDF '''\n", + " # Print to PDF\n", + " driver.set_script_timeout(60 * 60 * 24)\n", + " driver.execute_script('window.print();')\n", + " # Wait until the element becomes clickable\n", + " WebDriverWait(driver, timeout).until(expected_conditions.element_to_be_clickable((By.TAG_NAME, 'body')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "import base64\n", + "from selenium.webdriver.common.print_page_options import PrintOptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", + "def convert_html_with_ruby_to_pdf(html_file_path):\n", + " ''' Use Selenium to print the HTML file as PDF '''\n", + " # Get the filename without extension\n", + " base_name = os.path.splitext(html_file_path)[0]\n", + " pdf_file_path = f'{base_name}.pdf'\n", + "\n", + " # Set up ChromeDriver\n", + " global driver\n", + " if driver is None:\n", + " init('--disable-gpu', f'--print-to-pdf=\"{os.path.abspath(pdf_file_path)}\"')\n", + "\n", + " # Open the local HTML file\n", + " new('file://' + os.path.abspath(html_file_path))\n", + " # Wait for the page to fully load\n", + " wait(1.0)\n", + "\n", + " # Write the decoded data to a PDF file\n", + " print_options = PrintOptions()\n", + " print_options.scale = 1.3\n", + " with open(pdf_file_path, 'wb') as pdf_file:\n", + " pdf_file.write(base64.b64decode(driver.print_page(print_options)))\n", + "\n", + " close()\n", + " return pdf_file_path\n", + "\n", + "def convert_md_with_ruby_to_pdf(md_file_path):\n", + " ''' Convert the Markdown file containing tags into PDF '''\n", + " html_file_path, _ = convert_md_with_ruby_to_html(md_file_path)\n", + " return convert_html_with_ruby_to_pdf(html_file_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "init('--lang=en')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "convert_md_with_ruby_to_png('2024-11-21.md')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "convert_md_with_ruby_to_pdf('2024-11-21.md')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ipymock", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/nbs/2_browser.ipynb b/nbs/2_browser.ipynb index 9608d86..87ec3a6 100644 --- a/nbs/2_browser.ipynb +++ b/nbs/2_browser.ipynb @@ -295,17 +295,9 @@ "execution_count": null, "id": "a31d8134-0c69-4b4e-bcd4-4f696c94a3d4", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Amen.\n" - ] - } - ], + "outputs": [], "source": [ + "# notest\n", "try:\n", " for response in start_conversation('''\n", "> Lord, keep us in You to be one.\n", @@ -329,6 +321,7 @@ "metadata": {}, "outputs": [], "source": [ + "# notest\n", "try:\n", " for response in start_conversation({\n", " 'role': 'system',\n", @@ -355,6 +348,7 @@ "metadata": {}, "outputs": [], "source": [ + "# notest\n", "try:\n", " for response in start_conversation([{\n", " 'role': 'system',\n", @@ -391,6 +385,7 @@ "metadata": {}, "outputs": [], "source": [ + "# notest\n", "common.conversation_id = ''\n", "\n", "try:\n", @@ -435,7 +430,7 @@ "metadata": {}, "source": [ "---\n", - "selenium undetected chrome" + "Using Selenium Undetected Chrome for Operating ChatGPT" ] }, { @@ -451,9 +446,7 @@ "from selenium.webdriver.common.by import By\n", "from selenium.webdriver.common.keys import Keys\n", "from selenium.webdriver.support import expected_conditions\n", - "from selenium.webdriver.support.ui import WebDriverWait\n", - "\n", - "import undetected_chromedriver as uc" + "from selenium.webdriver.support.ui import WebDriverWait" ] }, { @@ -476,13 +469,25 @@ " node_code = node.find('code')\n", " return (\n", " f\"```{' '.join([c[len('language-'):] for c in node_code.get('class') if c.startswith('language-')])}\\n\"\n", - " f'{node_code.text.strip()}\\n'\n", + " f'{node_code.text.rstrip()}\\n'\n", " '```\\n'\n", " )\n", "\n", "markdownize = ChatGPTConverter().convert" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "85713f3b", + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from collections.abc import Iterable\n", + "from ipymock.automation import init as init_browser" + ] + }, { "cell_type": "code", "execution_count": null, @@ -491,128 +496,221 @@ "outputs": [], "source": [ "# export\n", - "def init(chrome_args = None):\n", - " options = uc.ChromeOptions()\n", - " if isinstance(chrome_args, list):\n", - " for arg in chrome_args:\n", - " options.add_argument(arg)\n", - " common.driver = uc.Chrome(options = options)\n", + "def init(chrome_args = set()):\n", + " if isinstance(chrome_args, Iterable):\n", + " chrome_args = set(chrome_args)\n", + " chrome_args.add('--lang=en')\n", + " chrome_args.add('--force-dark-mode')\n", + " init_browser(*chrome_args)\n", + " from ipymock.automation import driver\n", + " common.driver = driver\n", "\n", " login()\n", " open_chat(common.conversation_id)\n", "\n", " global start_conversation\n", - " start_conversation = ask\n", - "\n", + " start_conversation = ask" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c305a02", + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from ipymock.automation import new, wait, click, input, fill" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b72a4a", + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", "def login():\n", - " common.driver.get('https://chat.openai.com/auth/login')\n", - "\n", - " WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.presence_of_element_located((By.XPATH, '//*[text()=\"Log in\"]'))\n", - " )\n", - "\n", - " common.driver.execute_script('''\n", - " document.evaluate(\n", - " '//*[text()=\"Log in\"]',\n", - " document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null\n", - " ).snapshotItem(0).dispatchEvent(\n", - " new MouseEvent('click', {\n", - " view: window,\n", - " bubbles: true,\n", - " cancelable: true\n", - " })\n", - " );\n", - " ''')\n", - "\n", - " WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.presence_of_element_located((By.XPATH, '//button[@data-provider=\"google\"]'))\n", - " )\n", - "\n", - " common.driver.execute_script('''\n", - " document.evaluate(\n", - " '//button[@data-provider=\"google\"]',\n", - " document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null\n", - " ).snapshotItem(0).dispatchEvent(\n", - " new MouseEvent('click', {\n", - " view: window,\n", - " bubbles: true,\n", - " cancelable: true\n", - " })\n", - " );\n", - " ''')\n", - "\n", - " WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.presence_of_element_located((By.XPATH, '//input[@type=\"email\"]'))\n", - " )\n", - "\n", - " common.driver.execute_script(f'''\n", - " const google_email_input = document.evaluate('//input[@type=\"email\"]', document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotItem(0);\n", - " google_email_input.value = '{common.config['email']}';\n", - " google_email_input.dispatchEvent(\n", - " new Event('input', {{\n", - " view: window,\n", - " bubbles: true,\n", - " cancelable: true\n", - " }})\n", - " );\n", - " ''')\n", - "\n", - " WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.presence_of_element_located((By.XPATH, '//*[@id=\"identifierNext\"]'))\n", - " )\n", - "\n", - " common.driver.execute_script('''\n", - " document.evaluate(\n", - " '//*[@id=\"identifierNext\"]',\n", - " document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null\n", - " ).snapshotItem(0).dispatchEvent(\n", - " new MouseEvent('click', {\n", - " view: window,\n", - " bubbles: true,\n", - " cancelable: true\n", - " })\n", - " );\n", - " ''')\n", - "\n", - " WebDriverWait(common.driver, 10).until(\n", - " expected_conditions.element_to_be_clickable((By.XPATH, '//input[@type=\"password\"]'))\n", - " ).click()\n", - "\n", - " ActionChains(common.driver).send_keys(common.config['password']).send_keys(Keys.ENTER).perform()\n", - "\n", - " remove_portal()\n", + " new('https://chatgpt.com/auth/login')\n", "\n", + " # WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.presence_of_element_located((By.XPATH, '//*[text()=\"Log in\"]'))\n", + " # )\n", + " wait(5.0)\n", + "\n", + " # common.driver.execute_script('''\n", + " # document.evaluate(\n", + " # '//*[text()=\"Log in\"]',\n", + " # document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null\n", + " # ).snapshotItem(0).dispatchEvent(\n", + " # new MouseEvent('click', {\n", + " # view: window,\n", + " # bubbles: true,\n", + " # cancelable: true\n", + " # })\n", + " # );\n", + " # ''')\n", + " click('Log in')\n", + "\n", + " # WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.presence_of_element_located((By.XPATH, '//button[@data-provider=\"google\"]'))\n", + " # )\n", + " wait(5.0)\n", + "\n", + " # common.driver.execute_script('''\n", + " # document.evaluate(\n", + " # '//button[@data-provider=\"google\"]',\n", + " # document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null\n", + " # ).snapshotItem(0).dispatchEvent(\n", + " # new MouseEvent('click', {\n", + " # view: window,\n", + " # bubbles: true,\n", + " # cancelable: true\n", + " # })\n", + " # );\n", + " # ''')\n", + " click('Continue with Google')\n", + "\n", + " # WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.presence_of_element_located((By.XPATH, '//input[@type=\"email\"]'))\n", + " # )\n", + " wait(5.0)\n", + "\n", + " # common.driver.execute_script(f'''\n", + " # const google_email_input = document.evaluate('//input[@type=\"email\"]', document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotItem(0);\n", + " # google_email_input.value = '{common.config['email']}';\n", + " # google_email_input.dispatchEvent(\n", + " # new Event('input', {{\n", + " # view: window,\n", + " # bubbles: true,\n", + " # cancelable: true\n", + " # }})\n", + " # );\n", + " # ''')\n", + " input('Email or phone', common.config['email'])\n", + "\n", + " # WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.presence_of_element_located((By.XPATH, '//*[@id=\"identifierNext\"]'))\n", + " # )\n", + " wait(5.0)\n", + "\n", + " # common.driver.execute_script('''\n", + " # document.evaluate(\n", + " # '//*[@id=\"identifierNext\"]',\n", + " # document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null\n", + " # ).snapshotItem(0).dispatchEvent(\n", + " # new MouseEvent('click', {\n", + " # view: window,\n", + " # bubbles: true,\n", + " # cancelable: true\n", + " # })\n", + " # );\n", + " # ''')\n", + " click('Next')\n", + "\n", + " # WebDriverWait(common.driver, 10).until(\n", + " # expected_conditions.element_to_be_clickable((By.XPATH, '//input[@type=\"password\"]'))\n", + " # ).click()\n", + " wait(10.0)\n", + "\n", + " # ActionChains(common.driver).send_keys(common.config['password']).send_keys(Keys.ENTER).perform()\n", + " fill(common.config['password'])\n", + " fill(Keys.ENTER)\n", + " wait(stability_duration = 5.0)\n", + "\n", + " common.driver.maximize_window()\n", + " wait(1.0)\n", + "\n", + " remove_portal()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e39ec8f7", + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "init_browser('--lang=en', '--force-dark-mode')\n", + "login()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b35337e8", + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from selenium.common.exceptions import NoSuchElementException" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54334fd9", + "metadata": {}, + "outputs": [], + "source": [ + "# export\n", "def open_chat(conversation_id = ''):\n", + " from ipymock.automation import driver\n", + " common.driver = driver\n", " if conversation_id == '':\n", - " common.driver.get('https://chat.openai.com')\n", + " common.driver.get('https://chatgpt.com/')\n", " else:\n", - " common.driver.get(f'https://chat.openai.com/c/{conversation_id}')\n", + " common.driver.get(f'https://chatgpt.com/c/{conversation_id}')\n", " if common.conversation_id != conversation_id:\n", " common.conversation_id = conversation_id\n", " common.parent_message_id = ''\n", "\n", - " WebDriverWait(common.driver, 30).until(\n", - " lambda driver: driver.execute_script('return document.readyState') == 'complete'\n", - " )\n", + " # WebDriverWait(common.driver, 30).until(\n", + " # lambda driver: driver.execute_script('return document.readyState') == 'complete'\n", + " # )\n", + " wait(5.0)\n", "\n", " remove_portal()\n", "\n", "def remove_portal():\n", " while True:\n", + " # try:\n", + " # WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()=\"Next\"]'))\n", + " # ).click()\n", + " # except TimeoutException:\n", + " # break\n", " try:\n", - " WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()=\"Next\"]'))\n", - " ).click()\n", - " except TimeoutException:\n", + " click('Next')\n", + " wait(5.0)\n", + " except NoSuchElementException:\n", " break\n", + " # try:\n", + " # WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()=\"Done\"]'))\n", + " # ).click()\n", + " # except TimeoutException:\n", + " # pass\n", " try:\n", - " WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.element_to_be_clickable((By.XPATH, '//div[text()=\"Done\"]'))\n", - " ).click()\n", - " except TimeoutException:\n", + " click('Done')\n", + " wait(5.0)\n", + " except NoSuchElementException:\n", " pass" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e435c09", + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "open_chat(common.conversation_id)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -625,10 +723,67 @@ "chatgpt_disabled_button = (By.XPATH, '//textarea/following-sibling::button[@disabled]')\n", "chatgpt_enabled_button = (By.XPATH, '//textarea/following-sibling::button[not(@disabled)]')\n", "chatgpt_streaming = (By.CLASS_NAME, 'result-streaming')\n", - "chatgpt_response = (By.XPATH, '//div[starts-with(@class, \"flex flex-grow flex-col gap-3\")]')\n", + "chatgpt_response = (By.XPATH, '//div[starts-with(@class, \"markdown prose w-full break-words\")]')\n", "chatgpt_red_500 = (By.XPATH, '//div[contains(@class, \"border-red-500 bg-red-500/10\")]')\n", - "chatgpt_big_response = (By.XPATH, '//div[@class=\"flex-1 overflow-hidden\"]//div[p]')\n", - "chatgpt_small_response = (By.XPATH, '//div[starts-with(@class, \"markdown prose w-full break-words\")]')" + "chatgpt_big_response = (By.XPATH, '//div[@class=\"flex-1 overflow-hidden\"]//div[p or pre]')\n", + "chatgpt_small_response = (By.XPATH, '//div[@class=\"flex-1 overflow-hidden\"]//code')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08560320", + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from typing import Generator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ab97bb7", + "metadata": {}, + "outputs": [], + "source": [ + "# exporti\n", + "from ipymock.automation import exists, touch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee2bff2e", + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "ActionChains(common.driver).key_down(Keys.CONTROL).send_keys('v').key_up(Keys.CONTROL).perform()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae313fad", + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "ActionChains(common.driver).context_click().perform()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19198ada", + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "__file__ = '2_browser.ipynb'\n", + "send_button = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../assets/send-button.png'))\n", + "touch(send_button)" ] }, { @@ -640,30 +795,47 @@ "source": [ "# export\n", "def request(prompt: str) -> None:\n", - " try:\n", - " textbox = WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.element_to_be_clickable(chatgpt_textbox)\n", - " )\n", - " except TimeoutException:\n", + " # try:\n", + " # textbox = WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.element_to_be_clickable(chatgpt_textbox)\n", + " # )\n", + " # except TimeoutException:\n", + " # open_chat(common.conversation_id)\n", + " # textbox = WebDriverWait(common.driver, 5).until(\n", + " # expected_conditions.element_to_be_clickable(chatgpt_textbox)\n", + " # )\n", + " # textbox.click()\n", + " click('ChatGPT can make mistakes. Check important info.')\n", + " textbox = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../assets/message-chatgpt.png'))\n", + " textbox = 'assets/message-chatgpt.png'\n", + " if not exists(textbox):\n", " open_chat(common.conversation_id)\n", - " textbox = WebDriverWait(common.driver, 5).until(\n", - " expected_conditions.element_to_be_clickable(chatgpt_textbox)\n", - " )\n", - " textbox.click()\n", + " click('ChatGPT can make mistakes. Check important info.')\n", + " touch(textbox)\n", " # textbox.send_keys(prompt.strip())\n", - " common.driver.execute_script('''\n", - " var element = arguments[0], txt = arguments[1];\n", - " element.value += txt;\n", - " element.dispatchEvent(new Event(\"change\"));\n", - " ''',\n", - " textbox,\n", - " prompt.strip(),\n", - " )\n", + " # common.driver.execute_script('''\n", + " # var element = arguments[0], txt = arguments[1];\n", + " # element.value += txt;\n", + " # element.dispatchEvent(new Event(\"change\"));\n", + " # ''',\n", + " # textbox,\n", + " # prompt.strip(),\n", + " # )\n", + " for line in prompt.strip().split('\\n'):\n", + " fill(line)\n", + " ActionChains(common.driver).key_down(Keys.SHIFT).send_keys(Keys.ENTER).key_up(Keys.SHIFT).perform()\n", " # WebDriverWait(common.driver, 3).until_not(\n", " # expected_conditions.presence_of_element_located(chatgpt_disabled_button)\n", " # )\n", - " textbox.send_keys('\\n')\n", - " textbox.send_keys(Keys.ENTER)\n", + " click('ChatGPT can make mistakes. Check important info.')\n", + " wait(1.0)\n", + " # textbox.send_keys('\\n')\n", + " # textbox.send_keys(Keys.ENTER)\n", + " send_button = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../assets/send-button.png'))\n", + " send_button = 'assets/send-button.png'\n", + " touch(send_button)\n", + " click('ChatGPT can make mistakes. Check important info.')\n", + " wait(1.0)\n", " # try:\n", " # WebDriverWait(common.driver, 5).until(\n", " # expected_conditions.element_to_be_clickable(chatgpt_enabled_button)\n", @@ -672,31 +844,37 @@ " # pass\n", "\n", "def get_last_response():\n", + " responses = common.driver.find_elements(*chatgpt_small_response)\n", + " if responses != []:\n", + " return responses[-1]\n", " responses = common.driver.find_elements(*chatgpt_big_response)\n", " if responses != []:\n", " return responses[-1]\n", - " responses = common.driver.find_elements(*chatgpt_small_response)\n", + " responses = common.driver.find_elements(*chatgpt_response)\n", " if responses != []:\n", " return responses[-1]\n", "\n", - "def get_response() -> str:\n", + "def get_response() -> Generator[str, None, None]:\n", " try:\n", " result_streaming = WebDriverWait(common.driver, 30).until(\n", " expected_conditions.presence_of_element_located(chatgpt_streaming)\n", " )\n", " except TimeoutException:\n", - " response = common.driver.find_elements(*chatgpt_response)[-1]\n", - " is_error = common.driver.find_elements(*chatgpt_red_500) != []\n", + " response = get_last_response()\n", + " error = common.driver.find_elements(*chatgpt_red_500) != []\n", " sys.stderr.write(\n", " 'TimeoutException: having waited 30 seconds for result-streaming\\n'\n", " f'response.text = {response.text}\\n'\n", - " f'is_error = {is_error}\\n'\n", + " f'error = {error}\\n'\n", " )\n", - " if not is_error:\n", + " if not error:\n", " yield markdownize(response.get_attribute('innerHTML'))\n", - " return\n", + " result_streaming = common.driver.find_elements(*chatgpt_streaming)\n", " while result_streaming:\n", " response = get_last_response()\n", + " if response is None:\n", + " result_streaming = common.driver.find_elements(*chatgpt_streaming)\n", + " continue\n", " try:\n", " if 'text-red' in response.get_attribute('class'):\n", " sys.stderr.write(f'Error Responding: response.text = {response.text}\\n')\n", @@ -712,6 +890,30 @@ " return get_response()" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "bedb8932", + "metadata": {}, + "outputs": [], + "source": [ + "# notest\n", + "import IPython\n", + "\n", + "for response in ask('''\n", + "> Lord, keep us in You to be one.\n", + "We are the ultimate risk takers.\n", + "Our way is a way of risking life to **eternal life**.\n", + "For the eternal life of the living, risk life and make war against perishing.\n", + "For the eternal life of the dead, risk life and make war against death.\n", + ">\n", + "> ---\n", + "Holy Father, help us to overcome!\n", + "'''):\n", + " IPython.display.display(IPython.core.display.Markdown(response))\n", + " IPython.display.clear_output(wait=True)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -755,6 +957,7 @@ "metadata": {}, "outputs": [], "source": [ + "# notest\n", "try:\n", " for response in start_conversation('''\n", "> Lord, keep us in You to be one.\n", @@ -1067,9 +1270,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "ipymock", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" } }, "nbformat": 4, diff --git a/settings.ini b/settings.ini index 8b2ba92..85f606e 100644 --- a/settings.ini +++ b/settings.ini @@ -25,7 +25,7 @@ license = apache2 status = 2 # Optional. Same format as setuptools requirements -requirements = pytest==6.* nbdev==1.* openai==0.* sentence_transformers==2.* pydantic==1.* langchain==0.* faiss-cpu==1.* duckduckgo_search==2.* undetected_chromedriver==3.* selenium_profiles==2.* markdownify==0.* +requirements = pytest==6.* nbdev==1.* openai==0.* sentence_transformers==2.* pydantic==1.* langchain==0.* faiss-cpu==1.* duckduckgo_search==2.* undetected_chromedriver==3.* selenium_profiles==2.* markdownify==0.* airtest==1.* undetected_chromedriver==3.* webdriver_manager==4.* markdown==3.* # Optional. Same format as setuptools console_scripts # console_scripts = # Optional. Same format as setuptools dependency-links