fix option name for logging verbosity

src/youte/cli.py

@@ -248,7 +248,7 @@ def youte():
     type=click.INT,
     help="Maximum number of result pages to retrieve",
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def search(
     query: str,
     outfile: Path,
@@ -370,7 +370,7 @@ def search(
     help="Get all comments for one or a list of videos",
     is_flag=True,
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def comments(
     items: list[str],
     outfile: Path,
@@ -465,7 +465,7 @@ def comments(
     default=100,
     show_default=True,
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def replies(
     items: list[str],
     outfile: Path,
@@ -522,7 +522,7 @@ def replies(
     default=50,
     show_default=True,
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def videos(
     items: list[str],
     outfile: Path,
@@ -577,7 +577,7 @@ def videos(
     default=50,
     show_default=True,
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def channels(
     items: list[str],
     outfile: Path,
@@ -656,7 +656,7 @@ def channels(
     type=click.INT,
     help="Maximum number of result pages to retrieve",
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def related_to(
     items: list[str],
     outfile: Path,
@@ -725,7 +725,7 @@ def related_to(
     default=50,
     show_default=True,
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def chart(
     region_code: str,
     outfile: Path,
@@ -774,7 +774,7 @@ def chart(
 @click.option(
     "-o", "--output", type=click.File(mode="w"), help="Output text file to store IDs in"
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def dehydrate(infile: Path, output: IO) -> None:
     """Extract an ID list from a file of YouTube resources
 
@@ -922,7 +922,7 @@ def dehydrate(infile: Path, output: IO) -> None:
     type=click.INT,
     help="Maximum number of result pages to retrieve",
 )
-@click_log.simple_verbosity_option(logger)
+@click_log.simple_verbosity_option(logger, "--verbosity")
 def full_archive(
     query: str,
     select: str,
@@ -958,6 +958,13 @@ def full_archive(
 
     As this command attempts to archive all YouTube resources, it might take some time
     to finish. Make sure you have enough quota before running.
+
+    A useful option is `--select`, where you specify which resource types to
+    archive. It can be one or a comma-separated list containing 'video', 'channel',
+    'thread', and 'reply'. Comment thread replies are retrieved using thread IDs,
+    thus collecting comment threads is a must before getting replies. Because of that,
+    if you want to archive the replies, both 'thread' and 'reply' will have to be
+    specified.
     """
 
     _check_compatibility(select)

tests/test_archive.py

@@ -0,0 +1,86 @@
+import os
+
+from sqlalchemy import func
+from sqlalchemy.engine import Engine
+
+from youte import database, parser
+from youte.collector import Youte
+from youte.resources import Channels, Comments, Searches, Videos
+
+API_KEY = os.environ["STAGING_API_KEY"]
+
+
+def test_create_tables(path="test.db"):
+    engine = database.set_up_database(path)
+    assert os.path.exists(path)
+    assert isinstance(engine, Engine)
+    os.remove(path)
+
+
+def test_full_archive():
+    search_query = "stoicism"
+    yob = Youte(api_key=API_KEY)
+    engine = database.set_up_database("test.db")
+
+    s_results = [
+        result
+        for result in yob.search(search_query, max_result=50, max_pages_retrieved=2)
+    ]
+    assert len(s_results) == 2
+
+    searches = parser.parse_searches(s_results)
+    assert isinstance(searches, Searches)
+    assert len(searches.items) > 50
+
+    video_ids = [s.id for s in searches.items]
+    channel_ids = [c.channel_id for c in searches.items]
+
+    v_results = [r for r in yob.get_video_metadata(video_ids)]
+    assert len(v_results) > 0
+
+    videos = parser.parse_videos(v_results)
+    assert isinstance(videos, Videos)
+    assert len(videos.items) > 50
+
+    c_results = [r for r in yob.get_channel_metadata(channel_ids)]
+    assert len(c_results) > 0
+
+    channels = parser.parse_channels(c_results)
+    assert isinstance(channels, Channels)
+    assert len(channels.items) > 10
+
+    ids_for_comments = [v.id for v in videos.items if v.comment_count < 200]
+    if len(ids_for_comments) > 10:
+        ids_for_comments = ids_for_comments[:10]
+    cmt_results = [r for r in yob.get_comment_threads(video_ids=ids_for_comments)]
+    assert len(cmt_results) > 0
+
+    comments = parser.parse_comments(cmt_results)
+    assert isinstance(comments, Comments)
+    assert len(comments.items) > 5
+
+    thread_ids = [t.id for t in comments.items if t.total_reply_count > 0]
+    if len(thread_ids) > 10:
+        thread_ids = thread_ids[:10]
+    r_results = [r for r in yob.get_thread_replies(thread_ids)]
+    replies = parser.parse_comments(r_results)
+    assert isinstance(replies, Comments)
+    assert len(replies.items) > 0
+
+    database.populate_searches(engine, [searches])
+    database.populate_videos(engine, [videos])
+    database.populate_channels(engine, [channels])
+    database.populate_comments(engine, [comments, replies])
+
+    with engine.connect() as conn:
+        search_count = conn.execute(func.count(database.Search.id))
+        assert search_count.all()[0][0] > 20
+
+        video_count = conn.execute(func.count(database.Video.id))
+        assert video_count.all()[0][0] > 20
+
+        channel_count = conn.execute(func.count(database.Channel.id))
+        assert channel_count.all()[0][0] > 20
+
+        cmt_count = conn.execute(func.count(database.Comment.id))
+        assert cmt_count.all()[0][0] > 20

tests/test_cli.py

@@ -228,7 +228,15 @@ def test_cli_videos(runner, tmp_path, videos_args, extra_args, outfile_json):
 def test_cli_channels(runner, outfile_json):
     results = runner.invoke(
         youte,
-        ["channels", "-f", Path("tests") / "channel_ids.csv", "-o", outfile_json],
+        [
+            "channels",
+            "-f",
+            Path("tests") / "channel_ids.csv",
+            "-o",
+            outfile_json,
+            "--key",
+            API_KEY,
+        ],
     )
     assert results.exit_code == 0
 

tests/test_parser.py

@@ -50,8 +50,7 @@ def test_tidy_video_multiple(yob):
         video for video in yob.get_video_metadata(ids=["qFagsLxu2Xs", "4MQyV7Wluhs"])
     ]
     tidied = parse_videos(videos)  # type: ignore
-    tidied.to_json("test_tidy.json", pretty=True)
-    tidied.to_csv("test_tidy.csv")
+    print(tidied)
     assert isinstance(tidied, Videos)