diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 6fba1f4..e67701c 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10" , "3.11" ] + python-version: [ "3.8", "3.9", "3.10" , "3.11" , "3.12" ] steps: - uses: actions/checkout@v3 with: @@ -36,7 +36,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools pip -q install -r requirements.txt pip -q install -r requirements-dev.txt - name: Compile all @@ -64,7 +64,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - python-version: [ "3.7", "3.8", "3.9", "3.10" , "3.11" ] + python-version: [ "3.8", "3.8", "3.9", "3.10" , "3.11" , "3.12" ] needs: lint steps: - uses: FedericoCarboni/setup-ffmpeg@v2 @@ -80,7 +80,7 @@ jobs: - name: Install dependencies # Installing wheel due to https://github.com/pypa/pip/issues/8559 run: | - python3 -m pip -q install --upgrade pip wheel + python3 -m pip -q install --upgrade pip wheel setuptools python3 -m pip -q install -r requirements.txt --upgrade python3 -m pip -q install -r requirements-dev.txt --upgrade - name: Run tests on ${{ matrix.os }} with python ${{ matrix.python-version }} @@ -115,7 +115,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" - name: Install dependencies run: | python3 -m pip install --upgrade pip diff --git a/README.md b/README.md index 3ade30f..0e4b38e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ odmpy also has useful features for audiobooks such as adding of chapters metadat Works on Linux, macOS, and Windows. -Requires Python >= 3.7. +Requires Python >= 3.8. ![Screenshot](https://user-images.githubusercontent.com/104607/222746903-0089bea5-ba3f-4eef-8e14-b4870a5bbb27.png) diff --git a/odmpy/libby.py b/odmpy/libby.py index 179015a..7e1dd88 100644 --- a/odmpy/libby.py +++ b/odmpy/libby.py @@ -136,9 +136,9 @@ def parse_part_path(title: str, part_path: str) -> ChapterMarker: return ChapterMarker( title=title, part_name=mobj.group("part_name"), - start_second=float(mobj.group("second_stamp")) - if mobj.group("second_stamp") - else 0, + start_second=( + float(mobj.group("second_stamp")) if mobj.group("second_stamp") else 0 + ), end_second=0, ) @@ -812,16 +812,26 @@ def prepare_loan(self, loan: Dict) -> Tuple[str, Dict]: meta = self.open_loan(loan_type, card_id, title_id) download_base: str = meta["urls"]["web"] - # Sets a needed cookie + # Sets a needed cookie and parse the redirect HTML for meta. web_url = download_base + "?" + meta["message"] - _ = self.make_request( + html = self.make_request( web_url, headers={"Accept": "*/*"}, - method="HEAD", + method="GET", authenticated=False, return_res=True, ) - return download_base, meta + # audio [nav/toc, spine], ebook [nav/toc, spine, manifest] + # both in window.bData + regex = re.compile(r"window\.bData\s*=\s*({.*});") + match = regex.search(html.text) + if not match: + raise ValueError(f"Failed to parse window.bData for book info: {web_url}") + openbook = json.loads(match.group(1)) + + # set download_base for ebook + openbook["download_base"] = download_base + return download_base, openbook def process_audiobook( self, loan: Dict @@ -832,24 +842,19 @@ def process_audiobook( :param loan: :return: """ - download_base, meta = self.prepare_loan(loan) - # contains nav/toc and spine - openbook = self.make_request(meta["urls"]["openbook"]) + download_base, openbook = self.prepare_loan(loan) toc = parse_toc(download_base, openbook["nav"]["toc"], openbook["spine"]) return openbook, toc - def process_ebook(self, loan: Dict) -> Tuple[str, Dict, List[Dict]]: + def process_ebook(self, loan: Dict) -> Tuple[str, Dict]: """ Returns the data needed to download an ebook directly. :param loan: :return: """ - download_base, meta = self.prepare_loan(loan) - # contains nav/toc and spine, manifest - openbook = self.make_request(meta["urls"]["openbook"]) - rosters: List[Dict] = self.make_request(meta["urls"]["rosters"]) - return download_base, openbook, rosters + download_base, openbook = self.prepare_loan(loan) + return download_base, openbook def return_title(self, title_id: str, card_id: str) -> None: """ diff --git a/odmpy/libby_errors.py b/odmpy/libby_errors.py index 19db6a9..5d089d6 100644 --- a/odmpy/libby_errors.py +++ b/odmpy/libby_errors.py @@ -77,31 +77,33 @@ def process(http_err: requests.HTTPError) -> None: :return: """ # json response - if ( - http_err.response.status_code == HTTPStatus.BAD_REQUEST - and http_err.response.headers.get("content-type", "").startswith( - "application/json" - ) - ): - error = http_err.response.json() - if error.get("result", "") == "upstream_failure": - upstream = error.get("upstream", {}) - if upstream: - raise ClientBadRequestError( - msg=f'{upstream.get("userExplanation", "")} [errorcode: {upstream.get("errorCode", "")}]', - http_status=http_err.response.status_code, - error_response=http_err.response.text, - ) from http_err - - raise ClientBadRequestError( - msg=str(error), + if http_err.response is not None: + if hasattr(http_err.response, "json") and callable(http_err.response.json): + if ( + http_err.response.status_code == HTTPStatus.BAD_REQUEST + and http_err.response.headers.get("content-type", "").startswith( + "application/json" + ) + ): + error = http_err.response.json() + if error.get("result", "") == "upstream_failure": + upstream = error.get("upstream", {}) + if upstream: + raise ClientBadRequestError( + msg=f'{upstream.get("userExplanation", "")} [errorcode: {upstream.get("errorCode", "")}]', + http_status=http_err.response.status_code, + error_response=http_err.response.text, + ) from http_err + + raise ClientBadRequestError( + msg=str(error), + http_status=http_err.response.status_code, + error_response=http_err.response.text, + ) from http_err + + # final fallback + raise ClientError( + msg=str(http_err), http_status=http_err.response.status_code, error_response=http_err.response.text, ) from http_err - - # final fallback - raise ClientError( - msg=str(http_err), - http_status=http_err.response.status_code, - error_response=http_err.response.text, - ) from http_err diff --git a/odmpy/odm.py b/odmpy/odm.py index 5205ca1..91ea65b 100644 --- a/odmpy/odm.py +++ b/odmpy/odm.py @@ -377,11 +377,10 @@ def extract_loan_file( format_id = LibbyFormats.EBookOverdrive openbook: Dict = {} - rosters: List[Dict] = [] # pre-extract openbook first so that we can use it to create the book folder # with the creator names (needed to place the cover.jpg download) if format_id in (LibbyFormats.EBookOverdrive, LibbyFormats.MagazineOverDrive): - _, openbook, rosters = libby_client.process_ebook(selected_loan) + _, openbook = libby_client.process_ebook(selected_loan) cover_path = None if format_id in ( @@ -395,9 +394,7 @@ def extract_loan_file( file_ext = ( "acsm" if format_id in (LibbyFormats.EBookEPubAdobe, LibbyFormats.EBookPDFAdobe) - else "pdf" - if format_id == LibbyFormats.EBookPDFOpen - else "epub" + else "pdf" if format_id == LibbyFormats.EBookPDFOpen else "epub" ) book_folder, book_file_name = generate_names( title=selected_loan["title"], @@ -443,7 +440,6 @@ def extract_loan_file( loan=selected_loan, cover_path=cover_path, openbook=openbook, - rosters=rosters, libby_client=libby_client, args=args, logger=logger, @@ -1030,18 +1026,26 @@ def run(custom_args: Optional[List[str]] = None, be_quiet: bool = False) -> None "%s: %-55s %s %-25s \n * %s %s%s", colored(f"{index:2d}", attrs=["bold"]), colored(loan["title"], attrs=["bold"]), - "📰" - if args.include_magazines - and libby_client.is_downloadable_magazine_loan(loan) - else "📕" - if args.include_ebooks - and libby_client.is_downloadable_ebook_loan(loan) - else "🎧" - if args.include_ebooks or args.include_magazines - else "", - loan["firstCreatorName"] - if loan.get("firstCreatorName") - else loan.get("edition", ""), + ( + "📰" + if args.include_magazines + and libby_client.is_downloadable_magazine_loan(loan) + else ( + "📕" + if args.include_ebooks + and libby_client.is_downloadable_ebook_loan(loan) + else ( + "🎧" + if args.include_ebooks or args.include_magazines + else "" + ) + ) + ), + ( + loan["firstCreatorName"] + if loan.get("firstCreatorName") + else loan.get("edition", "") + ), f"Expires: {colored(f'{expiry_date:%Y-%m-%d}','blue' if libby_client.is_renewable(loan) else None)}", next( iter( @@ -1052,13 +1056,15 @@ def run(custom_args: Optional[List[str]] = None, be_quiet: bool = False) -> None ] ) ), - "" - if not libby_client.is_renewable(loan) - else ( - f'\n * {loan.get("availableCopies", 0)} ' - f'{ps(loan.get("availableCopies", 0), "copy", "copies")} available' - ) - + (f" (hold placed: {hold_date:%Y-%m-%d})" if hold else ""), + ( + "" + if not libby_client.is_renewable(loan) + else ( + f'\n * {loan.get("availableCopies", 0)} ' + f'{ps(loan.get("availableCopies", 0), "copy", "copies")} available' + ) + + (f" (hold placed: {hold_date:%Y-%m-%d})" if hold else "") + ), ) loan_choices: List[str] = [] diff --git a/odmpy/processing/audiobook.py b/odmpy/processing/audiobook.py index 230dbb4..4c11839 100644 --- a/odmpy/processing/audiobook.py +++ b/odmpy/processing/audiobook.py @@ -214,9 +214,11 @@ def process_audiobook_loan( part_download_url, headers={ "User-Agent": USER_AGENT, - "Range": f"bytes={already_downloaded_len}-" - if already_downloaded_len - else None, + "Range": ( + f"bytes={already_downloaded_len}-" + if already_downloaded_len + else None + ), }, timeout=args.timeout, stream=True, @@ -245,8 +247,9 @@ def process_audiobook_loan( ) except HTTPError as he: - logger.error(f"HTTPError: {str(he)}") - logger.debug(he.response.content) + if he.response is not None: + logger.error(f"HTTPError: {str(he)}") + logger.debug(he.response.content) raise OdmpyRuntimeError("HTTP Error while downloading part file.") except ConnectionError as ce: @@ -477,15 +480,19 @@ def process_audiobook_loan( create_opf( media_info, cover_filename if keep_cover else None, - file_tracks - if not args.merge_output - else [ - { - "file": book_filename - if args.merge_format == "mp3" - else book_m4b_filename - } - ], + ( + file_tracks + if not args.merge_output + else [ + { + "file": ( + book_filename + if args.merge_format == "mp3" + else book_m4b_filename + ) + } + ] + ), opf_file_path, logger, ) diff --git a/odmpy/processing/ebook.py b/odmpy/processing/ebook.py index 3451ae9..3fbc86b 100644 --- a/odmpy/processing/ebook.py +++ b/odmpy/processing/ebook.py @@ -369,7 +369,6 @@ def process_ebook_loan( loan: Dict, cover_path: Optional[Path], openbook: Dict, - rosters: List[Dict], libby_client: LibbyClient, args: argparse.Namespace, logger: logging.Logger, @@ -380,7 +379,6 @@ def process_ebook_loan( :param loan: :param cover_path: :param openbook: - :param rosters: :param libby_client: :param args: :param logger: @@ -419,15 +417,20 @@ def process_ebook_loan( with book_folder.joinpath("loan.json").open("w", encoding="utf-8") as f: json.dump(loan, f, indent=2) - with book_folder.joinpath("rosters.json").open("w", encoding="utf-8") as f: - json.dump(rosters, f, indent=2) - with book_folder.joinpath("openbook.json").open("w", encoding="utf-8") as f: json.dump(openbook, f, indent=2) - title_contents: Dict = next( - iter([r for r in rosters if r["group"] == "title-content"]), {} - ) + # old rosters: {"group": "title-content", "entries": [{"url": "http://..."}]} + # now just generate title_content entries from openbook + title_contents = [ + { + "url": openbook["download_base"] + item["path"], + "mediaType": item["media-type"], + "spinePosition": item["-odread-spine-position"], + } + for item in openbook["spine"] + ] + headers = libby_client.default_headers() headers["Accept"] = "*/*" contents_re = re.compile(r"parent\.__bif_cfc0\(self,'(?P.+)'\)") @@ -464,12 +467,11 @@ def process_ebook_loan( title_content_entries = list( filter( lambda e: _filter_content(e, media_info, toc_pages), - title_contents["entries"], + title_contents, ) ) - # Ignoring mypy error below because of https://github.com/python/mypy/issues/9372 title_content_entries = sorted( - title_content_entries, key=cmp_to_key(_sort_title_contents) # type: ignore[misc] + title_content_entries, key=cmp_to_key(_sort_title_contents) ) progress_bar = tqdm(title_content_entries, disable=args.hide_progress) has_ncx = False @@ -509,9 +511,11 @@ def process_ebook_loan( has_ncx = True manifest_entry = { "href": parsed_entry_url.path[1:], - "id": "ncx" - if media_type == "application/x-dtbncx+xml" - else _sanitise_opf_id(parsed_entry_url.path[1:]), + "id": ( + "ncx" + if media_type == "application/x-dtbncx+xml" + else _sanitise_opf_id(parsed_entry_url.path[1:]) + ), "media-type": media_type, } @@ -641,6 +645,31 @@ def process_ebook_loan( with open(asset_file_path, "wb") as f_out: f_out.write(res.content) + # HACK: download css and images to the same asset dir, fix soup + # e.g. '003' + # download into "Text/***_003_r1.jpg" and point to filename "***_003_r1.jpg" + if soup and media_type in ("application/xhtml+xml", "text/html"): + for tag, attrs, tag_target in [ + ("img", {"src": True}, "src"), + ("link", {"rel": "stylesheet"}, "href"), + ]: + for ele in soup.find_all(tag, attrs=attrs): # type: ignore[arg-type] + download_url = urlparse( + urljoin(parsed_entry_url.geturl(), ele[tag_target]) + ).geturl() + filename = os.path.basename(download_url) + ele[tag_target] = filename + + file_path = asset_folder.joinpath(filename) + if not file_path.exists(): + logger.info(f"Downloading {download_url} to {file_path}") + res = libby_client.make_request(download_url, return_res=True) + with open(file_path, "wb") as f_out: + f_out.write(res.content) + # overwrite the file with the updated soup + with open(asset_file_path, "w", encoding="utf-8") as f_out: + f_out.write(str(soup)) + if soup: # try to min. soup searches where possible if ( @@ -752,9 +781,11 @@ def process_ebook_loan( extract_isbn( media_info["formats"], format_types=[ - LibbyFormats.MagazineOverDrive - if loan["type"]["id"] == LibbyMediaTypes.Magazine - else LibbyFormats.EBookOverdrive + ( + LibbyFormats.MagazineOverDrive + if loan["type"]["id"] == LibbyMediaTypes.Magazine + else LibbyFormats.EBookOverdrive + ) ], ) or media_info["id"] @@ -788,9 +819,11 @@ def process_ebook_loan( package = build_opf_package( media_info, version=epub_version, - loan_format=LibbyFormats.MagazineOverDrive - if loan["type"]["id"] == LibbyMediaTypes.Magazine - else LibbyFormats.EBookOverdrive, + loan_format=( + LibbyFormats.MagazineOverDrive + if loan["type"]["id"] == LibbyMediaTypes.Magazine + else LibbyFormats.EBookOverdrive + ), ) if args.generate_opf: # save opf before the manifest and spine elements get added @@ -861,7 +894,7 @@ def process_ebook_loan( # Ignoring mypy error below because of https://github.com/python/mypy/issues/9372 spine_entries = sorted( - spine_entries, key=cmp_to_key(lambda a, b: _sort_spine_entries(a, b, toc_pages)) # type: ignore[misc] + spine_entries, key=cmp_to_key(lambda a, b: _sort_spine_entries(a, b, toc_pages)) # type: ignore[misc,arg-type] ) for spine_idx, entry in enumerate(spine_entries): if ( @@ -955,7 +988,6 @@ def process_ebook_loan( "media.json", "openbook.json", "loan.json", - "rosters.json", ): target = book_folder.joinpath(file_name) if target.exists(): diff --git a/odmpy/processing/odm.py b/odmpy/processing/odm.py index 7f2df52..a017a46 100644 --- a/odmpy/processing/odm.py +++ b/odmpy/processing/odm.py @@ -372,14 +372,15 @@ def process_odm( logger.debug(f"Saved license file {license_file}") except HTTPError as he: - if he.response.status_code == 404: - # odm file has expired - logger.error( - f'The loan file "{args.odm_file}" has expired. Please download again.' - ) - else: - logger.error(he.response.content) - raise OdmpyRuntimeError("HTTP Error while downloading license.") + if he.response is not None: + if he.response.status_code == 404: + # odm file has expired + logger.error( + f'The loan file "{args.odm_file}" has expired. Please download again.' + ) + else: + logger.error(he.response.content) + raise OdmpyRuntimeError("HTTP Error while downloading license.") except ConnectionError as ce: logger.error(f"ConnectionError: {str(ce)}") raise OdmpyRuntimeError("Connection Error while downloading license.") @@ -430,9 +431,11 @@ def process_odm( "User-Agent": UA, "ClientID": license_client_id, "License": lic_file_contents, - "Range": f"bytes={already_downloaded_len}-" - if already_downloaded_len - else None, + "Range": ( + f"bytes={already_downloaded_len}-" + if already_downloaded_len + else None + ), }, timeout=args.timeout, stream=True, @@ -461,8 +464,9 @@ def process_odm( ) except HTTPError as he: - logger.error(f"HTTPError: {str(he)}") - logger.debug(he.response.content) + if he.response is not None: + logger.error(f"HTTPError: {str(he)}") + logger.debug(he.response.content) raise OdmpyRuntimeError("HTTP Error while downloading part file.") except ConnectionError as ce: @@ -789,15 +793,19 @@ def process_odm( create_opf( media_info, cover_filename if keep_cover else None, - file_tracks - if not args.merge_output - else [ - { - "file": book_filename - if args.merge_format == "mp3" - else book_m4b_filename - } - ], + ( + file_tracks + if not args.merge_output + else [ + { + "file": ( + book_filename + if args.merge_format == "mp3" + else book_m4b_filename + ) + } + ] + ), opf_file_path, logger, ) @@ -832,11 +840,12 @@ def process_odm_return(args: argparse.Namespace, logger: logging.Logger) -> None early_return_res.raise_for_status() logger.info(f"Loan returned successfully: {args.odm_file}") except HTTPError as he: - if he.response.status_code == 403: - logger.warning("Loan is probably already returned.") - return - logger.error(f"HTTPError: {str(he)}") - logger.debug(he.response.content) + if he.response is not None: + if he.response.status_code == 403: + logger.warning("Loan is probably already returned.") + return + logger.error(f"HTTPError: {str(he)}") + logger.debug(he.response.content) raise OdmpyRuntimeError(f"HTTP error returning odm {args.odm_file, }") except ConnectionError as ce: logger.error(f"ConnectionError: {str(ce)}") diff --git a/odmpy/processing/shared.py b/odmpy/processing/shared.py index 9fb5103..27e87f8 100644 --- a/odmpy/processing/shared.py +++ b/odmpy/processing/shared.py @@ -125,9 +125,11 @@ def generate_names( # create book folder with just the title and first author book_folder_name = args.book_folder_format % { "Title": sanitize_path(title, exclude_chars=args.remove_from_paths), - "Author": sanitize_path(authors[0], exclude_chars=args.remove_from_paths) - if authors - else "", + "Author": ( + sanitize_path(authors[0], exclude_chars=args.remove_from_paths) + if authors + else "" + ), "Series": sanitize_path(series or "", exclude_chars=args.remove_from_paths), "ID": sanitize_path(title_id, exclude_chars=args.remove_from_paths), "ReadingOrder": sanitize_path( @@ -247,7 +249,9 @@ def write_tags( tag_langs = languages audiofile.tag.setTextFrame(LANGUAGE_FID, delimiter.join(tag_langs)) if published_date and (always_overwrite or not audiofile.tag.release_date): - audiofile.tag.release_date = published_date + audiofile.tag.release_date = LibbyClient.parse_datetime( + published_date + ).strftime("%Y-%m-%d") if cover_bytes: audiofile.tag.images.set( art.TO_ID3_ART_TYPES[art.FRONT_COVER][0], @@ -405,9 +409,9 @@ def merge_into_mp3( "-vcodec", "copy", "-b:a", - f"{audio_bitrate}k" - if audio_bitrate - else "64k", # explicitly set audio bitrate + ( + f"{audio_bitrate}k" if audio_bitrate else "64k" + ), # explicitly set audio bitrate "-f", "mp3", str(temp_book_filename), @@ -472,9 +476,9 @@ def convert_to_m4b( "-c:a", merge_codec, "-b:a", - f"{audio_bitrate}k" - if audio_bitrate - else "64k", # explicitly set audio bitrate + ( + f"{audio_bitrate}k" if audio_bitrate else "64k" + ), # explicitly set audio bitrate ] ) if cover_filename.exists(): diff --git a/setup.py b/setup.py index cd03202..ecf9eb2 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with odmpy. If not, see . # -import sys from setuptools import setup # type: ignore[import] @@ -38,8 +37,6 @@ "lxml>=4.9.0", "iso639-lang>=2.1.0", ] -if sys.version_info < (3, 8): - install_requires.append("typing_extensions") setup( name="odmpy", @@ -53,7 +50,7 @@ "odmpy = odmpy.__main__:main", ] }, - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, include_package_data=True, platforms="any", @@ -70,5 +67,6 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], ) diff --git a/tests/utils_tests.py b/tests/utils_tests.py index bbe68fc..4b793e6 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -14,9 +14,11 @@ class UtilsTests(unittest.TestCase): def test_sanitize_path(self): self.assertEqual( utils.sanitize_path(r'ac:d"e/f\g|h?i*j_ac:d"e/f\g|h?i*j', ""), - "abcdefghij_abcdefghij" - if is_windows - else r'ac:d"ef\g|h?i*j_ac:d"ef\g|h?i*j', + ( + "abcdefghij_abcdefghij" + if is_windows + else r'ac:d"ef\g|h?i*j_ac:d"ef\g|h?i*j' + ), ) self.assertEqual( utils.sanitize_path(r'ac:d"e/f\g|h?i*j'),