From 49c7b315988b1f6d06cf2c531d3de93ae0d0e785 Mon Sep 17 00:00:00 2001 From: Laurent Monin Date: Wed, 10 Jan 2024 17:49:31 +0100 Subject: [PATCH 1/3] Add support for multiple directories to scan command --- lb_content_resolver/database.py | 46 +++++++++++++++++++++------------ lb_content_resolver/utils.py | 10 +++++++ resolve.py | 8 +++--- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/lb_content_resolver/database.py b/lb_content_resolver/database.py index a6c5e41..e606a52 100755 --- a/lb_content_resolver/database.py +++ b/lb_content_resolver/database.py @@ -18,6 +18,9 @@ from lb_content_resolver.model.tag import Tag, RecordingTag from lb_content_resolver.formats import mp3, m4a, flac, ogg_opus, ogg_vorbis, wma +from lb_content_resolver.utils import existing_dirs + + SupportedFormat = namedtuple('SupportedFormat', ('extensions', 'handler')) SUPPORTED_FORMATS = ( SupportedFormat({'.flac'}, flac), @@ -73,11 +76,18 @@ def close(self): """ Close the db.""" db.close() - def scan(self, music_dir): + def scan(self, music_dirs): """ - Scan a music dir and add tracks to sqlite. + Scan music directories and add tracks to sqlite. """ - self.music_dir = os.path.abspath(music_dir) + if not music_dirs: + print("No directory to scan") + return + + self.music_dirs = tuple(sorted(set(existing_dirs(music_dirs)))) + if not self.music_dirs: + print("No valid directories to scan") + return # Keep some stats self.total = 0 @@ -90,12 +100,16 @@ def scan(self, music_dir): # Future improvement, commit to DB only every 1000 tracks or so. print("Check collection size...") self.track_count_estimate = 0 - self.traverse("", dry_run=True) + for music_dir in self.music_dirs: + print("Counting candidates in %r ..." % music_dir) + self.traverse(music_dir, "", dry_run=True) self.audio_file_count = self.track_count_estimate print("Found %s audio files" % self.audio_file_count) with tqdm(total=self.track_count_estimate) as self.progress_bar: - self.traverse("") + for music_dir in self.music_dirs: + print("Scanning %r ..." % music_dir) + self.traverse(music_dir, "") self.close() @@ -107,29 +121,29 @@ def scan(self, music_dir): if self.total != self.not_changed + self.updated + self.added + self.error: print("And for some reason these numbers don't add up to the total number of tracks. Hmmm.") - def traverse(self, relative_path, dry_run=False): + def traverse(self, music_dir, relative_path, dry_run=False): """ This recursive function searches for audio files and descends into sub directories """ if not relative_path: - fullpath = self.music_dir + fullpath = music_dir else: - fullpath = os.path.join(self.music_dir, relative_path) + fullpath = os.path.join(music_dir, relative_path) for f in sorted(os.listdir(fullpath)): if f in {'.', '..'}: continue new_relative_path = os.path.join(relative_path, f) - new_full_path = os.path.join(self.music_dir, new_relative_path) + new_full_path = os.path.join(music_dir, new_relative_path) if os.path.isfile(new_full_path) and match_extensions(new_full_path, ALL_EXTENSIONS): if not dry_run: - self.add(new_relative_path) + self.add(music_dir, new_relative_path) else: self.track_count_estimate += 1 elif os.path.isdir(new_full_path): - if not self.traverse(new_relative_path, dry_run): + if not self.traverse(music_dir, new_relative_path, dry_run): return False return True @@ -177,13 +191,13 @@ def add_or_update_recording(self, mdata): recording.save() return "updated", details - def read_metadata_and_add(self, relative_path, extension, mtime, update): + def read_metadata_and_add(self, music_dir, relative_path, extension, mtime, update): """ Read the metadata from supported files and then add the recording to the DB. """ - file_path = os.path.join(self.music_dir, relative_path) + file_path = os.path.join(music_dir, relative_path) # We've never seen this before, or it was updated since we last saw it. mdata = EXTENSION_HANDLER[extension].read(file_path) @@ -216,14 +230,14 @@ def convert_to_uuid(self, value): return None return None - def add(self, relative_path): + def add(self, music_dir, relative_path): """ Given a file, check to see if we already have it and if we do, if it has changed since the last time we read it. If it is new or has been changed, update in the DB. """ - fullpath = os.path.join(self.music_dir, relative_path) + fullpath = os.path.join(music_dir, relative_path) # update the progress bar self.progress_bar.update(1) @@ -256,7 +270,7 @@ def add(self, relative_path): self.progress_bar.write("unchanged %s" % base) return - status, details = self.read_metadata_and_add(relative_path, ext, ts, exists) + status, details = self.read_metadata_and_add(music_dir, relative_path, ext, ts, exists) if status == "updated": self.progress_bar.write(" update %s" % details) self.updated += 1 diff --git a/lb_content_resolver/utils.py b/lb_content_resolver/utils.py index f55d9de..5bf1f2b 100755 --- a/lb_content_resolver/utils.py +++ b/lb_content_resolver/utils.py @@ -1,3 +1,5 @@ +import os + from troi.splitter import plist from troi import Recording as TroiRecording @@ -92,3 +94,11 @@ class bcolors: ENDC = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' + + +def existing_dirs(paths): + """Yield absolute paths for all existing directories in the iterable passed""" + for path in paths: + abspath = os.path.abspath(path) + if os.path.isdir(abspath): + yield abspath diff --git a/resolve.py b/resolve.py index 1e55c64..255321d 100755 --- a/resolve.py +++ b/resolve.py @@ -88,13 +88,13 @@ def create(db_file): @click.command() @click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) -@click.argument('music_dir') -def scan(db_file, music_dir): - """Scan a directory and its subdirectories for music files to add to the collection""" +@click.argument('music_dirs', nargs=-1, type=click.Path()) +def scan(db_file, music_dirs): + """Scan one or more directories and their subdirectories for music files to add to the collection""" db_file = db_file_check(db_file) db = Database(db_file) db.open() - db.scan(music_dir) + db.scan(music_dirs) @click.command() From 9cdbab3fc831565d35cc1ac1bdac078008e2d1ae Mon Sep 17 00:00:00 2001 From: Laurent Monin Date: Wed, 10 Jan 2024 18:06:49 +0100 Subject: [PATCH 2/3] Let the user set default music directories in config If paths are passed to scan command, this list is ignored. Invalid directories are skipped. --- config.py.sample | 8 ++++++++ resolve.py | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/config.py.sample b/config.py.sample index fa007f4..858f351 100644 --- a/config.py.sample +++ b/config.py.sample @@ -6,3 +6,11 @@ SUBSONIC_HOST = "" # include http:// or https:// SUBSONIC_USER = "" SUBSONIC_PASSWORD = "" SUBSONIC_PORT = 4533 + +# List of music directories to scan by default +# If paths are passed to scan command, this list is ignored. +# Invalid directories are skipped. +MUSIC_DIRECTORIES = ( + 'My/Music/Directory 1', + 'My/Music/Directory 2', +) diff --git a/resolve.py b/resolve.py index 255321d..3459e28 100755 --- a/resolve.py +++ b/resolve.py @@ -72,6 +72,16 @@ def db_file_check(db_file): return db_file +def music_directories_from_config(): + """ Returns list of music directories if any in config file. """ + + try: + import config + return list(set(config.MUSIC_DIRECTORIES)) + except: + return [] + + @click.group() def cli(): pass @@ -90,10 +100,14 @@ def create(db_file): @click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) @click.argument('music_dirs', nargs=-1, type=click.Path()) def scan(db_file, music_dirs): - """Scan one or more directories and their subdirectories for music files to add to the collection""" + """Scan one or more directories and their subdirectories for music files to add to the collection. + If no path is passed, check for MUSIC_DIRECTORIES in config instead. + """ db_file = db_file_check(db_file) db = Database(db_file) db.open() + if not music_dirs: + music_dirs = music_directories_from_config() db.scan(music_dirs) From d8b938008722a06dcb6d32af6dfb195a0f663adb Mon Sep 17 00:00:00 2001 From: Laurent Monin Date: Wed, 10 Jan 2024 18:38:22 +0100 Subject: [PATCH 3/3] Update README.md --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e0e3b0a..9b2371c 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ If you decide not to use the config.py file, make sure to pass the path to the D command. All further examples in this file assume you added the config file and will therefore omit the -d option. +You can also define a set of directories to be used by default for the scan command in `MUSIC_DIRECTORIES`. + ## Scanning your collection Note: Soon we will eliminate the requirement to do a filesystem scan before also doing a subsonic @@ -66,10 +68,13 @@ Then prepare the index and scan a music collection. mp3, m4a, wma, OggVorbis, Og ``` ./resolve.py create -./resolve.py scan +./resolve.py scan ``` -If you remove from tracks from your collection, use cleanup to remove refereces to those tracks: +If you configured `MUSIC_DIRECTORIES` in config file, you can just call `./resolve.py scan`. +It should be noted paths passed on command line take precedence over this configuration. + +If you remove from tracks from your collection, use cleanup to remove references to those tracks: ``` ./resolve.py cleanup