From 9b31507a805f1a9faf436a548f9492a268d81049 Mon Sep 17 00:00:00 2001 From: Robert Kaye Date: Wed, 10 Jan 2024 12:12:56 +0100 Subject: [PATCH] Rework the index_dir and use db_files instead. --- README.md | 49 +++++------ config.py.sample | 3 + lb_content_resolver/content_resolver.py | 2 - lb_content_resolver/database.py | 24 +++--- lb_content_resolver/fuzzy_index.py | 7 +- resolve.py | 110 +++++++++++++++--------- 6 files changed, 111 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index fb415de..e0e3b0a 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,21 @@ source .virtualenv/bin/activate pip install -r requirements.txt ``` +### Setting up config.py + +While it isn't strictly necessary to setup config.py, it makes using the resolver easier: + +``` +cp config.py.sample config.py +``` + +Then edit config.py and set the location of where you're going to store your resolver database file +into DATABASE_FILE. If you plan to use a Subsonic API, the fill out the Subsonic section as well. + +If you decide not to use the config.py file, make sure to pass the path to the DB file with -d to each +command. All further examples in this file assume you added the config file and will therefore omit +the -d option. + ## Scanning your collection Note: Soon we will eliminate the requirement to do a filesystem scan before also doing a subsonic @@ -50,14 +65,14 @@ scan (if you plan to use subsonic). For now, do the file system scan, then the s Then prepare the index and scan a music collection. mp3, m4a, wma, OggVorbis, OggOpus and flac files are supported. ``` -./resolve.py create music_index -./resolve.py scan music_index +./resolve.py create +./resolve.py scan ``` If you remove from tracks from your collection, use cleanup to remove refereces to those tracks: ``` -./resolve.py cleanup music_index +./resolve.py cleanup ``` ### Scan a Subsonic collection @@ -71,7 +86,7 @@ cp config.py.sample config.py Then edit the file and add your subsonic configuration. ``` -./resolve.py subsonic music_index +./resolve.py subsonic ``` This will match your collection to the remove subsonic API collection. @@ -94,7 +109,7 @@ curl "https://api.listenbrainz.org/1/playlist/" > test.jspf Finally, resolve the playlist to local files: ``` -./resolve.py playlist music_index input.jspf output.m3u +./resolve.py playlist input.jspf output.m3u ``` Then open the m3u playlist with a local tool. @@ -124,21 +139,7 @@ to download more data for your MusicBrainz tagged music collection. First, download tag and popularity data: ``` -./resolve.py metadata music_index -``` - -Then, copy config.py.sample to config.py and then edit config.py: - -``` -cp config.py.sample config.py -edit config.py -``` - -Fill out the values for your subsonic server API and save the file. -Finally, match your collection against the subsonic collection: - -``` -./resolve.py subsonic music_index +./resolve.py metadata ``` ### Playlist generation @@ -167,7 +168,7 @@ isn't very suited for the prompt that was given. #### Artist Element ``` -./resolve.py lb-radio music_index easy 'artist:(taylor swift, drake)' +./resolve.py lb-radio easy 'artist:(taylor swift, drake)' ``` Generates a playlist with music from Taylor Swift and artists similar @@ -177,14 +178,14 @@ to her and Drake, and artists similar to him. #### Tag Element ``` -./resolve.py lb-radio music_index easy 'tag:(downtempo, trip hop)' +./resolve.py lb-radio easy 'tag:(downtempo, trip hop)' ``` This will generate a playlist on easy mode for recordings that are tagged with "downtempo" AND "trip hop". ``` -./resolve.py lb-radio music_index medium 'tag:(downtempo, trip hop)::or' +./resolve.py lb-radio medium 'tag:(downtempo, trip hop)::or' ``` This will generate a playlist on medium mode for recordings that are @@ -194,7 +195,7 @@ at the end of the prompt. You can include more than on tag query in a prompt: ``` -./resolve.py lb-radio music_index medium 'tag:(downtempo, trip hop)::or tag:(punk, ska)' +./resolve.py lb-radio medium 'tag:(downtempo, trip hop)::or tag:(punk, ska)' ``` #### Stats, Collections, Playlists and Rec diff --git a/config.py.sample b/config.py.sample index 2a22996..fa007f4 100644 --- a/config.py.sample +++ b/config.py.sample @@ -1,3 +1,6 @@ +# Where to find the database file +DATABASE_FILE = "" + # To connect to a subsonic API SUBSONIC_HOST = "" # include http:// or https:// SUBSONIC_USER = "" diff --git a/lb_content_resolver/content_resolver.py b/lb_content_resolver/content_resolver.py index 915a5c7..d5b2418 100755 --- a/lb_content_resolver/content_resolver.py +++ b/lb_content_resolver/content_resolver.py @@ -116,8 +116,6 @@ def resolve_playlist(self, match_threshold, recordings=None, jspf_playlist=None) if recordings is None and jspf_playlist is None: raise ValueError("Either recordings or jspf_playlist must be passed.") - print("\nResolve recordings to local files or subsonic ids") - artist_recording_data = [] if jspf_playlist is not None: if len(jspf_playlist["playlist"]["track"]) == 0: diff --git a/lb_content_resolver/database.py b/lb_content_resolver/database.py index 75eee1b..33d2973 100755 --- a/lb_content_resolver/database.py +++ b/lb_content_resolver/database.py @@ -23,24 +23,16 @@ class Database: ''' Keep a database with metadata for a collection of local music files. ''' - def __init__(self, index_dir): - self.index_dir = index_dir - self.db_file = os.path.join(index_dir, "lb_resolve.db") + def __init__(self, db_file): + self.db_file = db_file self.fuzzy_index = None def create(self): """ - Create the index directory for the data. Currently it contains only - the sqlite dir, but in the future we may serialize the fuzzy index here as well. + Create the database. Can be run again to create tables that have been recently added to the code, + but don't exist in the DB yet. """ - if not os.path.exists(self.index_dir): - try: - os.mkdir(self.index_dir) - except OSError as err: - print("Could not create index directory: %s (%s)" % (self.index_dir, err)) - return - setup_db(self.db_file) db.connect() db.create_tables([Recording, RecordingMetadata, Tag, RecordingTag, RecordingSubsonic, UnresolvedRecording]) @@ -84,7 +76,7 @@ def scan(self, music_dir): with tqdm(total=self.track_count_estimate) as self.progress_bar: self.traverse("") - self.close_db() + self.close() print("Checked %s tracks:" % self.total) print(" %5d tracks not changed since last run" % self.not_changed) @@ -285,12 +277,16 @@ def database_cleanup(self, dry_run): print("RM %s" % recording.file_path) recording_ids.append(recording.id) + if not recording_ids: + print("No cleanup needed, all recordings found") + return + if not dry_run: placeholders = ",".join(("?", ) * len(recording_ids)) db.execute_sql("""DELETE FROM recording WHERE recording.id IN (%s)""" % placeholders, tuple(recording_ids)) print("Stale references removed") else: - print("--delete not specified, no refeences removed") + print("--delete not specified, no refences removed") def metadata_sanity_check(self, include_subsonic=False): """ diff --git a/lb_content_resolver/fuzzy_index.py b/lb_content_resolver/fuzzy_index.py index 548c794..8142326 100755 --- a/lb_content_resolver/fuzzy_index.py +++ b/lb_content_resolver/fuzzy_index.py @@ -71,7 +71,10 @@ def search(self, query_data): output = [] for i, result in enumerate(results): - output.append({ "confidence": fabs(result[1][0]), - "recording_id": result[0][0] }) + if len(result[0]): + output.append({ "confidence": fabs(result[1][0]), + "recording_id": result[0][0] }) + else: + output.append({ "confidence": 0.0, "recording_id": 0 }) return output diff --git a/resolve.py b/resolve.py index e6b6837..cd9b51d 100755 --- a/resolve.py +++ b/resolve.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import sys import click @@ -18,8 +19,6 @@ from lb_content_resolver.unresolved_recording import UnresolvedRecordingTracker from troi.playlist import PLAYLIST_TRACK_EXTENSION_URI -# TODO: Think up a better way to specify the DB location - def output_playlist(db, jspf, upload_to_subsonic, save_to_playlist, dont_ask): if jspf is None: @@ -54,44 +53,67 @@ def output_playlist(db, jspf, upload_to_subsonic, save_to_playlist, dont_ask): print("Playlist displayed, but not saved. Use -p or -u options to save/upload playlists.") +def db_file_check(db_file): + """ Check the db_file argument and give useful user feedback. """ + + if not db_file: + try: + import config + except ModuleNotFoundError: + print("Database file not specified with -d (--db_file) argument. Consider adding it to config.py for ease of use.") + sys.exit(-1) + + if not config.DATABASE_FILE: + print("config.py found, but DATABASE_FILE is empty. Please add it or use -d option to specify it.") + sys.exit(-1) + + return config.DATABASE_FILE + else: + return db_file + + @click.group() def cli(): pass @click.command() -@click.argument('index_dir') -def create(index_dir): - """Create a new index directory to track a music collection""" - db = Database(index_dir) +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) +def create(db_file): + """Create a new database to track a music collection""" + db_file = db_file_check(db_file) + db = Database(db_file) db.create() @click.command() -@click.argument('index_dir') +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) @click.argument('music_dir') -def scan(index_dir, music_dir): +def scan(db_file, music_dir): """Scan a directory and its subdirectories for music files to add to the collection""" - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() db.scan(music_dir) @click.command() -@click.option('-d', '--delete', required=False, is_flag=True, default=True) -@click.argument('index_dir') -def cleanup(delete, index_dir): +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) +@click.option("-r", "--remove", required=False, is_flag=True, default=True) +def cleanup(db_file, remove): """Perform a database cleanup. Check that files exist and if they don't remove from the index""" - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() - db.database_cleanup(delete) + db.database_cleanup(remove) @click.command() -@click.argument('index_dir') -def metadata(index_dir): +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) +def metadata(db_file): """Lookup metadata (popularity and tags) for recordings""" - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() lookup = MetadataLookup() lookup.lookup() @@ -102,22 +124,24 @@ def metadata(index_dir): @click.command() -@click.argument('index_dir') -def subsonic(index_dir): +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) +def subsonic(db_file): """Scan a remote subsonic music collection""" - db = SubsonicDatabase(index_dir) + db_file = db_file_check(db_file) + db = SubsonicDatabase(db_file) db.open() db.sync() @click.command() -@click.argument('index_dir') +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) +@click.option('-t', '--threshold', default=.80) @click.argument('jspf_playlist') @click.argument('m3u_playlist') -@click.option('-t', '--threshold', default=.80) -def playlist(index_dir, jspf_playlist, m3u_playlist, threshold): +def playlist(db_file, threshold, jspf_playlist, m3u_playlist): """ Resolve a JSPF file with MusicBrainz recording MBIDs to files in the local collection""" - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() cr = ContentResolver() jspf = read_jspf_playlist(jspf_playlist) @@ -126,15 +150,16 @@ def playlist(index_dir, jspf_playlist, m3u_playlist, threshold): @click.command() +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) @click.option('-u', '--upload-to-subsonic', required=False, is_flag=True) @click.option('-p', '--save-to-playlist', required=False) @click.option('-y', '--dont-ask', required=False, is_flag=True, help="write playlist to m3u file") -@click.argument('index_dir') @click.argument('mode') @click.argument('prompt') -def lb_radio(upload_to_subsonic, save_to_playlist, dont_ask, index_dir, mode, prompt): +def lb_radio(db_file, upload_to_subsonic, save_to_playlist, dont_ask, mode, prompt): """Use the ListenBrainz Radio engine to create a playlist from a prompt, using a local music collection""" - db = SubsonicDatabase(index_dir) + db_file = db_file_check(db_file) + db = SubsonicDatabase(db_file) db.open() r = ListenBrainzRadioLocal() jspf = r.generate(mode, prompt) @@ -147,40 +172,41 @@ def lb_radio(upload_to_subsonic, save_to_playlist, dont_ask, index_dir, mode, pr @click.command() -@click.argument('index_dir') +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) @click.argument('count', required=False, default=250) -def top_tags(index_dir, count): +def top_tags(db_file, count): "Display the top most used tags in the music collection. Useful for writing LB Radio tag prompts" - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() tt = TopTags() tt.print_top_tags_tightly(count) @click.command() -@click.argument('index_dir') +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) @click.option('-e', '--exclude-different-release', required=False, default=False, is_flag=True) -def duplicates(exclude_different_release, index_dir): +def duplicates(db_file, exclude_different_release): "Print all the tracks in the DB that are duplciated as per recording_mbid" - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() fd = FindDuplicates(db) fd.print_duplicate_recordings(exclude_different_release) @click.command() +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) @click.option('-u', '--upload-to-subsonic', required=False, is_flag=True, default=False) @click.option('-p', '--save-to-playlist', required=False) @click.option('-y', '--dont-ask', required=False, is_flag=True, help="write playlist to m3u file") -@click.argument('index_dir') @click.argument('user_name') -def periodic_jams(upload_to_subsonic, save_to_playlist, dont_ask, index_dir, user_name): +def periodic_jams(db_file, upload_to_subsonic, save_to_playlist, dont_ask, user_name): "Generate a periodic jams playlist" - db = SubsonicDatabase(index_dir) + db_file = db_file_check(db_file) + db = SubsonicDatabase(db_file) db.open() - # TODO: ensure that we catch upload to subsonic when we have a FS playlist - target = "subsonic" if upload_to_subsonic else "filesystem" pj = LocalPeriodicJams(user_name, target) jspf = pj.generate() @@ -192,11 +218,11 @@ def periodic_jams(upload_to_subsonic, save_to_playlist, dont_ask, index_dir, use @click.command() -@click.argument('index_dir') -def unresolved(index_dir): +@click.option("-d", "--db_file", help="Database file for the local collection", required=False, is_flag=False) +def unresolved(db_file): "Show the top unresolved releases" - - db = Database(index_dir) + db_file = db_file_check(db_file) + db = Database(db_file) db.open() urt = UnresolvedRecordingTracker() releases = urt.get_releases()