From aad2148af614a255e674f7e0fd0532a84cdc741c Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Sat, 6 Apr 2024 20:17:30 +0530 Subject: [PATCH 1/4] Add --manifest-only option --- .../management/commands/exportcontent.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/kolibri/core/content/management/commands/exportcontent.py b/kolibri/core/content/management/commands/exportcontent.py index 23d3385631f..c1d97466d47 100644 --- a/kolibri/core/content/management/commands/exportcontent.py +++ b/kolibri/core/content/management/commands/exportcontent.py @@ -60,6 +60,12 @@ def add_arguments(self, parser): parser.add_argument("channel_id", type=str) parser.add_argument("destination", type=str) + parser.add_argument( + "--manifest-only", + action="store_true", + default=False, + help="Generate only the manifest.json file", + ) def update_job_metadata(self, total_bytes_to_transfer, total_resource_count): job = get_current_job() @@ -75,9 +81,6 @@ def handle_async(self, *args, **options): data_dir = os.path.realpath(options["destination"]) node_ids = options["node_ids"] exclude_node_ids = options["exclude_node_ids"] - logger.info( - "Exporting content for channel id {} to {}".format(channel_id, data_dir) - ) channel_metadata = ChannelMetadata.objects.get(id=channel_id) @@ -91,19 +94,23 @@ def handle_async(self, *args, **options): self.update_job_metadata(total_bytes_to_transfer, total_resource_count) - exported_files = [] - - with self.start_progress( - total=total_bytes_to_transfer - ) as overall_progress_update: - for f in files: - - if self.is_cancelled(): - break - - dest = self.export_file(f, data_dir, overall_progress_update) - if dest: - exported_files.append(dest) + # dont copy files if we are only exporting the manifest + if not options["manifest_only"]: + logger.info( + "Exporting content for channel id {} to {}".format(channel_id, data_dir) + ) + exported_files = [] + with self.start_progress( + total=total_bytes_to_transfer + ) as overall_progress_update: + for f in files: + + if self.is_cancelled(): + break + + dest = self.export_file(f, data_dir, overall_progress_update) + if dest: + exported_files.append(dest) # Reraise any cancellation self.check_for_cancel() From af50fcb79daf19ced273a7b12fc99acb6cead732 Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Thu, 11 Apr 2024 12:12:56 +0530 Subject: [PATCH 2/4] refactor: move content copy logic to copy_content_files --- .../management/commands/exportcontent.py | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/kolibri/core/content/management/commands/exportcontent.py b/kolibri/core/content/management/commands/exportcontent.py index c1d97466d47..9eca8673ac5 100644 --- a/kolibri/core/content/management/commands/exportcontent.py +++ b/kolibri/core/content/management/commands/exportcontent.py @@ -96,21 +96,9 @@ def handle_async(self, *args, **options): # dont copy files if we are only exporting the manifest if not options["manifest_only"]: - logger.info( - "Exporting content for channel id {} to {}".format(channel_id, data_dir) + self.copy_content_files( + channel_id, data_dir, files, total_bytes_to_transfer ) - exported_files = [] - with self.start_progress( - total=total_bytes_to_transfer - ) as overall_progress_update: - for f in files: - - if self.is_cancelled(): - break - - dest = self.export_file(f, data_dir, overall_progress_update) - if dest: - exported_files.append(dest) # Reraise any cancellation self.check_for_cancel() @@ -127,6 +115,24 @@ def handle_async(self, *args, **options): ) content_manifest.write(manifest_path) + def copy_content_files(self, channel_id, data_dir, files, total_bytes_to_transfer): + logger.info( + "Exporting content for channel id {} to {}".format(channel_id, data_dir) + ) + exported_files = [] + with self.start_progress( + total=total_bytes_to_transfer + ) as overall_progress_update: + for f in files: + if self.is_cancelled(): + break + + dest = self.export_file(f, data_dir, overall_progress_update) + if dest: + exported_files.append(dest) + + return exported_files + def export_file(self, f, data_dir, overall_progress_update): filename = get_content_file_name(f) try: From 4377ab54e42ec4a8e08ab9bab89981566219f4ca Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Thu, 11 Apr 2024 12:14:23 +0530 Subject: [PATCH 3/4] add test for exportcontent --manifest-only command --- .../core/content/test/test_import_export.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/kolibri/core/content/test/test_import_export.py b/kolibri/core/content/test/test_import_export.py index 853cfd38300..2ddcae92f98 100644 --- a/kolibri/core/content/test/test_import_export.py +++ b/kolibri/core/content/test/test_import_export.py @@ -2254,6 +2254,33 @@ def test_local_cancel_during_transfer( ) cancel_mock.assert_called_with() + @patch( + "kolibri.core.content.management.commands.exportcontent.Command.copy_content_files" + ) + def test_manifest_only( + self, + copy_content_files_mock, + ContentManifestMock, + get_content_nodes_data_mock, + get_import_export_nodes_mock, + ): + get_content_nodes_data_mock.return_value = ( + 1, + [LocalFile.objects.values("id", "file_size", "extension").first()], + 10, + ) + + # run with manifest-only option + call_command( + "exportcontent", self.the_channel_id, tempfile.mkdtemp(), manifest_only=True + ) + + copy_content_files_mock.assert_not_called() + + ContentManifestMock.return_value.write.assert_called_once() + + # Shall be enough mock assertions for now ? + class TestFilesToTransfer(TestCase): From 42be9a06e9241030588d7b6edc0e964ea098a8de Mon Sep 17 00:00:00 2001 From: Sujai Kumar Gupta Date: Thu, 11 Apr 2024 22:28:09 +0530 Subject: [PATCH 4/4] remove unused exported_files --- kolibri/core/content/management/commands/exportcontent.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/kolibri/core/content/management/commands/exportcontent.py b/kolibri/core/content/management/commands/exportcontent.py index 9eca8673ac5..fb2e7865bfb 100644 --- a/kolibri/core/content/management/commands/exportcontent.py +++ b/kolibri/core/content/management/commands/exportcontent.py @@ -119,19 +119,13 @@ def copy_content_files(self, channel_id, data_dir, files, total_bytes_to_transfe logger.info( "Exporting content for channel id {} to {}".format(channel_id, data_dir) ) - exported_files = [] with self.start_progress( total=total_bytes_to_transfer ) as overall_progress_update: for f in files: if self.is_cancelled(): break - - dest = self.export_file(f, data_dir, overall_progress_update) - if dest: - exported_files.append(dest) - - return exported_files + self.export_file(f, data_dir, overall_progress_update) def export_file(self, f, data_dir, overall_progress_update): filename = get_content_file_name(f)