From 93d07cd18756e49f1b29af7c74bd6f7bac66a6c4 Mon Sep 17 00:00:00 2001 From: chrwm Date: Mon, 31 Oct 2022 12:15:32 +0100 Subject: [PATCH 1/7] Set omit_none=False #71 With changing to False, metadata json keys and values will remain in the metadata as `null` after processing with omi. --- src/omi/dialects/oep/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omi/dialects/oep/compiler.py b/src/omi/dialects/oep/compiler.py index 3814ed9..05949d2 100644 --- a/src/omi/dialects/oep/compiler.py +++ b/src/omi/dialects/oep/compiler.py @@ -16,7 +16,7 @@ def _compile_date(self, date: datetime, format): else: return None - def _construct_dict(self, *args, omit_none=True, **kwargs): + def _construct_dict(self, *args, omit_none=False, **kwargs): """ Accepts a list of arguments of shape (name: str, field: Compileable) and returns a dictionary that maps name -> self.visit(field). If `omit_none` is true, fields that are `None` are ignored. @@ -368,7 +368,7 @@ def visit_metadata(self, metadata: oem_v15.OEPMetadata, *args, **kwargs): path="https://creativecommons.org/publicdomain/zero/1.0/", ), ), - _comment=self._construct_dict( + _comment=self._construct_dict( metadata=metadata.comment.metadata_info, dates=metadata.comment.dates, units=metadata.comment.units, From 0ac0d390e39c81e82ab906e2d51cd68319085490 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Fri, 19 Jan 2024 14:47:25 +0100 Subject: [PATCH 2/7] Add change again: was lost during solving merge conflict #71 --- src/omi/dialects/oep/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/omi/dialects/oep/compiler.py b/src/omi/dialects/oep/compiler.py index 1dbcbe4..4a1dfef 100644 --- a/src/omi/dialects/oep/compiler.py +++ b/src/omi/dialects/oep/compiler.py @@ -18,7 +18,7 @@ def compile_date_or_none(x, format=None): class JSONCompiler(Compiler): __METADATA_VERSION = "OEP-1.4.0" - def _construct_dict(self, *args, omit_none=True, **kwargs): + def _construct_dict(self, *args, omit_none=False, **kwargs): """ Accepts a list of arguments of shape (name: str, field: Compileable) and returns a dictionary that maps name -> self.visit(field). If `omit_none` is true, fields that are `None` are ignored. From 2c7e97054446751043c24d16ad952289f7d18430 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Fri, 19 Jan 2024 16:06:25 +0100 Subject: [PATCH 3/7] extend the cli toolbox by introducing the omit none option to the translate command #71 --- src/omi/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/omi/cli.py b/src/omi/cli.py index 72bb009..024d35e 100644 --- a/src/omi/cli.py +++ b/src/omi/cli.py @@ -29,12 +29,14 @@ def grp(): @click.option("-f", help="Dialect identifier of the input") @click.option("-t", default="oep-v1.5", help="Dialect identifier to translate to") @click.option("-o", default=None, help="Output file") +@click.option("-omit_nones", default=True, help="Use to either keep or omit all null / none values from the json.") @click.argument("file_path") -def translate(f, t, o, file_path): +def translate(f, t, o, omit_nones, file_path): with open(file_path, "r", encoding="utf-8") as infile: from_dialect = get_dialect(f)() obj = from_dialect.parse(infile.read()) to_dialect = get_dialect(t)() + to_dialect._compiler.OMIT_NONE_FIELDS = omit_nones s = to_dialect.compile_and_render(obj) if o: with open(o, "w", encoding="utf-8") as outfile: From 3b02795dfd9a90cee4053c1fb5d35bc5d0a168ea Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Fri, 19 Jan 2024 16:08:20 +0100 Subject: [PATCH 4/7] Introduce class parameter to be able to control the omit none functionality easily #71 --- src/omi/dialects/oep/compiler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/omi/dialects/oep/compiler.py b/src/omi/dialects/oep/compiler.py index 4a1dfef..50e2742 100644 --- a/src/omi/dialects/oep/compiler.py +++ b/src/omi/dialects/oep/compiler.py @@ -17,8 +17,9 @@ def compile_date_or_none(x, format=None): class JSONCompiler(Compiler): __METADATA_VERSION = "OEP-1.4.0" + OMIT_NONE_FIELDS = False - def _construct_dict(self, *args, omit_none=False, **kwargs): + def _construct_dict(self, *args, omit_none=None, **kwargs): """ Accepts a list of arguments of shape (name: str, field: Compileable) and returns a dictionary that maps name -> self.visit(field). If `omit_none` is true, fields that are `None` are ignored. @@ -32,6 +33,7 @@ def _construct_dict(self, *args, omit_none=False, **kwargs): ------- """ + omit_none = self.OMIT_NONE_FIELDS d = { field_name: self.visit(field) for field_name, field in args From 6da0b9ba79982fe8ab4652ba4093f1a2467c02cf Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Fri, 19 Jan 2024 16:09:23 +0100 Subject: [PATCH 5/7] update all tests to use the new omit none control parameter. All tests omit the none value from json because the static test structures used to compare the results also do not include nones / nulls #71 --- tests/test_cli/test_cli_translation.py | 11 ++++++++--- tests/test_dialects/test_oep/test_compiler.py | 4 ++++ tests/test_dialects/test_oep/test_translation.py | 2 ++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/test_cli/test_cli_translation.py b/tests/test_cli/test_cli_translation.py index 8df1680..6ec50ae 100644 --- a/tests/test_cli/test_cli_translation.py +++ b/tests/test_cli/test_cli_translation.py @@ -3,15 +3,20 @@ from omi.cli import translate import json + def test_cli_translation(): @click.command() - @click.argument('name') + @click.argument("name") def hello(name): - click.echo('Hello %s!' % name) + click.echo("Hello %s!" % name) runner = CliRunner() - result = runner.invoke(translate, ["-f", "oep-v1.3", "-t", "oep-v1.4", "tests/data/metadata_v13.json"]) + result = runner.invoke( + translate, + ["-f", "oep-v1.3", "-t", "oep-v1.4", "-omit_nones", "True", "tests/data/metadata_v13.json"], + ) with open("tests/data/metadata_v13_converted.json") as expected_file: expected = json.loads(expected_file.read()) + print(result) assert result.exit_code == 0 assert json.loads(result.output) == expected diff --git a/tests/test_dialects/test_oep/test_compiler.py b/tests/test_dialects/test_oep/test_compiler.py index 7916548..13eaaa4 100644 --- a/tests/test_dialects/test_oep/test_compiler.py +++ b/tests/test_dialects/test_oep/test_compiler.py @@ -7,6 +7,8 @@ def test_compiler_v1_5(): compiler = JSONCompilerOEM15() + # omit none values as expected result also does not include None´s + compiler.OMIT_NONE_FIELDS = True with open("tests/data/metadata_v15.json", "r", encoding="utf-8") as _input_file: expected_result = json.load(_input_file) result = compiler.visit(metadata_v_1_5) @@ -15,6 +17,8 @@ def test_compiler_v1_5(): def test_compiler_v1_4(): compiler = JSONCompiler() + # omit none values as expected result also does not include None´s + compiler.OMIT_NONE_FIELDS = True with open("tests/data/metadata_v14.json", "r", encoding="utf-8") as _input_file: expected_result = json.load(_input_file) result = compiler.visit(metadata_v_1_4) diff --git a/tests/test_dialects/test_oep/test_translation.py b/tests/test_dialects/test_oep/test_translation.py index 4a958b6..1448ebd 100644 --- a/tests/test_dialects/test_oep/test_translation.py +++ b/tests/test_dialects/test_oep/test_translation.py @@ -10,6 +10,8 @@ def test_translation_1_3_to_1_4(): parser = JSONParser_1_3() compiler = JSONCompiler() + # omit none values as expected result also does not include None´s + compiler.OMIT_NONE_FIELDS = True with open("tests/data/metadata_v13_minimal.json", "r") as _input_file: input_string = _input_file.read() # Step 1: Parse JSON to internal structure From 8992ff8fc984974f4adbb4fe80528c25ef77c6ad Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Fri, 19 Jan 2024 16:26:32 +0100 Subject: [PATCH 6/7] git commit omit none values in doctest #71 --- docs/oep_metadata.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/oep_metadata.rst b/docs/oep_metadata.rst index 19b926e..05dd42e 100644 --- a/docs/oep_metadata.rst +++ b/docs/oep_metadata.rst @@ -50,5 +50,6 @@ new metadata format by using the respective dialect .. doctest:: >>> dialect1_4 = OEP_V_1_4_Dialect() + >>> dialect1_4._compiler.OMIT_NONE_FIELDS = True >>> dialect1_4.compile(parsed) {'id': 'unique_id', 'metaMetadata': {'metadataVersion': 'OEP-1.4.0', 'metadataLicense': {'name': 'CC0-1.0', 'title': 'Creative Commons Zero v1.0 Universal', 'path': 'https://creativecommons.org/publicdomain/zero/1.0/'}}} From bff4a50acd00a7d34d6dd23d58feb08b29bedd94 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Fri, 19 Jan 2024 16:29:16 +0100 Subject: [PATCH 7/7] update changelog #71 --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 71fe4a0..9deb070 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ Changelog current (2022-XX-XX) -------------------- +* Introduce OMIT_NONE_FIELDS in JSONCompiler class to ease removing / keeping none values in the metadata. By default None values are kept. (#72)[https://github.com/OpenEnergyPlatform/omi/pull/72] 0.1.2 (2023-01-27) --------------------