From ecd3e54b78f830d916d979dfa35fa082c4ed97c4 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Fri, 8 Nov 2024 15:58:08 -0800 Subject: [PATCH 1/7] Fix incorrect return type for PipelinesConfig.get_pipeline_files. --- python/activator/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/activator/config.py b/python/activator/config.py index c2999ae7..c8022d22 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -170,7 +170,7 @@ def _check_pipelines(pipelines: collections.abc.Sequence[str]): if duplicates: raise ValueError(f"Pipeline names must be unique, found multiple copies of {duplicates}.") - def get_pipeline_files(self, visit: FannedOutVisit) -> str: + def get_pipeline_files(self, visit: FannedOutVisit) -> list[str]: """Identify the pipeline to be run, based on the provided visit. Parameters From 1f93558d7b6edb4bdba58165dd9758858a25205b Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Fri, 20 Sep 2024 11:43:59 -0700 Subject: [PATCH 2/7] Remove "not a singleton" disclaimer from PipelinesConfig. A typical run has two simultaneous instances (one for preprocessing, one for the main pipeline), so this disclaimer is redundant. --- python/activator/config.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/python/activator/config.py b/python/activator/config.py index c8022d22..88397f8e 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -51,13 +51,6 @@ class PipelinesConfig: and no two pipelines may share the same name. See examples below. - Notes - ----- - While it is not expected that there will ever be more than one - PipelinesConfig instance in a program's lifetime, this class is *not* a - singleton and objects must be passed explicitly to the code that - needs them. - Examples -------- A single-survey, single-pipeline config: From 0bbe11de5a33e93f52c145a3317c487143d2cfe8 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Fri, 20 Sep 2024 13:48:26 -0700 Subject: [PATCH 3/7] Switch pipelines config format from Python-like to YAML. Delegating parsing to YAML makes it much easier to add more fields to the pipelines spec, and even this basic implementation catches some escaping errors in the original tests. --- python/activator/config.py | 126 ++++++++++++----------- tests/test_config.py | 157 ++++++++++++++++------------- tests/test_middleware_interface.py | 20 ++-- 3 files changed, 168 insertions(+), 135 deletions(-) diff --git a/python/activator/config.py b/python/activator/config.py index 88397f8e..d2cf6e2e 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -26,7 +26,7 @@ import collections import collections.abc import os -import re +import yaml from .visit import FannedOutVisit @@ -42,34 +42,49 @@ class PipelinesConfig: Parameters ---------- config : `str` - A string describing pipeline selection criteria. The current format is - a space-delimited list of mappings, each of which has the format - ``(survey="")=[]``. The zero or more pipelines are - comma-delimited, and each pipeline path may contain environment - variables. The list may be replaced by the keyword "None" to mean no - pipeline should be run. No key or value may contain the "=" character, - and no two pipelines may share the same name. - See examples below. + A YAML-encoded list of mappings, each of which maps ``survey`` to a + survey name and ``pipelines`` to a list of zero or more pipelines. Each + pipeline path may contain environment variables, and the list of + pipelines may be replaced by the keyword "None" to mean no pipeline + should be run. No two pipelines may share the same survey. See examples + below. Examples -------- A single-survey, single-pipeline config: - >>> PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml]') # doctest: +ELLIPSIS + >>> PipelinesConfig(''' + ... - + ... survey: TestSurvey + ... pipelines: [/etc/pipelines/SingleFrame.yaml]''') # doctest: +ELLIPSIS A config with multiple surveys and pipelines, and environment variables: - >>> PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/ApPipe.yaml, /etc/pipelines/ISR.yaml] ' - ... '(survey="Camera Test")=[${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml] ' - ... '(survey="")=[${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml] ') - ... # doctest: +ELLIPSIS + >>> PipelinesConfig(''' + ... - + ... survey: TestSurvey + ... pipelines: [/etc/pipelines/ApPipe.yaml, /etc/pipelines/ISR.yaml] + ... - + ... survey: Camera Test + ... pipelines: + ... - ${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml + ... - + ... survey: "" + ... pipelines: + ... - ${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml + ... ''') # doctest: +ELLIPSIS A config that omits a pipeline for non-sky data: - >>> PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/ApPipe.yaml] ' - ... '(survey="Dome Flats")=None ') # doctest: +ELLIPSIS + >>> PipelinesConfig(''' + ... - + ... survey: TestSurvey + ... pipelines: [/etc/pipelines/ApPipe.yaml] + ... - + ... survey: Dome Flats + ... pipelines: None''') # doctest: +ELLIPSIS """ @@ -77,25 +92,24 @@ def __init__(self, config: str): if not config: raise ValueError("Must configure at least one pipeline.") - self._mapping = self._parse_config(config) + parsed_config = yaml.safe_load(config) + self._mapping = self._expand_config(parsed_config) for pipelines in self._mapping.values(): self._check_pipelines(pipelines) @staticmethod - def _parse_config(config: str) -> collections.abc.Mapping: - """Turn a config string into structured config information. + def _expand_config(config: collections.abc.Sequence) -> collections.abc.Mapping: + """Turn a config spec into structured config information. Parameters ---------- - config : `str` - A string describing pipeline selection criteria. The current format - a space-delimited list of mappings, each of which has the format - ``(survey="")=[]``. The zero or more pipelines - are comma-delimited, and each pipeline path may contain environment - variables. The list may be replaced by the keyword "None" to mean - no pipeline should be run. No key or value may contain the "=" - character. + config : sequence [mapping] + A sequence of mappings, each of which maps ``survey`` to a survey + name and ``pipelines`` to a list of zero or more pipelines. Each + pipeline path may contain environment variables, and the list of + pipelines may be replaced by `None` to mean no pipeline should be + run. No two pipelines may share the same survey. Returns ------- @@ -107,38 +121,36 @@ def _parse_config(config: str) -> collections.abc.Mapping: Raises ------ ValueError - Raised if the string cannot be parsed. + Raised if the input config is invalid. """ - # Use regex instead of str.split, in case keys or values also have - # spaces. - # Allow anything between the [ ] to avoid catastrophic backtracking - # when the input is invalid. If pickier matching is needed in the - # future, use a separate regex for filelist instead of making node - # more complex. - node = re.compile(r'\s*\(survey="(?P[^"\n=]*)"\)=' - r'(?:\[(?P[^]]*)\]|none)(?:\s+|$)', - re.IGNORECASE) - items = {} - pos = 0 - match = node.match(config, pos) - while match: - if match['filelist']: # exclude None and ""; latter gives unexpected behavior with split - filenames = [] - for file in match['filelist'].split(','): - file = file.strip() - if "\n" in file: - raise ValueError(f"Unexpected newline in '{file}'.") - filenames.append(file) - items[match['survey']] = filenames - else: - items[match['survey']] = [] - - pos = match.end() - match = node.match(config, pos) - if pos != len(config): - raise ValueError(f"Unexpected text at position {pos}: '{config[pos:]}'.") - + for node in config: + try: + specs = dict(node) + pipelines_value = specs.pop('pipelines') + if pipelines_value is None or pipelines_value == "None": + filenames = [] + elif isinstance(pipelines_value, str): # Strings are sequences! + raise ValueError(f"Pipelines spec must be list or None, got {pipelines_value}") + elif isinstance(pipelines_value, collections.abc.Sequence): + if any(not isinstance(x, str) for x in pipelines_value): + raise ValueError(f"Pipeline list {pipelines_value} has invalid paths.") + filenames = pipelines_value + else: + raise ValueError(f"Pipelines spec must be list or None, got {pipelines_value}") + + survey_value = specs.pop('survey') + if isinstance(survey_value, str): + survey = survey_value + else: + raise ValueError(f"{survey_value} is not a valid survey name.") + + if specs: + raise ValueError(f"Got unexpected keywords: {specs.keys()}") + except KeyError as e: + raise ValueError from e + + items[survey] = filenames return items @staticmethod diff --git a/tests/test_config.py b/tests/test_config.py index 450c78a6..517ca0c7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -58,18 +58,27 @@ def setUp(self): ) def test_main_survey(self): - config = PipelinesConfig( - ' (survey="TestSurvey")=[${PROMPT_PROCESSING_DIR}/pipelines/NotACam/ApPipe.yaml]') + config = PipelinesConfig(''' + - + survey: TestSurvey + pipelines: + - ${PROMPT_PROCESSING_DIR}/pipelines/NotACam/ApPipe.yaml''') self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join(TESTDIR, "..", "pipelines", "NotACam", "ApPipe.yaml"))] ) def test_selection(self): - config = PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml] ' - '(survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - '(survey="")=[Default.yaml] ' - ) + config = PipelinesConfig(''' + - + survey: TestSurvey + pipelines: [/etc/pipelines/SingleFrame.yaml] + - + survey: CameraTest + pipelines: ["${AP_PIPE_DIR}/pipelines/Isr.yaml"] + - + survey: "" + pipelines: [Default.yaml]''') self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] @@ -83,11 +92,10 @@ def test_selection(self): ["Default.yaml"] ) - def test_multiline(self): - config = PipelinesConfig('''(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml] - (survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] - ''' - ) + def test_singleline(self): + config = PipelinesConfig('[{survey: TestSurvey, pipelines: [/etc/pipelines/SingleFrame.yaml]},' + " {survey: CameraTest, pipelines: ['${AP_PIPE_DIR}/pipelines/Isr.yaml']}" + ']') self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] @@ -98,9 +106,12 @@ def test_multiline(self): ) def test_fallback(self): - config = PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml, ' - ' ${AP_PIPE_DIR}/pipelines/Isr.yaml]' - ) + config = PipelinesConfig(''' + - + survey: TestSurvey + pipelines: + - /etc/pipelines/SingleFrame.yaml + - ${AP_PIPE_DIR}/pipelines/Isr.yaml''') self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml")), @@ -108,9 +119,14 @@ def test_fallback(self): ) def test_space(self): - config = PipelinesConfig('(survey="TestSurvey")=[/dir with space/pipelines/SingleFrame.yaml] ' - '(survey="Camera Test")=[${AP_PIPE_DIR}/pipe lines/Isr.yaml] ' - ) + config = PipelinesConfig(''' + - + survey: TestSurvey + pipelines: [/dir with space/pipelines/SingleFrame.yaml] + - + survey: Camera Test + pipelines: + - ${AP_PIPE_DIR}/pipe lines/Isr.yaml''') self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/dir with space", "pipelines", "SingleFrame.yaml"))] @@ -121,10 +137,16 @@ def test_space(self): ) def test_extrachars(self): - config = PipelinesConfig('(survey="stylish-modern-survey")=[/etc/pipelines/SingleFrame.yaml] ' - '(survey="ScriptParams4,6")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - '(survey="slash/and\backslash")=[Default.yaml] ' - ) + config = PipelinesConfig(r''' + - + survey: stylish-modern-survey + pipelines: [/etc/pipelines/SingleFrame.yaml] + - + survey: ScriptParams4,6 + pipelines: ["${AP_PIPE_DIR}/pipelines/Isr.yaml"] + - + survey: 'slash/and\backslash' + pipelines: [Default.yaml]''') self.assertEqual( config.get_pipeline_files(dataclasses.replace(self.visit, survey="stylish-modern-survey")), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] @@ -134,15 +156,21 @@ def test_extrachars(self): [os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipelines", "Isr.yaml"))] ) self.assertEqual( - config.get_pipeline_files(dataclasses.replace(self.visit, survey="slash/and\backslash")), + config.get_pipeline_files(dataclasses.replace(self.visit, survey=r"slash/and\backslash")), ["Default.yaml"] ) def test_none(self): - config = PipelinesConfig('(survey="TestSurvey")=[None shall pass/pipelines/SingleFrame.yaml] ' - '(survey="Camera Test")=None ' - '(survey="CameraTest")=[] ' - ) + config = PipelinesConfig(''' + - + survey: TestSurvey + pipelines: [None shall pass/pipelines/SingleFrame.yaml] + - + survey: Camera Test + pipelines: None + - + survey: CameraTest + pipelines: []''') self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("None shall pass", "pipelines", "SingleFrame.yaml"))] @@ -153,9 +181,13 @@ def test_none(self): []) def test_nomatch(self): - config = PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml] ' - '(survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - ) + config = PipelinesConfig(''' + - + survey: TestSurvey + pipelines: [/etc/pipelines/SingleFrame.yaml] + - + survey: CameraTest + pipelines: ["${AP_PIPE_DIR}/pipelines/Isr.yaml"]''') with self.assertRaises(RuntimeError): config.get_pipeline_files(dataclasses.replace(self.visit, survey="Surprise")) @@ -165,55 +197,36 @@ def test_empty(self): with self.assertRaises(ValueError): PipelinesConfig(None) - def test_commas(self): + def test_notlist(self): with self.assertRaises(ValueError): - PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml], ' - '(survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - ) - with self.assertRaises(ValueError): - PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml],' - '(survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - ) - - def test_bad_line_breaks(self): - with self.assertRaises(ValueError): - PipelinesConfig('''(survey="Test - Survey")=[/etc/pipelines/SingleFrame.yaml]''' - ) - with self.assertRaises(ValueError): - PipelinesConfig('''(survey="TestSurvey")=[/etc/pipelines/ - SingleFrame.yaml]''' - ) - - def test_unlabeled(self): - with self.assertRaises(ValueError): - PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml], ' - '("CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - ) + PipelinesConfig(''' + - + reason: TestSurvey + pipelines: /etc/pipelines/SingleFrame.yaml''') def test_oddlabel(self): with self.assertRaises(ValueError): - PipelinesConfig('(reason="TestSurvey")=[/etc/pipelines/SingleFrame.yaml]') - - def test_nospace(self): - with self.assertRaises(ValueError): - PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/SingleFrame.yaml]' - '(survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml]' - ) - - def test_noequal(self): - with self.assertRaises(ValueError): - PipelinesConfig('[/etc/pipelines/SingleFrame.yaml]') - + PipelinesConfig(''' + - + reason: TestSurvey + pipelines: [/etc/pipelines/SingleFrame.yaml]''') with self.assertRaises(ValueError): - PipelinesConfig('[/etc/pipelines/SingleFrame.yaml] ' - '(survey="CameraTest")=[${AP_PIPE_DIR}/pipelines/Isr.yaml] ' - ) + PipelinesConfig(''' + - + survey: TestSurvey + comment: This is a fancy survey with simple processing. + pipelines: [/etc/pipelines/SingleFrame.yaml]''') def test_duplicates(self): with self.assertRaises(ValueError): - PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/ApPipe.yaml,' - ' ${AP_PIPE_DIR}/pipelines/ApPipe.yaml]') + PipelinesConfig(''' + - + survey: TestSurvey + pipelines: + - /etc/pipelines/ApPipe.yaml + - ${AP_PIPE_DIR}/pipelines/ApPipe.yaml]''') with self.assertRaises(ValueError): - PipelinesConfig('(survey="TestSurvey")=[/etc/pipelines/ApPipe.yaml,' - ' /etc/pipelines/ApPipe.yaml#isr]') + PipelinesConfig(''' + - + survey: TestSurvey + pipelines: [/etc/pipelines/ApPipe.yaml, /etc/pipelines/ApPipe.yaml#isr]''') diff --git a/tests/test_middleware_interface.py b/tests/test_middleware_interface.py index 22541680..c18e4b83 100644 --- a/tests/test_middleware_interface.py +++ b/tests/test_middleware_interface.py @@ -66,13 +66,21 @@ # A pipelines config that returns the test pipelines. # Unless a test imposes otherwise, the first pipeline should run, and # the second should not be attempted. -pipelines = PipelinesConfig('''(survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/tests/data/ApPipe.yaml, - ${PROMPT_PROCESSING_DIR}/tests/data/SingleFrame.yaml] - ''') -pre_pipelines_empty = PipelinesConfig('(survey="SURVEY")=[]') +pipelines = PipelinesConfig(''' + - + survey: SURVEY + pipelines: + - ${PROMPT_PROCESSING_DIR}/tests/data/ApPipe.yaml + - ${PROMPT_PROCESSING_DIR}/tests/data/SingleFrame.yaml''') +pre_pipelines_empty = PipelinesConfig(''' + - survey: SURVEY + pipelines: []''') pre_pipelines_full = PipelinesConfig( - '''(survey="SURVEY")=[${PROMPT_PROCESSING_DIR}/tests/data/Preprocess.yaml, - ${PROMPT_PROCESSING_DIR}/tests/data/MinPrep.yaml] + '''- + survey: SURVEY + pipelines: + - ${PROMPT_PROCESSING_DIR}/tests/data/Preprocess.yaml + - ${PROMPT_PROCESSING_DIR}/tests/data/MinPrep.yaml ''') From 93f072a7f7ff65a8ffacb187003b6f4ba575bc82 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Tue, 5 Nov 2024 15:19:17 -0800 Subject: [PATCH 4/7] Factor YAML parsing out of PipelinesConfig. Making the activator responsible for YAML input leads to better seperation of concerns. It also makes unit tests slightly better-behaved, because they can be expressed in pure Python. However, this change makes some test cases obsolete, because they specifically test text formatting. --- python/activator/activator.py | 28 +++++- python/activator/config.py | 69 ++++++------- tests/test_config.py | 155 +++++++---------------------- tests/test_middleware_interface.py | 27 ++--- 4 files changed, 102 insertions(+), 177 deletions(-) diff --git a/python/activator/activator.py b/python/activator/activator.py index d1039577..47ec126e 100644 --- a/python/activator/activator.py +++ b/python/activator/activator.py @@ -30,6 +30,7 @@ import sys import time import signal +import yaml import uuid import boto3 @@ -75,15 +76,34 @@ kafka_group_id = str(uuid.uuid4()) # The topic on which to listen to updates to image_bucket bucket_topic = os.environ.get("BUCKET_TOPIC", "rubin-prompt-processing") -# The preprocessing pipelines to execute and the conditions in which to choose them. -pre_pipelines = PipelinesConfig(os.environ["PREPROCESSING_PIPELINES_CONFIG"]) -# The main pipelines to execute and the conditions in which to choose them. -main_pipelines = PipelinesConfig(os.environ["MAIN_PIPELINES_CONFIG"]) _log = logging.getLogger("lsst." + __name__) _log.setLevel(logging.DEBUG) +def _config_from_yaml(yaml_string): + """Initialize a PipelinesConfig from a YAML-formatted string. + + Parameters + ---------- + yaml_string : `str` + A YAML representation of the structured config. See + `~activator.config.PipelineConfig` for details. + + Returns + ------- + config : `activator.config.PipelineConfig` + The corresponding config object. + """ + return PipelinesConfig(yaml.safe_load(yaml_string)) + + +# The preprocessing pipelines to execute and the conditions in which to choose them. +pre_pipelines = _config_from_yaml(os.environ["PREPROCESSING_PIPELINES_CONFIG"]) +# The main pipelines to execute and the conditions in which to choose them. +main_pipelines = _config_from_yaml(os.environ["MAIN_PIPELINES_CONFIG"]) + + def find_local_repos(base_path): """Search for existing local repos. diff --git a/python/activator/config.py b/python/activator/config.py index d2cf6e2e..c154bcdf 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -26,7 +26,6 @@ import collections import collections.abc import os -import yaml from .visit import FannedOutVisit @@ -41,59 +40,53 @@ class PipelinesConfig: Parameters ---------- - config : `str` - A YAML-encoded list of mappings, each of which maps ``survey`` to a - survey name and ``pipelines`` to a list of zero or more pipelines. Each - pipeline path may contain environment variables, and the list of - pipelines may be replaced by the keyword "None" to mean no pipeline - should be run. No two pipelines may share the same survey. See examples - below. + config : sequence [mapping] + A sequence of mappings ("nodes"), each with the following keys: + + ``"survey"`` + The survey that triggers the pipelines (`str`). No two nodes may + share the same survey. + ``"pipelines"`` + A list of zero or more pipelines (sequence [`str`] or `None`). Each + pipeline path may contain environment variables, and the list of + pipelines may be replaced by `None` to mean no pipeline should + be run. Examples -------- A single-survey, single-pipeline config: - >>> PipelinesConfig(''' - ... - - ... survey: TestSurvey - ... pipelines: [/etc/pipelines/SingleFrame.yaml]''') # doctest: +ELLIPSIS + >>> PipelinesConfig([{"survey": "TestSurvey", + ... "pipelines": ["/etc/pipelines/SingleFrame.yaml"]}, + ... ]) # doctest: +ELLIPSIS A config with multiple surveys and pipelines, and environment variables: - >>> PipelinesConfig(''' - ... - - ... survey: TestSurvey - ... pipelines: [/etc/pipelines/ApPipe.yaml, /etc/pipelines/ISR.yaml] - ... - - ... survey: Camera Test - ... pipelines: - ... - ${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml - ... - - ... survey: "" - ... pipelines: - ... - ${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml - ... ''') # doctest: +ELLIPSIS + >>> PipelinesConfig([{"survey": "TestSurvey", + ... "pipelines": ["/etc/pipelines/ApPipe.yaml", "/etc/pipelines/ISR.yaml"]}, + ... {"survey": "Camera Test", + ... "pipelines": ["${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml"]}, + ... {"survey": "", + ... "pipelines": ["${AP_PIPE_DIR}/pipelines/LSSTComCam/Isr.yaml"]}, + ... ]) # doctest: +ELLIPSIS A config that omits a pipeline for non-sky data: - >>> PipelinesConfig(''' - ... - - ... survey: TestSurvey - ... pipelines: [/etc/pipelines/ApPipe.yaml] - ... - - ... survey: Dome Flats - ... pipelines: None''') # doctest: +ELLIPSIS + >>> PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/ApPipe.yaml"]}, + {"survey": "Dome Flats", + "pipelines": None}, + ... ]) # doctest: +ELLIPSIS """ - def __init__(self, config: str): + def __init__(self, config: collections.abc.Sequence): if not config: raise ValueError("Must configure at least one pipeline.") - parsed_config = yaml.safe_load(config) - self._mapping = self._expand_config(parsed_config) + self._mapping = self._expand_config(config) for pipelines in self._mapping.values(): self._check_pipelines(pipelines) @@ -105,11 +98,7 @@ def _expand_config(config: collections.abc.Sequence) -> collections.abc.Mapping: Parameters ---------- config : sequence [mapping] - A sequence of mappings, each of which maps ``survey`` to a survey - name and ``pipelines`` to a list of zero or more pipelines. Each - pipeline path may contain environment variables, and the list of - pipelines may be replaced by `None` to mean no pipeline should be - run. No two pipelines may share the same survey. + A sequence of mappings, see class docs for details. Returns ------- diff --git a/tests/test_config.py b/tests/test_config.py index 517ca0c7..477975a1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -58,27 +58,23 @@ def setUp(self): ) def test_main_survey(self): - config = PipelinesConfig(''' - - - survey: TestSurvey - pipelines: - - ${PROMPT_PROCESSING_DIR}/pipelines/NotACam/ApPipe.yaml''') + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["${PROMPT_PROCESSING_DIR}/pipelines/NotACam/ApPipe.yaml"], + }]) self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join(TESTDIR, "..", "pipelines", "NotACam", "ApPipe.yaml"))] ) def test_selection(self): - config = PipelinesConfig(''' - - - survey: TestSurvey - pipelines: [/etc/pipelines/SingleFrame.yaml] - - - survey: CameraTest - pipelines: ["${AP_PIPE_DIR}/pipelines/Isr.yaml"] - - - survey: "" - pipelines: [Default.yaml]''') + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/SingleFrame.yaml"], + }, + {"survey": "CameraTest", + "pipelines": ["${AP_PIPE_DIR}/pipelines/Isr.yaml"], + }, + {"survey": "", "pipelines": ["Default.yaml"]}, + ]) self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] @@ -92,85 +88,23 @@ def test_selection(self): ["Default.yaml"] ) - def test_singleline(self): - config = PipelinesConfig('[{survey: TestSurvey, pipelines: [/etc/pipelines/SingleFrame.yaml]},' - " {survey: CameraTest, pipelines: ['${AP_PIPE_DIR}/pipelines/Isr.yaml']}" - ']') - self.assertEqual( - config.get_pipeline_files(self.visit), - [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] - ) - self.assertEqual( - config.get_pipeline_files(dataclasses.replace(self.visit, survey="CameraTest")), - [os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipelines", "Isr.yaml"))] - ) - def test_fallback(self): - config = PipelinesConfig(''' - - - survey: TestSurvey - pipelines: - - /etc/pipelines/SingleFrame.yaml - - ${AP_PIPE_DIR}/pipelines/Isr.yaml''') + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/SingleFrame.yaml", + "${AP_PIPE_DIR}/pipelines/Isr.yaml", + ]}]) self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml")), os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipelines", "Isr.yaml"))] ) - def test_space(self): - config = PipelinesConfig(''' - - - survey: TestSurvey - pipelines: [/dir with space/pipelines/SingleFrame.yaml] - - - survey: Camera Test - pipelines: - - ${AP_PIPE_DIR}/pipe lines/Isr.yaml''') - self.assertEqual( - config.get_pipeline_files(self.visit), - [os.path.normpath(os.path.join("/dir with space", "pipelines", "SingleFrame.yaml"))] - ) - self.assertEqual( - config.get_pipeline_files(dataclasses.replace(self.visit, survey="Camera Test")), - [os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipe lines", "Isr.yaml"))] - ) - - def test_extrachars(self): - config = PipelinesConfig(r''' - - - survey: stylish-modern-survey - pipelines: [/etc/pipelines/SingleFrame.yaml] - - - survey: ScriptParams4,6 - pipelines: ["${AP_PIPE_DIR}/pipelines/Isr.yaml"] - - - survey: 'slash/and\backslash' - pipelines: [Default.yaml]''') - self.assertEqual( - config.get_pipeline_files(dataclasses.replace(self.visit, survey="stylish-modern-survey")), - [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] - ) - self.assertEqual( - config.get_pipeline_files(dataclasses.replace(self.visit, survey="ScriptParams4,6")), - [os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipelines", "Isr.yaml"))] - ) - self.assertEqual( - config.get_pipeline_files(dataclasses.replace(self.visit, survey=r"slash/and\backslash")), - ["Default.yaml"] - ) - def test_none(self): - config = PipelinesConfig(''' - - - survey: TestSurvey - pipelines: [None shall pass/pipelines/SingleFrame.yaml] - - - survey: Camera Test - pipelines: None - - - survey: CameraTest - pipelines: []''') + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["None shall pass/pipelines/SingleFrame.yaml"]}, + {"survey": "Camera Test", "pipelines": None}, + {"survey": "CameraTest", "pipelines": []}, + ]) self.assertEqual( config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("None shall pass", "pipelines", "SingleFrame.yaml"))] @@ -181,52 +115,39 @@ def test_none(self): []) def test_nomatch(self): - config = PipelinesConfig(''' - - - survey: TestSurvey - pipelines: [/etc/pipelines/SingleFrame.yaml] - - - survey: CameraTest - pipelines: ["${AP_PIPE_DIR}/pipelines/Isr.yaml"]''') + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/SingleFrame.yaml"], + }, + {"survey": "CameraTest", + "pipelines": ["${AP_PIPE_DIR}/pipelines/Isr.yaml"], + }, + ]) with self.assertRaises(RuntimeError): config.get_pipeline_files(dataclasses.replace(self.visit, survey="Surprise")) def test_empty(self): with self.assertRaises(ValueError): - PipelinesConfig('') + PipelinesConfig([]) with self.assertRaises(ValueError): PipelinesConfig(None) def test_notlist(self): with self.assertRaises(ValueError): - PipelinesConfig(''' - - - reason: TestSurvey - pipelines: /etc/pipelines/SingleFrame.yaml''') + PipelinesConfig([{"survey": "TestSurvey", "pipelines": "/etc/pipelines/SingleFrame.yaml"}]) def test_oddlabel(self): with self.assertRaises(ValueError): - PipelinesConfig(''' - - - reason: TestSurvey - pipelines: [/etc/pipelines/SingleFrame.yaml]''') + PipelinesConfig([{"reason": "TestSurvey", "pipelines": ["/etc/pipelines/SingleFrame.yaml"]}]) with self.assertRaises(ValueError): - PipelinesConfig(''' - - - survey: TestSurvey - comment: This is a fancy survey with simple processing. - pipelines: [/etc/pipelines/SingleFrame.yaml]''') + PipelinesConfig([{"survey": "TestSurvey", + "comment": "This is a fancy survey with simple processing.", + "pipelines": ["/etc/pipelines/SingleFrame.yaml"]}]) def test_duplicates(self): with self.assertRaises(ValueError): - PipelinesConfig(''' - - - survey: TestSurvey - pipelines: - - /etc/pipelines/ApPipe.yaml - - ${AP_PIPE_DIR}/pipelines/ApPipe.yaml]''') + PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/ApPipe.yaml", + "${AP_PIPE_DIR}/pipelines/ApPipe.yaml"]}]) with self.assertRaises(ValueError): - PipelinesConfig(''' - - - survey: TestSurvey - pipelines: [/etc/pipelines/ApPipe.yaml, /etc/pipelines/ApPipe.yaml#isr]''') + PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/ApPipe.yaml", "/etc/pipelines/ApPipe.yaml#isr"]}]) diff --git a/tests/test_middleware_interface.py b/tests/test_middleware_interface.py index c18e4b83..424f59e9 100644 --- a/tests/test_middleware_interface.py +++ b/tests/test_middleware_interface.py @@ -66,22 +66,17 @@ # A pipelines config that returns the test pipelines. # Unless a test imposes otherwise, the first pipeline should run, and # the second should not be attempted. -pipelines = PipelinesConfig(''' - - - survey: SURVEY - pipelines: - - ${PROMPT_PROCESSING_DIR}/tests/data/ApPipe.yaml - - ${PROMPT_PROCESSING_DIR}/tests/data/SingleFrame.yaml''') -pre_pipelines_empty = PipelinesConfig(''' - - survey: SURVEY - pipelines: []''') -pre_pipelines_full = PipelinesConfig( - '''- - survey: SURVEY - pipelines: - - ${PROMPT_PROCESSING_DIR}/tests/data/Preprocess.yaml - - ${PROMPT_PROCESSING_DIR}/tests/data/MinPrep.yaml - ''') +pipelines = PipelinesConfig([{"survey": "SURVEY", + "pipelines": ["${PROMPT_PROCESSING_DIR}/tests/data/ApPipe.yaml", + "${PROMPT_PROCESSING_DIR}/tests/data/SingleFrame.yaml", + ], + }]) +pre_pipelines_empty = PipelinesConfig([{"survey": "SURVEY", "pipelines": None}]) +pre_pipelines_full = PipelinesConfig([{"survey": "SURVEY", + "pipelines": ["${PROMPT_PROCESSING_DIR}/tests/data/Preprocess.yaml", + "${PROMPT_PROCESSING_DIR}/tests/data/MinPrep.yaml", + ], + }]) def fake_file_data(filename, dimensions, instrument, visit): From 7e8698bf154ec4bd89d79af690293abc36a1b79a Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Fri, 8 Nov 2024 16:37:53 -0800 Subject: [PATCH 5/7] Allow ordered evaluation of pipeline configs. Previously, pipeline configs behaved as a mapping from survey to pipeline list; this design didn't support more complex specs (wildcards, spatial constraints, etc.). Taking the first block that "matches" a visit allows for more free-form specification of when pipelines should be run. --- python/activator/config.py | 117 ++++++++++++++++++++++++------------- tests/test_config.py | 14 +++++ 2 files changed, 91 insertions(+), 40 deletions(-) diff --git a/python/activator/config.py b/python/activator/config.py index c154bcdf..f8b706da 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -26,6 +26,7 @@ import collections import collections.abc import os +import typing from .visit import FannedOutVisit @@ -44,14 +45,16 @@ class PipelinesConfig: A sequence of mappings ("nodes"), each with the following keys: ``"survey"`` - The survey that triggers the pipelines (`str`). No two nodes may - share the same survey. + The survey that triggers the pipelines (`str`). ``"pipelines"`` A list of zero or more pipelines (sequence [`str`] or `None`). Each pipeline path may contain environment variables, and the list of pipelines may be replaced by `None` to mean no pipeline should be run. + Nodes are arranged by precedence, i.e., the first node that matches a + visit is used. + Examples -------- A single-survey, single-pipeline config: @@ -82,55 +85,35 @@ class PipelinesConfig: """ - def __init__(self, config: collections.abc.Sequence): - if not config: - raise ValueError("Must configure at least one pipeline.") - - self._mapping = self._expand_config(config) - - for pipelines in self._mapping.values(): - self._check_pipelines(pipelines) - - @staticmethod - def _expand_config(config: collections.abc.Sequence) -> collections.abc.Mapping: - """Turn a config spec into structured config information. + class _Spec: + """A single case of which pipelines should be run in particular + circumstances. Parameters ---------- - config : sequence [mapping] - A sequence of mappings, see class docs for details. - - Returns - ------- - config : mapping [`str`, `list` [`str`]] - A mapping from the survey type to the pipeline(s) to run for that - survey. A more complex key or container type may be needed in the - future, if other pipeline selection criteria are added. - - Raises - ------ - ValueError - Raised if the input config is invalid. + config : mapping [`str`] + A config node with the same keys as documented for the + `PipelinesConfig` constructor. """ - items = {} - for node in config: + + def __init__(self, config: collections.abc.Mapping[str, typing.Any]): + specs = dict(config) try: - specs = dict(node) pipelines_value = specs.pop('pipelines') if pipelines_value is None or pipelines_value == "None": - filenames = [] + self._filenames = [] elif isinstance(pipelines_value, str): # Strings are sequences! raise ValueError(f"Pipelines spec must be list or None, got {pipelines_value}") elif isinstance(pipelines_value, collections.abc.Sequence): if any(not isinstance(x, str) for x in pipelines_value): raise ValueError(f"Pipeline list {pipelines_value} has invalid paths.") - filenames = pipelines_value + self._filenames = pipelines_value else: raise ValueError(f"Pipelines spec must be list or None, got {pipelines_value}") survey_value = specs.pop('survey') if isinstance(survey_value, str): - survey = survey_value + self._survey = survey_value else: raise ValueError(f"{survey_value} is not a valid survey name.") @@ -139,7 +122,59 @@ def _expand_config(config: collections.abc.Sequence) -> collections.abc.Mapping: except KeyError as e: raise ValueError from e - items[survey] = filenames + def matches(self, visit: FannedOutVisit) -> bool: + """Test whether a visit matches the conditions for this spec. + + Parameters + ---------- + visit : `activator.visit.FannedOutVisit` + The visit to test against this spec. + + Returns + ------- + matches : `bool` + `True` if the visit meets all conditions, `False` otherwise. + """ + return self._survey == visit.survey + + @property + def pipeline_files(self) -> collections.abc.Sequence[str]: + """An ordered list of pipelines to run in this spec (sequence [`str`]). + """ + return self._filenames + + def __init__(self, config: collections.abc.Sequence): + if not config: + raise ValueError("Must configure at least one pipeline.") + + self._specs = self._expand_config(config) + + for spec in self._specs: + self._check_pipelines(spec.pipeline_files) + + @staticmethod + def _expand_config(config: collections.abc.Sequence) -> collections.abc.Sequence[_Spec]: + """Turn a config spec into structured config information. + + Parameters + ---------- + config : sequence [mapping] + A sequence of mappings, see class docs for details. + + Returns + ------- + config : sequence [`PipelinesConfig._Spec`] + A sequence of node objects specifying the pipeline(s) to run and the + conditions in which to run them. + + Raises + ------ + ValueError + Raised if the input config is invalid. + """ + items = [] + for node in config: + items.append(PipelinesConfig._Spec(node)) return items @staticmethod @@ -167,6 +202,9 @@ def _check_pipelines(pipelines: collections.abc.Sequence[str]): def get_pipeline_files(self, visit: FannedOutVisit) -> list[str]: """Identify the pipeline to be run, based on the provided visit. + The first node that matches the visit is returned, and no other nodes + are considered even if they would provide a "tighter" match. + Parameters ---------- visit : `activator.visit.FannedOutVisit` @@ -178,8 +216,7 @@ def get_pipeline_files(self, visit: FannedOutVisit) -> list[str]: Path(s) to the configured pipeline file(s). An empty list means that *no* pipeline should be run on this visit. """ - try: - values = self._mapping[visit.survey] - except KeyError as e: - raise RuntimeError(f"Unsupported survey: {visit.survey}") from e - return [os.path.expandvars(path) for path in values] + for node in self._specs: + if node.matches(visit): + return [os.path.expandvars(path) for path in node.pipeline_files] + raise RuntimeError(f"Unsupported survey: {visit.survey}") diff --git a/tests/test_config.py b/tests/test_config.py index 477975a1..72e650d5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -151,3 +151,17 @@ def test_duplicates(self): with self.assertRaises(ValueError): PipelinesConfig([{"survey": "TestSurvey", "pipelines": ["/etc/pipelines/ApPipe.yaml", "/etc/pipelines/ApPipe.yaml#isr"]}]) + + def test_order(self): + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/SingleFrame.yaml"], + }, + {"survey": "TestSurvey", + "pipelines": ["${AP_PIPE_DIR}/pipelines/Isr.yaml"], + }, + ]) + # Second TestSurvey spec should be ignored + self.assertEqual( + config.get_pipeline_files(self.visit), + [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] + ) From e527b2c2472f24a8202b39d083d972686feb91ca Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Fri, 8 Nov 2024 16:43:54 -0800 Subject: [PATCH 6/7] Move pipelines checks from PipelinesConfig to _Spec. This change allows for cleaner separation of concerns, as all input handling is now done by _Spec. --- python/activator/config.py | 48 ++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/python/activator/config.py b/python/activator/config.py index f8b706da..0eb17e73 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -110,6 +110,7 @@ def __init__(self, config: collections.abc.Mapping[str, typing.Any]): self._filenames = pipelines_value else: raise ValueError(f"Pipelines spec must be list or None, got {pipelines_value}") + self._check_pipelines(self._filenames) survey_value = specs.pop('survey') if isinstance(survey_value, str): @@ -122,6 +123,28 @@ def __init__(self, config: collections.abc.Mapping[str, typing.Any]): except KeyError as e: raise ValueError from e + @staticmethod + def _check_pipelines(pipelines: collections.abc.Sequence[str]): + """Test the correctness of a list of pipelines. + + At present, the only test is that no two pipelines have the same + filename, which is used as a pipeline ID elsewhere in the Prompt + Processing service. + + Parameters + ---------- + pipelines : sequence [`str`] + + Raises + ------ + ValueError + Raised if the pipeline list is invalid. + """ + filenames = collections.Counter(os.path.splitext(os.path.basename(path))[0] for path in pipelines) + duplicates = [filename for filename, num in filenames.items() if num > 1] + if duplicates: + raise ValueError(f"Pipeline names must be unique, found multiple copies of {duplicates}.") + def matches(self, visit: FannedOutVisit) -> bool: """Test whether a visit matches the conditions for this spec. @@ -149,9 +172,6 @@ def __init__(self, config: collections.abc.Sequence): self._specs = self._expand_config(config) - for spec in self._specs: - self._check_pipelines(spec.pipeline_files) - @staticmethod def _expand_config(config: collections.abc.Sequence) -> collections.abc.Sequence[_Spec]: """Turn a config spec into structured config information. @@ -177,28 +197,6 @@ def _expand_config(config: collections.abc.Sequence) -> collections.abc.Sequence items.append(PipelinesConfig._Spec(node)) return items - @staticmethod - def _check_pipelines(pipelines: collections.abc.Sequence[str]): - """Test the correctness of a list of pipelines. - - At present, the only test is that no two pipelines have the same - filename, which is used as a pipeline ID elsewhere in the Prompt - Processing service. - - Parameters - ---------- - pipelines : sequence [`str`] - - Raises - ------ - ValueError - Raised if the pipeline list is invalid. - """ - filenames = collections.Counter(os.path.splitext(os.path.basename(path))[0] for path in pipelines) - duplicates = [filename for filename, num in filenames.items() if num > 1] - if duplicates: - raise ValueError(f"Pipeline names must be unique, found multiple copies of {duplicates}.") - def get_pipeline_files(self, visit: FannedOutVisit) -> list[str]: """Identify the pipeline to be run, based on the provided visit. From 46a0f27d1b431f2b0e823a4134feb51633649268 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Fri, 8 Nov 2024 17:27:30 -0800 Subject: [PATCH 7/7] Make survey optional in PipelinesConfig. No longer requiring a survey makes it possible to naturally add in other optional constraints, such as position. --- python/activator/config.py | 13 ++++++++----- tests/test_config.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/python/activator/config.py b/python/activator/config.py index 0eb17e73..f39ec44f 100644 --- a/python/activator/config.py +++ b/python/activator/config.py @@ -45,7 +45,7 @@ class PipelinesConfig: A sequence of mappings ("nodes"), each with the following keys: ``"survey"`` - The survey that triggers the pipelines (`str`). + The survey that triggers the pipelines (`str`, optional). ``"pipelines"`` A list of zero or more pipelines (sequence [`str`] or `None`). Each pipeline path may contain environment variables, and the list of @@ -53,7 +53,8 @@ class PipelinesConfig: be run. Nodes are arranged by precedence, i.e., the first node that matches a - visit is used. + visit is used. A node containing only ``pipelines`` is unconstrained + and matches *any* visit. Examples -------- @@ -112,8 +113,8 @@ def __init__(self, config: collections.abc.Mapping[str, typing.Any]): raise ValueError(f"Pipelines spec must be list or None, got {pipelines_value}") self._check_pipelines(self._filenames) - survey_value = specs.pop('survey') - if isinstance(survey_value, str): + survey_value = specs.pop('survey', None) + if isinstance(survey_value, str) or survey_value is None: self._survey = survey_value else: raise ValueError(f"{survey_value} is not a valid survey name.") @@ -158,7 +159,9 @@ def matches(self, visit: FannedOutVisit) -> bool: matches : `bool` `True` if the visit meets all conditions, `False` otherwise. """ - return self._survey == visit.survey + if self._survey is not None and self._survey != visit.survey: + return False + return True @property def pipeline_files(self) -> collections.abc.Sequence[str]: diff --git a/tests/test_config.py b/tests/test_config.py index 72e650d5..c831ea13 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -152,6 +152,12 @@ def test_duplicates(self): PipelinesConfig([{"survey": "TestSurvey", "pipelines": ["/etc/pipelines/ApPipe.yaml", "/etc/pipelines/ApPipe.yaml#isr"]}]) + def test_needpipeline(self): + with self.assertRaises(ValueError): + PipelinesConfig([{"survey": "TestSurvey"}]) + with self.assertRaises(ValueError): + PipelinesConfig([{}]) + def test_order(self): config = PipelinesConfig([{"survey": "TestSurvey", "pipelines": ["/etc/pipelines/SingleFrame.yaml"], @@ -165,3 +171,25 @@ def test_order(self): config.get_pipeline_files(self.visit), [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] ) + + def test_unconstrained(self): + config = PipelinesConfig([{"survey": "TestSurvey", + "pipelines": ["/etc/pipelines/SingleFrame.yaml"], + }, + {"pipelines": ["${AP_PIPE_DIR}/pipelines/Isr.yaml"]}, + {"survey": "", "pipelines": ["Default.yaml"]}, + ]) + self.assertEqual( + config.get_pipeline_files(self.visit), + [os.path.normpath(os.path.join("/etc", "pipelines", "SingleFrame.yaml"))] + ) + self.assertEqual( + # Matches the second node, which has no constraints + config.get_pipeline_files(dataclasses.replace(self.visit, survey="CameraTest")), + [os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipelines", "Isr.yaml"))] + ) + self.assertEqual( + # Matches the second node, which has no constraints + config.get_pipeline_files(dataclasses.replace(self.visit, survey="")), + [os.path.normpath(os.path.join(getPackageDir("ap_pipe"), "pipelines", "Isr.yaml"))] + )