IOSvc: Minor fixes for improving usage and errors (#213)

k4FWCore/components/IIOSvc.h

@@ -28,7 +28,7 @@
 #include <vector>
 
 /**
- * The interface implemented by any class making IO and reading RawEvent Data
+ * The interface implemented by any class making IO with functional algorithms
  */
 class IIOSvc : virtual public IInterface {
 public:

k4FWCore/components/IOSvc.cpp

@@ -38,7 +38,19 @@ StatusCode IOSvc::initialize() {
     error() << "Unable to initialize base class Service." << endmsg;
     return sc;
   }
+
+  if (!m_importedFromk4FWCore) {
+    error() << "Use 'from k4FWCore import IOSvc' instead of 'from Configurables import IOSvc' to access the service"
+            << endmsg;
+    return StatusCode::FAILURE;
+  }
+
+  if (!m_readingFileNamesDeprecated.empty()) {
+    warning() << ".input is deprecated, use .Input instead in the steering file" << endmsg;
+    m_readingFileNames = m_readingFileNamesDeprecated;
+  }
   if (!m_readingFileNames.empty()) {
+    info() << m_readingFileNames.size() << " files to be read" << endmsg;
     m_reader = std::make_unique<podio::ROOTReader>();
     try {
       m_reader->openFiles(m_readingFileNames);
@@ -49,6 +61,11 @@ StatusCode IOSvc::initialize() {
     m_entries = m_reader->getEntries(podio::Category::Event);
   }
 
+  if (!m_writingFileNameDeprecated.empty()) {
+    warning() << ".output is deprecated, use .Output instead in the steering file" << endmsg;
+    m_writingFileName = m_writingFileNameDeprecated;
+  }
+
   m_switch = KeepDropSwitch(m_outputCommands);
 
   m_incidentSvc = service("IncidentSvc");

k4FWCore/components/IOSvc.h

@@ -56,11 +56,17 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 protected:
   Gaudi::Property<std::vector<std::string>> m_collectionNames{
       this, "CollectionNames", {}, "List of collections to read"};
-  Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "input", {}, "List of files to read"};
-  Gaudi::Property<std::string>              m_writingFileName{this, "output", {}, "List of files to write output to"};
+  Gaudi::Property<std::vector<std::string>> m_readingFileNamesDeprecated{this, "input", {}, "List of files to read"};
+  Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "Input", {}, "List of files to read"};
+  Gaudi::Property<std::string> m_writingFileNameDeprecated{this, "output", {}, "List of files to write output to"};
+  Gaudi::Property<std::string> m_writingFileName{this, "Output", {}, "List of files to write output to"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
-  Gaudi::Property<std::string> m_inputType{this, "ioType", "ROOT", "Type of input file (ROOT, RNTuple)"};
+  Gaudi::Property<std::string> m_inputType{this, "IOType", "ROOT", "Type of input file (ROOT, RNTuple)"};
+
+  Gaudi::Property<bool> m_importedFromk4FWCore{
+      this, "ImportedFromk4FWCore", false,
+      "This is set to true when IOSvc is imported from k4FWCore instead of Configurables in python"};
 
   std::mutex m_changeBufferLock;
 

k4FWCore/python/k4FWCore/utils.py

@@ -18,10 +18,43 @@
 # limitations under the License.
 #
 import os
+import re
 from io import TextIOWrapper
 from typing import Union
 
 
+def check_wrong_imports(code: str) -> None:
+    """Check for wrong imports in the given code.
+
+    This function checks the given code for any imports of IOSvc or ApplicationMgr
+    from Configurables instead of k4FWCore. If such an import is found, an ImportError
+    is raised.
+
+    If IOSvc and ApplicationMgr are not imported from k4FWCore it's possible
+    that k4run will fail, for example, by not adding the reader and writer
+    algorithms manually, which is done by the wrapper in k4FWCore. The checks
+    are not comprehensive, it's still possible to import IOSvc or ApplicationMgr
+    from Configurables.
+
+    Args:
+        code (str): The code to check for wrong imports.
+
+    Raises:
+        ImportError: If the code contains an import of IOSvc or ApplicationMgr from Configurables.
+
+    """
+    # Check first that IOSvc is being used, in that case
+    # Importing either ApplicationMgr or IOSvc from Configurables is not allowed
+    iosvc_regex = re.compile(
+        r"^\s*from\s+(Configurables|k4FWCore)\s+import\s+\(?.*IOSvc.*\)?", re.MULTILINE
+    )
+    regex = re.compile(
+        r"^\s*from\s+Configurables\s+import\s+\(?.*(ApplicationMgr|IOSvc).*\)?", re.MULTILINE
+    )
+    if re.search(iosvc_regex, code) and re.search(regex, code):
+        raise ImportError("Importing ApplicationMgr or IOSvc from Configurables is not allowed.")
+
+
 def load_file(opt_file: Union[TextIOWrapper, str, os.PathLike]) -> None:
     """Loads and executes the content of a given file in the current interpreter session.
 
@@ -44,8 +77,9 @@ def load_file(opt_file: Union[TextIOWrapper, str, os.PathLike]) -> None:
     """
     if isinstance(opt_file, (str, os.PathLike)):
         with open(opt_file, "r") as file:
-            code = compile(file.read(), file.name, "exec")
+            code = file.read()
     else:
-        code = compile(opt_file.read(), opt_file.name, "exec")
+        code = opt_file.read()
+    check_wrong_imports(str(code))
 
     exec(code, globals())

python/k4FWCore/ApplicationMgr.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 from Configurables import ApplicationMgr as AppMgr
-from Configurables import Reader, Writer, IOSvc, MetadataSvc, Gaudi__Sequencer
+from Configurables import Reader, Writer, IOSvc, MetadataSvc, Gaudi__Sequencer, EventLoopMgr
 import os
 from podio.root_io import Reader as PodioReader
 
@@ -37,6 +37,12 @@ class ApplicationMgr:
 
     def __init__(self, **kwargs):
         self._mgr = AppMgr(**kwargs)
+        # If there isn't an EventLoopMgr then it's the default
+        # This will suppress two warnings about not using external input
+        try:
+            self._mgr.EventLoop
+        except AttributeError:
+            self._mgr.EventLoop = EventLoopMgr(Warnings=False)
 
         for conf in frozenset(self._mgr.allConfigurables.values()):
             if isinstance(conf, MetadataSvc):
@@ -45,18 +51,24 @@ def __init__(self, **kwargs):
                 continue
             props = conf.getPropertiesWithDescription()
             reader = writer = None
-            add_reader = add_writer = False
+            add_reader = False
             for alg in self._mgr.TopAlg:
                 if isinstance(alg, Reader):
                     reader = alg
                 elif isinstance(alg, Writer):
                     writer = alg
-            if reader is None and props["input"][0]:
+            # Remove "input" when the corresponding property is removed from IOSvc
+            if reader is None and (props["input"][0] or props["Input"][0]):
                 reader = Reader("k4FWCore__Reader")
                 add_reader = True
             # It seems for a single string the default without a value is '<no value>'
             # while for a list it's an empty list
-            if writer is None and props["output"][0] and props["output"][0] != "<no value>":
+            # Remove "output" when the corresponding property is removed from IOSvc
+            if (
+                writer is None
+                and (props["output"][0] and props["output"][0] != "<no value>")
+                or (props["Output"][0] and props["Output"][0] != "<no value>")
+            ):
                 writer = Writer("k4FWCore__Writer")
             # Let's tell the Reader one of the input files so it can
             # know which collections it's going to read
@@ -67,15 +79,24 @@ def __init__(self, **kwargs):
                 # (possibly) the first 9 complete and 9 more are scheduled, out of
                 # which only one will be finished without errors. If we know the
                 # number of events in advance then we can just schedule those.
+                inp = None
                 if props["input"][0]:
-                    if os.path.exists(props["input"][0][0]):
-                        path = props["input"][0][0]
+                    inp = "input"
+                elif props["Input"][0]:
+                    inp = "Input"
+                if inp:
+                    if os.path.exists(props[inp][0][0]):
+                        path = props[inp][0][0]
                     else:
-                        path = os.getcwd() + "/" + props["input"][0][0]
+                        path = os.getcwd() + "/" + props[inp][0][0]
                     podio_reader = PodioReader(path)
                     if self._mgr.EvtMax == -1:
                         self._mgr.EvtMax = podio_reader._reader.getEntries("events")
-                    frame = podio_reader.get("events")[0]
+                    try:
+                        frame = podio_reader.get("events")[0]
+                    except IndexError:
+                        print("Warning, the events category wasn't found in the input file")
+                        raise
                     collections = list(frame.getAvailableCollections())
                     reader.InputCollections = collections
             self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg
@@ -99,3 +120,12 @@ def __init__(self, **kwargs):
                         ],
                     )
                 ]
+
+    def __getattr__(self, name):
+        return getattr(self._mgr, name)
+
+    def __setattr__(self, name, value):
+        if name == "_mgr":
+            super().__setattr__(name, value)
+        else:
+            setattr(self._mgr, name, value)

python/k4FWCore/IOSvc.py

@@ -22,7 +22,7 @@
 
 class IOSvc:
     def __init__(self, *args, **kwargs):
-        self._svc = IO(**kwargs)
+        self._svc = IO(**kwargs, ImportedFromk4FWCore=True)
         MetadataSvc("MetadataSvc")
 
     def __getattr__(self, attr):
@@ -34,10 +34,10 @@ def __setattr__(self, attr, value):
             return
 
         # Allow to specify a single string for input when what we want is a list
-        if attr == "input":
+        if attr == "input" or attr == "Input":
             if isinstance(value, str):
                 value = [value]
-        if attr == "output":
+        elif attr == "output" or attr == "Output":
             if os.path.dirname(value) and not os.path.exists(os.path.dirname(value)):
                 os.makedirs(os.path.dirname(value))
         setattr(self._svc, attr, value)

test/k4FWCoreTest/CMakeLists.txt

@@ -138,6 +138,9 @@ add_test_with_env(FunctionalFilterFile options/ExampleFunctionalFilterFile.py)
 add_test_with_env(FunctionalMetadata options/ExampleFunctionalMetadata.py)
 add_test_with_env(FunctionalMetadataOldAlgorithm options/ExampleFunctionalMetadataOldAlgorithm.py)
 
+add_test_with_env(FunctionalWrongImport options/ExampleFunctionalWrongImport.py)
+set_tests_properties(FunctionalWrongImport PROPERTIES PASS_REGULAR_EXPRESSION "ImportError: Importing ApplicationMgr or IOSvc from Configurables is not allowed.")
+
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
 set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc;FunctionalTransformerHist;FunctionalCollectionMerger;FunctionalFilterFile;FunctionalMetadata;FunctionalMetadataOldAlgorithm")
 

test/k4FWCoreTest/options/ExampleFunctionalCollectionMerger.py

@@ -28,8 +28,8 @@
 from Configurables import ExampleFunctionalProducer
 
 svc = IOSvc("IOSvc")
-svc.input = "functional_producer_multiple.root"
-svc.output = "functional_merged_collections.root"
+svc.Input = "functional_producer_multiple.root"
+svc.Output = "functional_merged_collections.root"
 svc.outputCommands = [
     "drop *",
     "keep MCParticles1",

test/k4FWCoreTest/options/ExampleFunctionalFile.py

@@ -26,8 +26,8 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = "functional_producer.root"
-svc.output = "functional_transformer.root"
+svc.Input = "functional_producer.root"
+svc.Output = "functional_transformer.root"
 
 transformer = ExampleFunctionalTransformer(
     "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]

test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py

@@ -26,8 +26,8 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = "functional_producer_multiple.root"
-svc.output = "functional_transformer_multiple.root"
+svc.Input = "functional_producer_multiple.root"
+svc.Output = "functional_transformer_multiple.root"
 
 transformer = ExampleFunctionalTransformerMultiple(
     "Transformer",

test/k4FWCoreTest/options/ExampleFunctionalFilterFile.py

@@ -30,7 +30,7 @@
 from Configurables import EventDataSvc
 
 iosvc = IOSvc("IOSvc")
-iosvc.output = "functional_filter.root"
+iosvc.Output = "functional_filter.root"
 
 transformer = ExampleFunctionalTransformer(
     "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]

test/k4FWCoreTest/options/ExampleFunctionalMTFile.py

@@ -35,14 +35,17 @@
 )
 
 slimeventloopmgr = HiveSlimEventLoopMgr(
-    "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
+    "HiveSlimEventLoopMgr",
+    SchedulerName="AvalancheSchedulerSvc",
+    Warnings=False,
+    OutputLevel=WARNING,
 )
 
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True, OutputLevel=WARNING)
 
 svc = IOSvc("IOSvc")
-svc.input = "functional_producer_multiple.root"
-svc.output = "functional_transformerMT.root"
+svc.Input = "functional_producer_multiple.root"
+svc.Output = "functional_transformerMT.root"
 
 consumer = ExampleFunctionalConsumer(
     "Consumer1",

test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py

@@ -38,7 +38,9 @@
     ForceLeaves=True,
 )
 
-slimeventloopmgr = HiveSlimEventLoopMgr(SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING)
+slimeventloopmgr = HiveSlimEventLoopMgr(
+    SchedulerName="AvalancheSchedulerSvc", Warnings=False, OutputLevel=WARNING
+)
 
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 scheduler.ShowDataDependencies = True

test/k4FWCoreTest/options/ExampleFunctionalMetadata.py

@@ -28,7 +28,7 @@
 from Configurables import EventDataSvc
 
 iosvc = IOSvc()
-iosvc.output = "functional_metadata.root"
+iosvc.Output = "functional_metadata.root"
 
 producer = ExampleFunctionalMetadataProducer("Producer", OutputCollection=["MCParticles"])
 consumer = ExampleFunctionalMetadataConsumer("Consumer", InputCollection=["MCParticles"])

test/k4FWCoreTest/options/ExampleFunctionalMetadataOldAlgorithm.py

@@ -21,7 +21,7 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 iosvc = IOSvc()
-iosvc.output = "functional_metadata_old_algorithm.root"
+iosvc.Output = "functional_metadata_old_algorithm.root"
 
 producer = k4FWCoreTest_cellID_writer()
 consumer = k4FWCoreTest_cellID_reader()

test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py

@@ -25,8 +25,8 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = "functional_producer_multiple.root"
-svc.output = "functional_transformer_multiple_output_commands.root"
+svc.Input = "functional_producer_multiple.root"
+svc.Output = "functional_transformer_multiple_output_commands.root"
 svc.outputCommands = [
     "drop Tracks",
     "drop Counter",

test/k4FWCoreTest/options/ExampleFunctionalProducer.py

@@ -30,9 +30,9 @@
 
 iosvc = IOSvc("IOSvc")
 name = "functional_producer.root" if not args[0].second else "functional_producer2.root"
-iosvc.output = name
+iosvc.Output = name
 # Collections can be dropped
-# out.outputCommands = ["drop *"]
+# out.OutputCommands = ["drop *"]
 
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")

test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py

@@ -26,7 +26,7 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 io = IOSvc("IOSvc")
-io.output = "/tmp/a/b/c/functional_producer.root"
+io.Output = "/tmp/a/b/c/functional_producer.root"
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
 

test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py

@@ -27,9 +27,9 @@
 
 
 iosvc = IOSvc("IOSvc")
-iosvc.output = "functional_producer_multiple.root"
+iosvc.Output = "functional_producer_multiple.root"
 # Collections can be dropped
-# out.outputCommands = ["drop *"]
+# out.OutputCommands = ["drop *"]
 
 producer = ExampleFunctionalProducerMultiple(
     "ExampleFunctionalProducerMultiple",

test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py

@@ -25,7 +25,7 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = [
+svc.Input = [
     "functional_producer.root",
     "functional_producer2.root",
 ]

test/k4FWCoreTest/options/ExampleFunctionalTransformerHist.py

@@ -37,7 +37,10 @@
         ForceLeaves=True,
     )
     slimeventloopmgr = HiveSlimEventLoopMgr(
-        "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=INFO
+        "HiveSlimEventLoopMgr",
+        SchedulerName="AvalancheSchedulerSvc",
+        Warnings=False,
+        OutputLevel=INFO,
     )
 
     scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=INFO)