Merge pull request #294 from HSF/dev

Dev

main/lib/idds/tests/test_iworkflow/test_iworkflow_local.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <wen.guan@cern.ch>, 2024
+
+
+"""
+Test workflow.
+"""
+
+import inspect     # noqa F401
+import logging
+import os          # noqa F401
+import shutil      # noqa F401
+import sys         # noqa F401
+
+# from nose.tools import assert_equal
+from idds.common.utils import setup_logging, run_process, json_dumps, json_loads, create_archive_file    # noqa F401
+
+from idds.iworkflow.workflow import Workflow, workflow       # workflow    # noqa F401
+from idds.iworkflow.work import work
+
+
+setup_logging(__name__)
+
+
+@work
+def test_func(name):
+    print('test_func starts')
+    print(name)
+    print('test_func ends')
+    return 'test result: %s' % name
+
+
+def test_func1(name):
+    print('test_func1 starts')
+    print(name)
+    print('test_func1 ends')
+    return 'test result: %s' % name
+
+
+# @workflow(service='idds', local=True, cloud='US', queue='FUNCX_TEST')   # queue = 'BNL_OSG_2'
+@workflow(service='panda', local=True, cloud='US', queue='BNL_OSG_2')
+def test_workflow():
+    print("test workflow starts")
+    ret = test_func(name='idds')
+    print(ret)
+    print("test workflow ends")
+
+
+@work
+def get_params():
+    list_params = [i for i in range(10)]
+    return list_params
+
+
+def test_workflow_mulitple_work():
+    print("test workflow multiple work starts")
+    list_params = get_params()
+
+    ret = test_func(list_params)
+    print(ret)
+    print("test workflow multiple work ends")
+
+
+def submit_workflow(wf):
+    req_id = wf.submit()
+    print("req id: %s" % req_id)
+
+
+def run_workflow_wrapper(wf):
+    cmd = wf.get_runner()
+    logging.info(f'To run workflow: {cmd}')
+
+    exit_code = run_process(cmd, wait=True)
+    logging.info(f'Run workflow finished with exit code: {exit_code}')
+    return exit_code
+
+
+def run_workflow_remote_wrapper(wf):
+    cmd = wf.get_runner()
+    logging.info('To run workflow: %s' % cmd)
+
+    work_dir = '/tmp/idds'
+    shutil.rmtree(work_dir)
+    os.makedirs(work_dir)
+    os.chdir(work_dir)
+    logging.info("current dir: %s" % os.getcwd())
+
+    # print(dir(wf))
+    # print(inspect.getmodule(wf))
+    # print(inspect.getfile(wf))
+    setup = wf.setup_source_files()
+    logging.info("setup: %s" % setup)
+
+    exc_cmd = 'cd %s' % work_dir
+    exc_cmd += "; wget https://wguan-wisc.web.cern.ch/wguan-wisc/run_workflow_wrapper"
+    exc_cmd += "; chmod +x run_workflow_wrapper; bash run_workflow_wrapper %s" % cmd
+    logging.info("exc_cmd: %s" % exc_cmd)
+    exit_code = run_process(exc_cmd, wait=True)
+    logging.info(f'Run workflow finished with exit code: {exit_code}')
+    return exit_code
+
+
+def test_create_archive_file(wf):
+    archive_name = wf._context.get_archive_name()
+    source_dir = wf._context._source_dir
+    logging.info("archive_name :%s, source dir: %s" % (archive_name, source_dir))
+    archive_file = create_archive_file('/tmp', archive_name, [source_dir])
+    logging.info("created archive file: %s" % archive_file)
+
+
+if __name__ == '__main__':
+    logging.info("start")
+    os.chdir(os.path.dirname(os.path.realpath(__file__)))
+    ret = test_workflow()
+    print('result: %s' % str(ret))

main/lib/idds/tests/test_iworkflow/test_iworkflow_local_mul.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <wen.guan@cern.ch>, 2024
+
+
+"""
+Test workflow.
+"""
+
+import inspect     # noqa F401
+import logging
+import os          # noqa F401
+import shutil      # noqa F401
+import sys         # noqa F401
+
+# from nose.tools import assert_equal
+from idds.common.utils import setup_logging, run_process, json_dumps, json_loads, create_archive_file    # noqa F401
+
+from idds.iworkflow.workflow import Workflow, workflow       # workflow    # noqa F401
+from idds.iworkflow.work import work
+
+
+setup_logging(__name__)
+
+
+@work
+def test_func(name):
+    print('test_func starts')
+    print(name)
+    print('test_func ends')
+    return 'test result: %s' % name
+
+
+def test_func1(name):
+    print('test_func1 starts')
+    print(name)
+    print('test_func1 ends')
+    return 'test result: %s' % name
+
+
+# @workflow(service='idds', local=True, cloud='US', queue='FUNCX_TEST')   # queue = 'BNL_OSG_2'
+@workflow(service='panda', local=True, cloud='US', queue='BNL_OSG_2')
+def test_workflow():
+    print("test workflow starts")
+    ret = test_func(name='idds', group_kwargs=[{'name': 'idds1'}, {'name': 'idds2'}, {'name': 'idds3'}, {'name': 'idds4'}, {'name': 'idds5'}])
+    print(ret)
+    print("test workflow ends")
+
+
+@work
+def get_params():
+    list_params = [i for i in range(10)]
+    return list_params
+
+
+def test_workflow_mulitple_work():
+    print("test workflow multiple work starts")
+    list_params = get_params()
+
+    ret = test_func(list_params)
+    print(ret)
+    print("test workflow multiple work ends")
+
+
+def submit_workflow(wf):
+    req_id = wf.submit()
+    print("req id: %s" % req_id)
+
+
+def run_workflow_wrapper(wf):
+    cmd = wf.get_runner()
+    logging.info(f'To run workflow: {cmd}')
+
+    exit_code = run_process(cmd, wait=True)
+    logging.info(f'Run workflow finished with exit code: {exit_code}')
+    return exit_code
+
+
+def run_workflow_remote_wrapper(wf):
+    cmd = wf.get_runner()
+    logging.info('To run workflow: %s' % cmd)
+
+    work_dir = '/tmp/idds'
+    shutil.rmtree(work_dir)
+    os.makedirs(work_dir)
+    os.chdir(work_dir)
+    logging.info("current dir: %s" % os.getcwd())
+
+    # print(dir(wf))
+    # print(inspect.getmodule(wf))
+    # print(inspect.getfile(wf))
+    setup = wf.setup_source_files()
+    logging.info("setup: %s" % setup)
+
+    exc_cmd = 'cd %s' % work_dir
+    exc_cmd += "; wget https://wguan-wisc.web.cern.ch/wguan-wisc/run_workflow_wrapper"
+    exc_cmd += "; chmod +x run_workflow_wrapper; bash run_workflow_wrapper %s" % cmd
+    logging.info("exc_cmd: %s" % exc_cmd)
+    exit_code = run_process(exc_cmd, wait=True)
+    logging.info(f'Run workflow finished with exit code: {exit_code}')
+    return exit_code
+
+
+def test_create_archive_file(wf):
+    archive_name = wf._context.get_archive_name()
+    source_dir = wf._context._source_dir
+    logging.info("archive_name :%s, source dir: %s" % (archive_name, source_dir))
+    archive_file = create_archive_file('/tmp', archive_name, [source_dir])
+    logging.info("created archive file: %s" % archive_file)
+
+
+if __name__ == '__main__':
+    logging.info("start")
+    os.chdir(os.path.dirname(os.path.realpath(__file__)))
+    ret = test_workflow()
+    print('result: %s' % str(ret))

main/tools/container/singularity/commands

@@ -1,4 +1,5 @@
 
+# aipanda180
 # cp -r /eos/user/w/wguan/idds_ml/singularity/* /opt/singularity
 
 cd /opt/singularity/

main/tools/container/singularity/idds_ml_ax_al9.def

@@ -0,0 +1,32 @@
+Bootstrap: docker
+From: almalinux:9.2
+
+%post
+    # yum update -q -y
+    yum install -q -y wget make git gcc openssl-devel bzip2-devel libffi-devel which pip
+
+    ln -s /usr/bin/python3 /usr/bin/python
+
+    pip install --upgrade pip
+    pip install nevergrad
+    pip install theano keras h5py matplotlib tabulate
+    pip install bayesian-optimization
+    pip install xgboost
+    pip install lightgbm
+    pip install ax-platform
+
+    pip install torch pandas numpy matplotlib wandb botorch
+
+    # clean the cache
+    rm -fr /root/.cache
+
+%environment
+    # export LC_ALL=C
+    # export PATH=/usr/games:$PATH
+
+%labels
+    Maintainer iDDS_HPO_Nevergrad(wen.guan@cern.ch)
+    Version v1.0
+
+%runscript
+    echo "iDDS ML hyper parameter optimization plugin"

main/tools/container/singularity/ml_command

@@ -0,0 +1,62 @@
+[root@aipanda180 singularity]# apptainer remote add --no-login SylabsCloud cloud.sylabs.io
+INFO:    Remote "SylabsCloud" added.
+
+[root@aipanda180 singularity]# apptainer remote login
+Generate an access token at https://cloud.sylabs.io/auth/tokens, and paste it here.
+Token entered will be hidden for security.
+Access Token: 
+INFO:    Access Token Verified!
+INFO:    Token stored in /root/.apptainer/remote.yaml
+
+
+[root@aipanda180 singularity]# singularity keys newpair
+Enter your name (e.g., John Doe) : Wen Guan
+Enter your email address (e.g., john.doe@example.com) : wguan.icedew@gmail.com
+Enter optional comment (e.g., development keys) : dev
+Enter a passphrase : 
+Retype your passphrase : 
+WARNING: passphrases do not match
+Enter a passphrase : 
+Retype your passphrase : 
+Generating Entity and OpenPGP Key Pair... done
+[root@aipanda180 singularity]# singularity keys list
+Public key listing (/root/.apptainer/keys/pgp-public):
+
+0)  User:              Wen Guan (dev) <wguan.icedew@gmail.com>
+    Creation time:     2024-04-02 11:47:04 +0200 CEST
+    Fingerprint:       11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65
+    Length (in bits):  4096
+
+
+[root@aipanda180 singularity]# singularity keys push 11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65
+public key `11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65' pushed to server successfully
+[root@aipanda180 singularity]# ls /root/.apptainer/
+cache  keys  remote-cache  remote.yaml
+[root@aipanda180 singularity]# ls /root/.apptainer/keys/
+pgp-public  pgp-secret
+
+
+[root@aipanda180 singularity]# singularity sign idds_ml_ax_al9.simg 
+INFO:    Signing image with PGP key material
+Enter key passphrase : 
+INFO:    Signature created and applied to image 'idds_ml_ax_al9.simg'
+[root@aipanda180 singularity]# singularity verify idds_ml_ax_al9.simg 
+INFO:    Verifying image with PGP key material
+[LOCAL]   Signing entity: Wen Guan (dev) <wguan.icedew@gmail.com>
+[LOCAL]   Fingerprint: 11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65
+Objects verified:
+ID  |GROUP   |LINK    |TYPE
+------------------------------------------------
+1   |1       |NONE    |Def.FILE
+2   |1       |NONE    |JSON.Generic
+3   |1       |NONE    |FS
+INFO:    Verified signature(s) from image 'idds_ml_ax_al9.simg'
+
+
+[root@aipanda180 singularity]# cp /root/.apptainer/
+cache/        keys/         remote-cache/ remote.yaml   
+[root@aipanda180 singularity]# cp /root/.apptainer/remote.yaml /afs/cern.ch/user/w/wguan/private/apptainer/
+[root@aipanda180 singularity]# cp -r /root/.apptainer/keys /afs/cern.ch/user/w/wguan/private/apptainer/
+
+
+[root@aipanda180 singularity]# singularity push idds_ml_ax_al9.simg library://wguanicedew/ml/idds_ml_ax_al9.sif:latest

main/tools/env/setup_panda.sh

@@ -71,12 +71,12 @@ else
     export PANDA_CONFIG_ROOT=~/.panda/
 
     # export IDDS_HOST=https://aipanda015.cern.ch:443/idds
-
+   
     # dev
-    export IDDS_HOST=https://aipanda104.cern.ch:443/idds
+    # export IDDS_HOST=https://aipanda104.cern.ch:443/idds
 
     # doma
-    # export IDDS_HOST=https://aipanda105.cern.ch:443/idds
+    export IDDS_HOST=https://aipanda105.cern.ch:443/idds
 
     # export IDDS_BROKERS=atlas-test-mb.cern.ch:61013
     # export IDDS_BROKER_DESTINATION=/topic/doma.idds

monitor/data/conf.js

@@ -1,9 +1,9 @@
 
 var appConfig = {
-    'iddsAPI_request': "https://lxplus964.cern.ch:443/idds/monitor_request/null/null",
-    'iddsAPI_transform': "https://lxplus964.cern.ch:443/idds/monitor_transform/null/null",
-    'iddsAPI_processing': "https://lxplus964.cern.ch:443/idds/monitor_processing/null/null",
-    'iddsAPI_request_detail': "https://lxplus964.cern.ch:443/idds/monitor/null/null/true/false/false",
-    'iddsAPI_transform_detail': "https://lxplus964.cern.ch:443/idds/monitor/null/null/false/true/false",
-    'iddsAPI_processing_detail': "https://lxplus964.cern.ch:443/idds/monitor/null/null/false/false/true"
+    'iddsAPI_request': "https://lxplus998.cern.ch:443/idds/monitor_request/null/null",
+    'iddsAPI_transform': "https://lxplus998.cern.ch:443/idds/monitor_transform/null/null",
+    'iddsAPI_processing': "https://lxplus998.cern.ch:443/idds/monitor_processing/null/null",
+    'iddsAPI_request_detail': "https://lxplus998.cern.ch:443/idds/monitor/null/null/true/false/false",
+    'iddsAPI_transform_detail': "https://lxplus998.cern.ch:443/idds/monitor/null/null/false/true/false",
+    'iddsAPI_processing_detail': "https://lxplus998.cern.ch:443/idds/monitor/null/null/false/false/true"
 }