Update for R2 aas paper

add some splicing examples
look at new qd; do  you have all the necessary options for aas paper?
sort out test_cases

README.rst

@@ -39,16 +39,41 @@ https://github.com/mynl/aggregate
 Installation
 ------------
 
-::
+To install into a new ``Python>=3.10`` virtual environment::
+
+    python -m venv path/to/your/venv``
+    cd path/to/your/venv
+
+followed by::
+
+    \path\to\env\Scripts\activate
+
+on Windows, or::
+
+    source /path/to/env/bin/activate
 
-  pip install aggregate
+on Linux/Unix or MacOS. Finally, install the package::
 
+    pip install aggregate[dev]
+
+All the code examples have been tested in such a virtual environment and the documentation will build.
 
 
 Version History
 -----------------
 
 
+0.21.4
+~~~~~~~~
+
+* Updated requirement using ``pipreqs`` recommendations
+* Color graphics in documentation
+* Added ``expected_shift_reduce = 16  # Set this to the number of expected shift/reduce conflicts`` to ``parser.py``
+  to avoid warnings. The conflicts are resolved in the correct way for the grammar to work.
+* Issues: there is a difference between ``dfreq[1]`` and ``1 claim ... fixed``, e.g.,
+  when using spliced severities. These should not  occur.
+
+
 0.21.3
 ~~~~~~~~
 

aggregate/__init__.py

@@ -49,7 +49,7 @@
 __email__ = "steve@convexrisk.com"
 __status__ = "beta"
 # only need to change here, feeds conf.py (docs) and pyproject.toml (build)
-__version__ = "0.21.3"
+__version__ = "0.21.4"
 
 
 

aggregate/agg/test_suite.agg

@@ -136,7 +136,6 @@ agg F.Expos01   10 claims         sev lognorm 50 cv 0.8 poisson    note{specify
 agg F.Expos02  500 loss           sev lognorm 50 cv 0.8 poisson    note{specify expected loss, derive number of claims}
 agg F.Expos03 1000 prem at .5 lr  sev lognorm 50 cv 0.8 poisson    note{specify premium and loss ratio, derive number of claims}
 
-
 # Mixed and Spliced severities
 # ============================
 agg G.Mixed00  1  claim  50 xs 0 sev lognorm 10 cv [0.2 0.4 0.6 0.8 1.0] wts [.2 .3 .3 .15 .05]             poisson            note{no shared mixing}
@@ -150,7 +149,6 @@ agg G.Mixed08  1  claim          sev [100 200 250 300] *  beta [1 200 500 100] [
 agg G.Mixed09  8  claim          sev      100 * [lognorm expon] [.5 1] wts [0.6 .4]                         mixed gamma 0.3    note{different severities}
 agg G.Mixed10  1  claim          sev [50 100] * [lognorm expon]  [2 1]  + 10 wts=2                          mixed gamma 0.3
 agg G.Mixed11  1  claim          sev [50 100] * [lognorm expon]  [2 1]  + 10                                mixed gamma 0.3
-# agg G.Spliced01 1 claim 50 xs 0 sev lognorm 10 cv [0.2 0.4 0.6 0.8 1.0] wts [.2 .3 .3 .15 .05]             spliced gamma 0.3  note{shared mixing, compare audit and report dfs}
 
 # Limit profiles
 # ==============
@@ -174,7 +172,6 @@ agg I.Blend10  [500 800 200]   loss                             sev lognorm 10 c
 agg I.Blend11  [1000 2000 500] prem at [.8 .7 .5] lr            sev lognorm 10 cv [.2 .35 .5] wts [1/2 3/8 1/8] mixed gamma 0.5  note{log2=17;}
 agg I.Blend12  [500 800 200]   loss                             sev lognorm 10 cv [.2 .35 .5] wts=3             mixed gamma 0.5
 
-
 # Reinsurance
 # ===========
 agg J.Re01  5 claims 100 xs 0 sev lognorm 10 cv .75 occurrence net of 50% so 5 xs 0 and 5 po 15 xs 5 and 30 xs 20   poisson

aggregate/decl_pygments.py

@@ -13,21 +13,6 @@
 __all__ = ['AggLexer']
 
 
-# def colorize(code):
-#     # step 2: apply custom style
-#     # embed Style inside HTML (self-contained, no external CSS-file
-#     # formatter.noclasses = True  # inline style to each element directly
-#     formatter = HtmlFormatter(style='monokai', full=True)
-#     return highlight(code, AggLexer(), formatter)
-
-
-# def rawhtml(code):
-#     formatter = HtmlFormatter(style='monokai', full=False)
-#     return highlight(code, AggLexer(), formatter)
-
-
-# define custom style -> see older version; don't want to do this
-
 class AggLexer(RegexLexer):
     """
     Aggregate program language lexer. (Based on Python lexer. )

aggregate/distributions.py

@@ -2160,25 +2160,25 @@ def apply_agg_reins(self, debug=False, padding=1, tilt_vector=None):
         logger.info(f'Applying agg reins to {self.name}\tOld mean and cv= {_m:,.3f}\t{_m:,.3f}\n'
                     f'New mean and cv = {_m2:,.3f}\t{_cv2:,.3f}')
 
-    def reinsurance_description(self, kind='both', width=70):
+    def reinsurance_description(self, kind='both', width=0):
         """
         Text description of the reinsurance.
 
         :param kind: both, occ, or agg
-        :param width: width of text for textwrap.fill
+        :param width: width of text for textwrap.fill; omitted if width==0
         """
         ans = []
         if self.occ_reins is not None and kind in ['occ', 'both']:
             ans.append(self.occ_kind)
             ra = []
             for (s, y, a) in self.occ_reins:
                 if np.isinf(y):
-                    ra.append(f'{s:,.2%} share of unlimited xs {a:,.2f}')
+                    ra.append(f'{s:,.0%} share of unlimited xs {a:,.0f}')
                 else:
                     if s == y:
-                        ra.append(f'{y:,.2f} xs {a:,.2f}')
+                        ra.append(f'{y:,.0f} xs {a:,.0f}')
                     else:
-                        ra.append(f'{s:,.2%} share of {y:,.2f} xs {a:,.2f}')
+                        ra.append(f'{s:,.0%} share of {y:,.0f} xs {a:,.0f}')
             ans.append(' and '.join(ra))
             ans.append('per occurrence')
         if self.agg_reins is not None and kind in ['agg', 'both']:
@@ -2188,12 +2188,12 @@ def reinsurance_description(self, kind='both', width=70):
             ra = []
             for (s, y, a) in self.agg_reins:
                 if np.isinf(y):
-                    ra.append(f'{s:,.2%} share of unlimited xs {a:,.2f}')
+                    ra.append(f'{s:,.0%} share of unlimited xs {a:,.0f}')
                 else:
                     if s == y:
-                        ra.append(f'{y:,.2f} xs {a:,.2f}')
+                        ra.append(f'{y:,.0f} xs {a:,.0f}')
                     else:
-                        ra.append(f'{s:,.2%} share of {y:,.2f} xs {a:,.2f}')
+                        ra.append(f'{s:,.0%} share of {y:,.0f} xs {a:,.0f}')
             ans.append(' and '.join(ra))
             ans.append('in the aggregate.')
         if len(ans):
@@ -2204,7 +2204,8 @@ def reinsurance_description(self, kind='both', width=70):
             reins = ' '.join(ans)
         else:
             reins = 'No reinsurance'
-        reins = fill(reins, width)
+        if width:
+            reins = fill(reins, width)
         return reins
 
     def reinsurance_kinds(self):

aggregate/extensions/test_suite.py

@@ -1,101 +1,95 @@
 # code for running test cases, producing HTML, etc.
 
 from .. import pprint_ex
-
-# from ..aggregate.utilities  import iman_conover, mu_sigma_from_mean_cv
-# # from aggregate.utils import rearrangement_algorithm_max_VaR
-# from .. aggregate.utilities import random_corr_matrix
+from .. import build as build_uw
 import logging
 import matplotlib.pyplot as plt
 from pathlib import Path
 import re
 
-
 logger = logging.getLogger(__name__)
 
 
 class TestSuite(object):
 
-    p = None
-    build = None
-    tests = ''
-
-    @classmethod
-    def __init__(cls, build=None, out_dir_name=''):
+    def __init__(self, build_in=None, fn='test_suite.agg', out_dir_name=''):
         """
         Run test suite fn. Create specified objects. Save graphics and info to HTML. Wrap
         HTML with template.
 
         TODO: convert wrapping to Jinja!
 
-        To run whole test_suite
-        ::
+        To run whole test_suite::
 
             python -m aggregate.extensions.test_suite
 
+        :param build_in: build object, allows input custom build object
+        :param fn: test suite file name, default test_suite.agg
+        :param out_dir_name: output directory name, default site_dir/generated
         """
 
-        if build is None:
-            from .. import build
-
-        cls.build = build
+        self.build = build_in if build_in else build_uw
 
         if out_dir_name != '':
-            cls.p = Path(out_dir_name)
-            if cls.p.exists() is False:
+            self.out_dir = Path(out_dir_name)
+            if self.out_dir.exists() is False:
                 raise FileExistsError(f'Directory {out_dir_name} does not exist.')
         else:
-            cls.p = cls.build.site_dir.parent / 'generated'
-            cls.p.mkdir(exist_ok=True)
-            (cls.p / "img").mkdir(exist_ok=True)
+            self.out_dir = self.build.site_dir.parent / 'generated'
+            self.out_dir.mkdir(exist_ok=True)
+            (self.out_dir / "img").mkdir(exist_ok=True)
 
-        logger.info(f'Output directory {cls.p.resolve()}')
+        logger.info(f'Output directory {self.out_dir.resolve()}')
 
-        # extract from comments; this is just FYI
-        fn = 'test_suite.agg'
-        suite = build.default_dir / fn
+        suite = self.build.default_dir / fn
+        assert suite.exists(), f'Requested test suite file {suite} does not exist.'
         txt = suite.read_text(encoding='utf-8')
         tests = [i for i in txt.split('\n') if re.match(r'# [A-Z]\.', i)]
-        cls.tests = [i.replace("# ", "").split('. ') for i in tests]
+        self.tests = [i.replace("# ", "").split('. ') for i in tests]
 
-    @classmethod
-    def run(cls, regex, title, fig_prefix, fig_format='svg', fig_size=(8,2.4), **kwargs):
+    def run(self, regex, title, filename, browse=False, fig_format='svg', fig_size=(8,2.4), **kwargs):
         """
+        Run all tests matching regex. Save graphics and info to HTML.
+        Wrap HTML with template. To run whole test_suite use::
+
+            python -m aggregate.extensions.test_suite
 
         :param regex: regex of tests to run, e.g., 'agg [ABC]\. '
         :param title: title for blob
-        :param fig_prefix: file name prefix for saved immage files (convenience)
+        :param filename: file name prefix for saved immage files (convenience)
+        :param browse: open browser to output file
         :param fig_format:  html or markdown (md); html uses svg output, markdown uses pdf
         :param fig_size:
         :param kwargs: passed to savefig
         """
-        logger.warning(f'figure prefix = {fig_prefix}')
-
         ans = []
-        for n in cls.build.qshow(regex).index:
-            a = cls.build(n)
+        for n in self.build.qshow(regex, tacit=False).index:
+            a = self.build(n)
             ans.append(a.html_info_blob().replace('h3>', 'h2>'))
-            ans.append(pprint_ex(a.program, 50, True, True))
-            ans.append(cls.style_df(a.describe).to_html())
+            ans.append(pprint_ex(a.program, 50, True))
+            ans.append(self.style_df(a.describe).to_html())
             ans.append('<br>')
-            fn = cls.p / f'img/{fig_prefix}_tmp_{hash(a):0x}.{fig_format}'
+            fn = self.out_dir / f'img/{filename}_tmp_{hash(a):0x}.{fig_format}'
             a.plot(figsize=fig_size)
             a.figure.savefig(fn, **kwargs)
             ans.append(f'<img src="{fn.resolve()}" />')
             plt.close(a.figure)
             logger.warning(f'Created {n}, mean {a.agg_m:.2f}')
-
-        blob = '\n'.join(ans)
-        fn = cls.p / f'{fig_prefix}.html'
+        blob = '\n'.join([i if type(i)==str else i.data for i in ans])
+        fn = self.out_dir / f'{filename}.html'
         fn.write_text(blob, encoding='utf-8')
 
-        fn2 = cls.p / f'{fn.stem}_wrapped.html'
-        fn3 = cls.build.template_dir / 'test_suite_template.html'
+        fn2 = self.out_dir / f'{fn.stem}_wrapped.html'
+        fn3 = self.build.template_dir / 'test_suite_template.html'
         # TODO JINJA!
         template = fn3.read_text()
         template = template.replace('HEADING GOES HERE', title).replace(
             'CONTENTHERE', blob)
         fn2.write_text(template, encoding='utf-8')
+        logger.info(f'Output written to {fn2.resolve()}')
+        if browse:
+            import webbrowser
+            webbrowser.open(fn2.resolve().as_uri())
 
     @staticmethod
     def style_df(df):
@@ -166,7 +160,8 @@ def run_test_suite():
     # run all the aggs
     # TODO FIX for Portfolios
     # t.run(regex=r'^C\.', title='C only', fig_prefix="auto", fig_format='png', dpi=300)
-    t.run(regex=r'^[A-KNO]\.', title='Full Test Suite', fig_prefix="auto", fig_format='png', dpi=300)
+    t.run(regex=r'^[A-KNO]', title='Full Test Suite', filename='A_tests', browse=True,
+          fig_format='png', dpi=300)
 
 
 if __name__ == '__main__':

aggregate/parser.py

@@ -199,6 +199,8 @@ class UnderwritingParser(Parser):
 
     """
 
+    expected_shift_reduce = 16  # Set this to the number of expected shift/reduce conflicts
+
     debugfile = None
     # uncomment to write detailed grammar rules
     # debugfile = Path.home() / 'aggregate/parser/parser.out'

aggregate/underwriter.py

@@ -829,10 +829,11 @@ def qlist(self, regex):
         """
         return self.show(regex, kind='', plot=False, describe=False, verbose=True)
 
-    def qshow(self, regex):
+    def qshow(self, regex, tacit=True):
         """
         Wrapper for show to just show (display) elements in knowledge that match ``regex``.
-        No reutrn value.
+        No reutrn value if tacit, else returns a dataframe.
+
         """
         def ff(x):
             fs = '{x:120s}'
@@ -843,9 +844,12 @@ def ff(x):
             r' note\{[^}]+\}', '').str.replace('  +', ' ')  # , flags=re.MULTILINE)
         # bit['program'] = bit['program'].str.replace(' ( +)', ' ') #, flags=re.MULTILINE)
         # bit['program'] = bit['program'].str.replace(r' note\{[^}]+\}$|  *', ' '   ) #, flags=re.MULTILINE)
-        qd(bit,
-           line_width=160, max_colwidth=130, col_space=15, justify='left',
-           max_rows=200, formatters={'program': ff})
+        if tacit:
+            qd(bit,
+               line_width=160, max_colwidth=130, col_space=15, justify='left',
+               max_rows=200, formatters={'program': ff})
+        else:
+            return bit
 
     def show(self, regex, kind='', plot=True, describe=True, logger_level=30, verbose=False, **kwargs):
         """

aggregate/utilities.py

@@ -2962,7 +2962,7 @@ def moms_analytic(fz, limit, attachment, n, analytic=True):
     return ans
 
 
-def qd(*argv, accuracy=3, align=True, trim=True, **kwargs):
+def qd(*argv, accuracy=3, align=True, trim=True, ff=None, **kwargs):
     """
     Endless quest for a robust display format!
 
@@ -2973,13 +2973,24 @@ def qd(*argv, accuracy=3, align=True, trim=True, **kwargs):
     :param: argv: list of objects to print
     :param: accuracy: number of decimal places to display
     :param: align: if True, align columns at decimal point (sEngFormatter)
+    :param: trim: if True, trim trailing zeros (sEngFormatter)
+    :param: ff: if not None, use this function to format floats, or 'basic', or 'binary'
     :kwargs: passed to pd.DataFrame.to_string for dataframes only. e.g., pass dict of formatters by column.
 
     """
     from .distributions import Aggregate
     from .portfolio import Portfolio
     # ff = sEngFormatter(accuracy=accuracy - (2 if align else 0), min_prefix=0, max_prefix=12, align=align, trim=trim)
-    ff = kwargs.pop('ff', lambda x: f'{x:.5g}')
+    if ff is None:
+        ff = lambda x: f'{x:.5g}'
+    elif ff == 'basic':
+        ff = lambda x: f'{x:.1%}' if x < 1 else f'{x:12,.0f}'
+    elif ff == 'int_ratio':
+        def format_function(x):
+            ir = np.round(x, 13).as_integer_ratio()
+            return f'{int(x)}' if x in [0, 1] else f'  {ir[0]}/{ir[1]}'
+
+        ff = format_function
     # split output
     for x in argv:
         if isinstance(x, (Aggregate, Portfolio)):

docs/conf.py

@@ -14,6 +14,9 @@
 sys.path.insert(0, os.path.abspath('../'))
 import aggregate as agg
 
+# color graphs
+agg.knobble_fonts(True)
+
 # graphics defaults - better res graphics
 plt.rcParams['figure.dpi'] = 300
 

pipreq_requirements.txt

@@ -0,0 +1,10 @@
+cycler>=0.12.1
+ipython>=8.17.2
+Jinja2>=3.1.2
+matplotlib>=3.8.2
+numpy>=1.26.3
+pandas>=2.1.4
+psutil>=5.9.6
+Pygments>=2.16.1
+scipy>=1.11.4
+titlecase>=2.4.1