From 6b7a30821b87900b1e2d404ee9804c069af86dd2 Mon Sep 17 00:00:00 2001 From: Stephen Mildenhall Date: Tue, 23 Jan 2024 14:31:51 +0000 Subject: [PATCH] Update for R2 aas paper add some splicing examples look at new qd; do you have all the necessary options for aas paper? sort out test_cases --- README.rst | 29 ++++++++++- aggregate/__init__.py | 2 +- aggregate/agg/test_suite.agg | 3 -- aggregate/decl_pygments.py | 15 ------ aggregate/distributions.py | 19 +++---- aggregate/extensions/test_suite.py | 81 ++++++++++++++---------------- aggregate/parser.py | 2 + aggregate/underwriter.py | 14 ++++-- aggregate/utilities.py | 15 +++++- docs/conf.py | 3 ++ pipreq_requirements.txt | 10 ++++ pyproject.toml | 5 +- 12 files changed, 115 insertions(+), 83 deletions(-) create mode 100644 pipreq_requirements.txt diff --git a/README.rst b/README.rst index 55ff23e..359b779 100644 --- a/README.rst +++ b/README.rst @@ -39,16 +39,41 @@ https://github.com/mynl/aggregate Installation ------------ -:: +To install into a new ``Python>=3.10`` virtual environment:: + + python -m venv path/to/your/venv`` + cd path/to/your/venv + +followed by:: + + \path\to\env\Scripts\activate + +on Windows, or:: + + source /path/to/env/bin/activate - pip install aggregate +on Linux/Unix or MacOS. Finally, install the package:: + pip install aggregate[dev] + +All the code examples have been tested in such a virtual environment and the documentation will build. Version History ----------------- +0.21.4 +~~~~~~~~ + +* Updated requirement using ``pipreqs`` recommendations +* Color graphics in documentation +* Added ``expected_shift_reduce = 16 # Set this to the number of expected shift/reduce conflicts`` to ``parser.py`` + to avoid warnings. The conflicts are resolved in the correct way for the grammar to work. +* Issues: there is a difference between ``dfreq[1]`` and ``1 claim ... fixed``, e.g., + when using spliced severities. These should not occur. + + 0.21.3 ~~~~~~~~ diff --git a/aggregate/__init__.py b/aggregate/__init__.py index b7d813f..2bbf462 100644 --- a/aggregate/__init__.py +++ b/aggregate/__init__.py @@ -49,7 +49,7 @@ __email__ = "steve@convexrisk.com" __status__ = "beta" # only need to change here, feeds conf.py (docs) and pyproject.toml (build) -__version__ = "0.21.3" +__version__ = "0.21.4" diff --git a/aggregate/agg/test_suite.agg b/aggregate/agg/test_suite.agg index 98c6db6..6040896 100644 --- a/aggregate/agg/test_suite.agg +++ b/aggregate/agg/test_suite.agg @@ -136,7 +136,6 @@ agg F.Expos01 10 claims sev lognorm 50 cv 0.8 poisson note{specify agg F.Expos02 500 loss sev lognorm 50 cv 0.8 poisson note{specify expected loss, derive number of claims} agg F.Expos03 1000 prem at .5 lr sev lognorm 50 cv 0.8 poisson note{specify premium and loss ratio, derive number of claims} - # Mixed and Spliced severities # ============================ agg G.Mixed00 1 claim 50 xs 0 sev lognorm 10 cv [0.2 0.4 0.6 0.8 1.0] wts [.2 .3 .3 .15 .05] poisson note{no shared mixing} @@ -150,7 +149,6 @@ agg G.Mixed08 1 claim sev [100 200 250 300] * beta [1 200 500 100] [ agg G.Mixed09 8 claim sev 100 * [lognorm expon] [.5 1] wts [0.6 .4] mixed gamma 0.3 note{different severities} agg G.Mixed10 1 claim sev [50 100] * [lognorm expon] [2 1] + 10 wts=2 mixed gamma 0.3 agg G.Mixed11 1 claim sev [50 100] * [lognorm expon] [2 1] + 10 mixed gamma 0.3 -# agg G.Spliced01 1 claim 50 xs 0 sev lognorm 10 cv [0.2 0.4 0.6 0.8 1.0] wts [.2 .3 .3 .15 .05] spliced gamma 0.3 note{shared mixing, compare audit and report dfs} # Limit profiles # ============== @@ -174,7 +172,6 @@ agg I.Blend10 [500 800 200] loss sev lognorm 10 c agg I.Blend11 [1000 2000 500] prem at [.8 .7 .5] lr sev lognorm 10 cv [.2 .35 .5] wts [1/2 3/8 1/8] mixed gamma 0.5 note{log2=17;} agg I.Blend12 [500 800 200] loss sev lognorm 10 cv [.2 .35 .5] wts=3 mixed gamma 0.5 - # Reinsurance # =========== agg J.Re01 5 claims 100 xs 0 sev lognorm 10 cv .75 occurrence net of 50% so 5 xs 0 and 5 po 15 xs 5 and 30 xs 20 poisson diff --git a/aggregate/decl_pygments.py b/aggregate/decl_pygments.py index f3fa5fe..f892332 100644 --- a/aggregate/decl_pygments.py +++ b/aggregate/decl_pygments.py @@ -13,21 +13,6 @@ __all__ = ['AggLexer'] -# def colorize(code): -# # step 2: apply custom style -# # embed Style inside HTML (self-contained, no external CSS-file -# # formatter.noclasses = True # inline style to each element directly -# formatter = HtmlFormatter(style='monokai', full=True) -# return highlight(code, AggLexer(), formatter) - - -# def rawhtml(code): -# formatter = HtmlFormatter(style='monokai', full=False) -# return highlight(code, AggLexer(), formatter) - - -# define custom style -> see older version; don't want to do this - class AggLexer(RegexLexer): """ Aggregate program language lexer. (Based on Python lexer. ) diff --git a/aggregate/distributions.py b/aggregate/distributions.py index 3bedbb1..0f9f713 100644 --- a/aggregate/distributions.py +++ b/aggregate/distributions.py @@ -2160,12 +2160,12 @@ def apply_agg_reins(self, debug=False, padding=1, tilt_vector=None): logger.info(f'Applying agg reins to {self.name}\tOld mean and cv= {_m:,.3f}\t{_m:,.3f}\n' f'New mean and cv = {_m2:,.3f}\t{_cv2:,.3f}') - def reinsurance_description(self, kind='both', width=70): + def reinsurance_description(self, kind='both', width=0): """ Text description of the reinsurance. :param kind: both, occ, or agg - :param width: width of text for textwrap.fill + :param width: width of text for textwrap.fill; omitted if width==0 """ ans = [] if self.occ_reins is not None and kind in ['occ', 'both']: @@ -2173,12 +2173,12 @@ def reinsurance_description(self, kind='both', width=70): ra = [] for (s, y, a) in self.occ_reins: if np.isinf(y): - ra.append(f'{s:,.2%} share of unlimited xs {a:,.2f}') + ra.append(f'{s:,.0%} share of unlimited xs {a:,.0f}') else: if s == y: - ra.append(f'{y:,.2f} xs {a:,.2f}') + ra.append(f'{y:,.0f} xs {a:,.0f}') else: - ra.append(f'{s:,.2%} share of {y:,.2f} xs {a:,.2f}') + ra.append(f'{s:,.0%} share of {y:,.0f} xs {a:,.0f}') ans.append(' and '.join(ra)) ans.append('per occurrence') if self.agg_reins is not None and kind in ['agg', 'both']: @@ -2188,12 +2188,12 @@ def reinsurance_description(self, kind='both', width=70): ra = [] for (s, y, a) in self.agg_reins: if np.isinf(y): - ra.append(f'{s:,.2%} share of unlimited xs {a:,.2f}') + ra.append(f'{s:,.0%} share of unlimited xs {a:,.0f}') else: if s == y: - ra.append(f'{y:,.2f} xs {a:,.2f}') + ra.append(f'{y:,.0f} xs {a:,.0f}') else: - ra.append(f'{s:,.2%} share of {y:,.2f} xs {a:,.2f}') + ra.append(f'{s:,.0%} share of {y:,.0f} xs {a:,.0f}') ans.append(' and '.join(ra)) ans.append('in the aggregate.') if len(ans): @@ -2204,7 +2204,8 @@ def reinsurance_description(self, kind='both', width=70): reins = ' '.join(ans) else: reins = 'No reinsurance' - reins = fill(reins, width) + if width: + reins = fill(reins, width) return reins def reinsurance_kinds(self): diff --git a/aggregate/extensions/test_suite.py b/aggregate/extensions/test_suite.py index 4e934a7..794cc07 100644 --- a/aggregate/extensions/test_suite.py +++ b/aggregate/extensions/test_suite.py @@ -1,101 +1,95 @@ # code for running test cases, producing HTML, etc. from .. import pprint_ex - -# from ..aggregate.utilities import iman_conover, mu_sigma_from_mean_cv -# # from aggregate.utils import rearrangement_algorithm_max_VaR -# from .. aggregate.utilities import random_corr_matrix +from .. import build as build_uw import logging import matplotlib.pyplot as plt from pathlib import Path import re - logger = logging.getLogger(__name__) class TestSuite(object): - p = None - build = None - tests = '' - - @classmethod - def __init__(cls, build=None, out_dir_name=''): + def __init__(self, build_in=None, fn='test_suite.agg', out_dir_name=''): """ Run test suite fn. Create specified objects. Save graphics and info to HTML. Wrap HTML with template. TODO: convert wrapping to Jinja! - To run whole test_suite - :: + To run whole test_suite:: python -m aggregate.extensions.test_suite + :param build_in: build object, allows input custom build object + :param fn: test suite file name, default test_suite.agg + :param out_dir_name: output directory name, default site_dir/generated """ - if build is None: - from .. import build - - cls.build = build + self.build = build_in if build_in else build_uw if out_dir_name != '': - cls.p = Path(out_dir_name) - if cls.p.exists() is False: + self.out_dir = Path(out_dir_name) + if self.out_dir.exists() is False: raise FileExistsError(f'Directory {out_dir_name} does not exist.') else: - cls.p = cls.build.site_dir.parent / 'generated' - cls.p.mkdir(exist_ok=True) - (cls.p / "img").mkdir(exist_ok=True) + self.out_dir = self.build.site_dir.parent / 'generated' + self.out_dir.mkdir(exist_ok=True) + (self.out_dir / "img").mkdir(exist_ok=True) - logger.info(f'Output directory {cls.p.resolve()}') + logger.info(f'Output directory {self.out_dir.resolve()}') - # extract from comments; this is just FYI - fn = 'test_suite.agg' - suite = build.default_dir / fn + suite = self.build.default_dir / fn + assert suite.exists(), f'Requested test suite file {suite} does not exist.' txt = suite.read_text(encoding='utf-8') tests = [i for i in txt.split('\n') if re.match(r'# [A-Z]\.', i)] - cls.tests = [i.replace("# ", "").split('. ') for i in tests] + self.tests = [i.replace("# ", "").split('. ') for i in tests] - @classmethod - def run(cls, regex, title, fig_prefix, fig_format='svg', fig_size=(8,2.4), **kwargs): + def run(self, regex, title, filename, browse=False, fig_format='svg', fig_size=(8,2.4), **kwargs): """ + Run all tests matching regex. Save graphics and info to HTML. + Wrap HTML with template. To run whole test_suite use:: + + python -m aggregate.extensions.test_suite :param regex: regex of tests to run, e.g., 'agg [ABC]\. ' :param title: title for blob - :param fig_prefix: file name prefix for saved immage files (convenience) + :param filename: file name prefix for saved immage files (convenience) + :param browse: open browser to output file :param fig_format: html or markdown (md); html uses svg output, markdown uses pdf :param fig_size: :param kwargs: passed to savefig """ - logger.warning(f'figure prefix = {fig_prefix}') - ans = [] - for n in cls.build.qshow(regex).index: - a = cls.build(n) + for n in self.build.qshow(regex, tacit=False).index: + a = self.build(n) ans.append(a.html_info_blob().replace('h3>', 'h2>')) - ans.append(pprint_ex(a.program, 50, True, True)) - ans.append(cls.style_df(a.describe).to_html()) + ans.append(pprint_ex(a.program, 50, True)) + ans.append(self.style_df(a.describe).to_html()) ans.append('
') - fn = cls.p / f'img/{fig_prefix}_tmp_{hash(a):0x}.{fig_format}' + fn = self.out_dir / f'img/{filename}_tmp_{hash(a):0x}.{fig_format}' a.plot(figsize=fig_size) a.figure.savefig(fn, **kwargs) ans.append(f'') plt.close(a.figure) logger.warning(f'Created {n}, mean {a.agg_m:.2f}') - - blob = '\n'.join(ans) - fn = cls.p / f'{fig_prefix}.html' + blob = '\n'.join([i if type(i)==str else i.data for i in ans]) + fn = self.out_dir / f'{filename}.html' fn.write_text(blob, encoding='utf-8') - fn2 = cls.p / f'{fn.stem}_wrapped.html' - fn3 = cls.build.template_dir / 'test_suite_template.html' + fn2 = self.out_dir / f'{fn.stem}_wrapped.html' + fn3 = self.build.template_dir / 'test_suite_template.html' # TODO JINJA! template = fn3.read_text() template = template.replace('HEADING GOES HERE', title).replace( 'CONTENTHERE', blob) fn2.write_text(template, encoding='utf-8') + logger.info(f'Output written to {fn2.resolve()}') + if browse: + import webbrowser + webbrowser.open(fn2.resolve().as_uri()) @staticmethod def style_df(df): @@ -166,7 +160,8 @@ def run_test_suite(): # run all the aggs # TODO FIX for Portfolios # t.run(regex=r'^C\.', title='C only', fig_prefix="auto", fig_format='png', dpi=300) - t.run(regex=r'^[A-KNO]\.', title='Full Test Suite', fig_prefix="auto", fig_format='png', dpi=300) + t.run(regex=r'^[A-KNO]', title='Full Test Suite', filename='A_tests', browse=True, + fig_format='png', dpi=300) if __name__ == '__main__': diff --git a/aggregate/parser.py b/aggregate/parser.py index 0d74ec4..02e5aee 100644 --- a/aggregate/parser.py +++ b/aggregate/parser.py @@ -199,6 +199,8 @@ class UnderwritingParser(Parser): """ + expected_shift_reduce = 16 # Set this to the number of expected shift/reduce conflicts + debugfile = None # uncomment to write detailed grammar rules # debugfile = Path.home() / 'aggregate/parser/parser.out' diff --git a/aggregate/underwriter.py b/aggregate/underwriter.py index 46fb091..1986352 100644 --- a/aggregate/underwriter.py +++ b/aggregate/underwriter.py @@ -829,10 +829,11 @@ def qlist(self, regex): """ return self.show(regex, kind='', plot=False, describe=False, verbose=True) - def qshow(self, regex): + def qshow(self, regex, tacit=True): """ Wrapper for show to just show (display) elements in knowledge that match ``regex``. - No reutrn value. + No reutrn value if tacit, else returns a dataframe. + """ def ff(x): fs = '{x:120s}' @@ -843,9 +844,12 @@ def ff(x): r' note\{[^}]+\}', '').str.replace(' +', ' ') # , flags=re.MULTILINE) # bit['program'] = bit['program'].str.replace(' ( +)', ' ') #, flags=re.MULTILINE) # bit['program'] = bit['program'].str.replace(r' note\{[^}]+\}$| *', ' ' ) #, flags=re.MULTILINE) - qd(bit, - line_width=160, max_colwidth=130, col_space=15, justify='left', - max_rows=200, formatters={'program': ff}) + if tacit: + qd(bit, + line_width=160, max_colwidth=130, col_space=15, justify='left', + max_rows=200, formatters={'program': ff}) + else: + return bit def show(self, regex, kind='', plot=True, describe=True, logger_level=30, verbose=False, **kwargs): """ diff --git a/aggregate/utilities.py b/aggregate/utilities.py index d271397..bb65e8f 100644 --- a/aggregate/utilities.py +++ b/aggregate/utilities.py @@ -2962,7 +2962,7 @@ def moms_analytic(fz, limit, attachment, n, analytic=True): return ans -def qd(*argv, accuracy=3, align=True, trim=True, **kwargs): +def qd(*argv, accuracy=3, align=True, trim=True, ff=None, **kwargs): """ Endless quest for a robust display format! @@ -2973,13 +2973,24 @@ def qd(*argv, accuracy=3, align=True, trim=True, **kwargs): :param: argv: list of objects to print :param: accuracy: number of decimal places to display :param: align: if True, align columns at decimal point (sEngFormatter) + :param: trim: if True, trim trailing zeros (sEngFormatter) + :param: ff: if not None, use this function to format floats, or 'basic', or 'binary' :kwargs: passed to pd.DataFrame.to_string for dataframes only. e.g., pass dict of formatters by column. """ from .distributions import Aggregate from .portfolio import Portfolio # ff = sEngFormatter(accuracy=accuracy - (2 if align else 0), min_prefix=0, max_prefix=12, align=align, trim=trim) - ff = kwargs.pop('ff', lambda x: f'{x:.5g}') + if ff is None: + ff = lambda x: f'{x:.5g}' + elif ff == 'basic': + ff = lambda x: f'{x:.1%}' if x < 1 else f'{x:12,.0f}' + elif ff == 'int_ratio': + def format_function(x): + ir = np.round(x, 13).as_integer_ratio() + return f'{int(x)}' if x in [0, 1] else f' {ir[0]}/{ir[1]}' + + ff = format_function # split output for x in argv: if isinstance(x, (Aggregate, Portfolio)): diff --git a/docs/conf.py b/docs/conf.py index 2662e27..bc54576 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,9 @@ sys.path.insert(0, os.path.abspath('../')) import aggregate as agg +# color graphs +agg.knobble_fonts(True) + # graphics defaults - better res graphics plt.rcParams['figure.dpi'] = 300 diff --git a/pipreq_requirements.txt b/pipreq_requirements.txt new file mode 100644 index 0000000..2200f15 --- /dev/null +++ b/pipreq_requirements.txt @@ -0,0 +1,10 @@ +cycler>=0.12.1 +ipython>=8.17.2 +Jinja2>=3.1.2 +matplotlib>=3.8.2 +numpy>=1.26.3 +pandas>=2.1.4 +psutil>=5.9.6 +Pygments>=2.16.1 +scipy>=1.11.4 +titlecase>=2.4.1 diff --git a/pyproject.toml b/pyproject.toml index aaea639..ea504bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,9 @@ dependencies = [ "numpy>=1.26", "pandas>=2.1", "psutil", + "Pygments", "scipy>=1.11", -# "sly", "titlecase", - "pygments" ] license = {text = "BSD-3-Clause"} requires-python = ">=3.10" @@ -47,7 +46,7 @@ Documentation = "https://aggregate.readthedocs.io/en/latest/" [project.optional-dependencies] dev = [ - "docutils==0.16", + "docutils<0.17", "jupyter-sphinx", "nbsphinx", "pickleshare",