diff --git a/README.rst b/README.rst
index 55ff23e..359b779 100644
--- a/README.rst
+++ b/README.rst
@@ -39,16 +39,41 @@ https://github.com/mynl/aggregate
Installation
------------
-::
+To install into a new ``Python>=3.10`` virtual environment::
+
+ python -m venv path/to/your/venv``
+ cd path/to/your/venv
+
+followed by::
+
+ \path\to\env\Scripts\activate
+
+on Windows, or::
+
+ source /path/to/env/bin/activate
- pip install aggregate
+on Linux/Unix or MacOS. Finally, install the package::
+ pip install aggregate[dev]
+
+All the code examples have been tested in such a virtual environment and the documentation will build.
Version History
-----------------
+0.21.4
+~~~~~~~~
+
+* Updated requirement using ``pipreqs`` recommendations
+* Color graphics in documentation
+* Added ``expected_shift_reduce = 16 # Set this to the number of expected shift/reduce conflicts`` to ``parser.py``
+ to avoid warnings. The conflicts are resolved in the correct way for the grammar to work.
+* Issues: there is a difference between ``dfreq[1]`` and ``1 claim ... fixed``, e.g.,
+ when using spliced severities. These should not occur.
+
+
0.21.3
~~~~~~~~
diff --git a/aggregate/__init__.py b/aggregate/__init__.py
index b7d813f..2bbf462 100644
--- a/aggregate/__init__.py
+++ b/aggregate/__init__.py
@@ -49,7 +49,7 @@
__email__ = "steve@convexrisk.com"
__status__ = "beta"
# only need to change here, feeds conf.py (docs) and pyproject.toml (build)
-__version__ = "0.21.3"
+__version__ = "0.21.4"
diff --git a/aggregate/agg/test_suite.agg b/aggregate/agg/test_suite.agg
index 98c6db6..6040896 100644
--- a/aggregate/agg/test_suite.agg
+++ b/aggregate/agg/test_suite.agg
@@ -136,7 +136,6 @@ agg F.Expos01 10 claims sev lognorm 50 cv 0.8 poisson note{specify
agg F.Expos02 500 loss sev lognorm 50 cv 0.8 poisson note{specify expected loss, derive number of claims}
agg F.Expos03 1000 prem at .5 lr sev lognorm 50 cv 0.8 poisson note{specify premium and loss ratio, derive number of claims}
-
# Mixed and Spliced severities
# ============================
agg G.Mixed00 1 claim 50 xs 0 sev lognorm 10 cv [0.2 0.4 0.6 0.8 1.0] wts [.2 .3 .3 .15 .05] poisson note{no shared mixing}
@@ -150,7 +149,6 @@ agg G.Mixed08 1 claim sev [100 200 250 300] * beta [1 200 500 100] [
agg G.Mixed09 8 claim sev 100 * [lognorm expon] [.5 1] wts [0.6 .4] mixed gamma 0.3 note{different severities}
agg G.Mixed10 1 claim sev [50 100] * [lognorm expon] [2 1] + 10 wts=2 mixed gamma 0.3
agg G.Mixed11 1 claim sev [50 100] * [lognorm expon] [2 1] + 10 mixed gamma 0.3
-# agg G.Spliced01 1 claim 50 xs 0 sev lognorm 10 cv [0.2 0.4 0.6 0.8 1.0] wts [.2 .3 .3 .15 .05] spliced gamma 0.3 note{shared mixing, compare audit and report dfs}
# Limit profiles
# ==============
@@ -174,7 +172,6 @@ agg I.Blend10 [500 800 200] loss sev lognorm 10 c
agg I.Blend11 [1000 2000 500] prem at [.8 .7 .5] lr sev lognorm 10 cv [.2 .35 .5] wts [1/2 3/8 1/8] mixed gamma 0.5 note{log2=17;}
agg I.Blend12 [500 800 200] loss sev lognorm 10 cv [.2 .35 .5] wts=3 mixed gamma 0.5
-
# Reinsurance
# ===========
agg J.Re01 5 claims 100 xs 0 sev lognorm 10 cv .75 occurrence net of 50% so 5 xs 0 and 5 po 15 xs 5 and 30 xs 20 poisson
diff --git a/aggregate/decl_pygments.py b/aggregate/decl_pygments.py
index f3fa5fe..f892332 100644
--- a/aggregate/decl_pygments.py
+++ b/aggregate/decl_pygments.py
@@ -13,21 +13,6 @@
__all__ = ['AggLexer']
-# def colorize(code):
-# # step 2: apply custom style
-# # embed Style inside HTML (self-contained, no external CSS-file
-# # formatter.noclasses = True # inline style to each element directly
-# formatter = HtmlFormatter(style='monokai', full=True)
-# return highlight(code, AggLexer(), formatter)
-
-
-# def rawhtml(code):
-# formatter = HtmlFormatter(style='monokai', full=False)
-# return highlight(code, AggLexer(), formatter)
-
-
-# define custom style -> see older version; don't want to do this
-
class AggLexer(RegexLexer):
"""
Aggregate program language lexer. (Based on Python lexer. )
diff --git a/aggregate/distributions.py b/aggregate/distributions.py
index 3bedbb1..0f9f713 100644
--- a/aggregate/distributions.py
+++ b/aggregate/distributions.py
@@ -2160,12 +2160,12 @@ def apply_agg_reins(self, debug=False, padding=1, tilt_vector=None):
logger.info(f'Applying agg reins to {self.name}\tOld mean and cv= {_m:,.3f}\t{_m:,.3f}\n'
f'New mean and cv = {_m2:,.3f}\t{_cv2:,.3f}')
- def reinsurance_description(self, kind='both', width=70):
+ def reinsurance_description(self, kind='both', width=0):
"""
Text description of the reinsurance.
:param kind: both, occ, or agg
- :param width: width of text for textwrap.fill
+ :param width: width of text for textwrap.fill; omitted if width==0
"""
ans = []
if self.occ_reins is not None and kind in ['occ', 'both']:
@@ -2173,12 +2173,12 @@ def reinsurance_description(self, kind='both', width=70):
ra = []
for (s, y, a) in self.occ_reins:
if np.isinf(y):
- ra.append(f'{s:,.2%} share of unlimited xs {a:,.2f}')
+ ra.append(f'{s:,.0%} share of unlimited xs {a:,.0f}')
else:
if s == y:
- ra.append(f'{y:,.2f} xs {a:,.2f}')
+ ra.append(f'{y:,.0f} xs {a:,.0f}')
else:
- ra.append(f'{s:,.2%} share of {y:,.2f} xs {a:,.2f}')
+ ra.append(f'{s:,.0%} share of {y:,.0f} xs {a:,.0f}')
ans.append(' and '.join(ra))
ans.append('per occurrence')
if self.agg_reins is not None and kind in ['agg', 'both']:
@@ -2188,12 +2188,12 @@ def reinsurance_description(self, kind='both', width=70):
ra = []
for (s, y, a) in self.agg_reins:
if np.isinf(y):
- ra.append(f'{s:,.2%} share of unlimited xs {a:,.2f}')
+ ra.append(f'{s:,.0%} share of unlimited xs {a:,.0f}')
else:
if s == y:
- ra.append(f'{y:,.2f} xs {a:,.2f}')
+ ra.append(f'{y:,.0f} xs {a:,.0f}')
else:
- ra.append(f'{s:,.2%} share of {y:,.2f} xs {a:,.2f}')
+ ra.append(f'{s:,.0%} share of {y:,.0f} xs {a:,.0f}')
ans.append(' and '.join(ra))
ans.append('in the aggregate.')
if len(ans):
@@ -2204,7 +2204,8 @@ def reinsurance_description(self, kind='both', width=70):
reins = ' '.join(ans)
else:
reins = 'No reinsurance'
- reins = fill(reins, width)
+ if width:
+ reins = fill(reins, width)
return reins
def reinsurance_kinds(self):
diff --git a/aggregate/extensions/test_suite.py b/aggregate/extensions/test_suite.py
index 4e934a7..794cc07 100644
--- a/aggregate/extensions/test_suite.py
+++ b/aggregate/extensions/test_suite.py
@@ -1,101 +1,95 @@
# code for running test cases, producing HTML, etc.
from .. import pprint_ex
-
-# from ..aggregate.utilities import iman_conover, mu_sigma_from_mean_cv
-# # from aggregate.utils import rearrangement_algorithm_max_VaR
-# from .. aggregate.utilities import random_corr_matrix
+from .. import build as build_uw
import logging
import matplotlib.pyplot as plt
from pathlib import Path
import re
-
logger = logging.getLogger(__name__)
class TestSuite(object):
- p = None
- build = None
- tests = ''
-
- @classmethod
- def __init__(cls, build=None, out_dir_name=''):
+ def __init__(self, build_in=None, fn='test_suite.agg', out_dir_name=''):
"""
Run test suite fn. Create specified objects. Save graphics and info to HTML. Wrap
HTML with template.
TODO: convert wrapping to Jinja!
- To run whole test_suite
- ::
+ To run whole test_suite::
python -m aggregate.extensions.test_suite
+ :param build_in: build object, allows input custom build object
+ :param fn: test suite file name, default test_suite.agg
+ :param out_dir_name: output directory name, default site_dir/generated
"""
- if build is None:
- from .. import build
-
- cls.build = build
+ self.build = build_in if build_in else build_uw
if out_dir_name != '':
- cls.p = Path(out_dir_name)
- if cls.p.exists() is False:
+ self.out_dir = Path(out_dir_name)
+ if self.out_dir.exists() is False:
raise FileExistsError(f'Directory {out_dir_name} does not exist.')
else:
- cls.p = cls.build.site_dir.parent / 'generated'
- cls.p.mkdir(exist_ok=True)
- (cls.p / "img").mkdir(exist_ok=True)
+ self.out_dir = self.build.site_dir.parent / 'generated'
+ self.out_dir.mkdir(exist_ok=True)
+ (self.out_dir / "img").mkdir(exist_ok=True)
- logger.info(f'Output directory {cls.p.resolve()}')
+ logger.info(f'Output directory {self.out_dir.resolve()}')
- # extract from comments; this is just FYI
- fn = 'test_suite.agg'
- suite = build.default_dir / fn
+ suite = self.build.default_dir / fn
+ assert suite.exists(), f'Requested test suite file {suite} does not exist.'
txt = suite.read_text(encoding='utf-8')
tests = [i for i in txt.split('\n') if re.match(r'# [A-Z]\.', i)]
- cls.tests = [i.replace("# ", "").split('. ') for i in tests]
+ self.tests = [i.replace("# ", "").split('. ') for i in tests]
- @classmethod
- def run(cls, regex, title, fig_prefix, fig_format='svg', fig_size=(8,2.4), **kwargs):
+ def run(self, regex, title, filename, browse=False, fig_format='svg', fig_size=(8,2.4), **kwargs):
"""
+ Run all tests matching regex. Save graphics and info to HTML.
+ Wrap HTML with template. To run whole test_suite use::
+
+ python -m aggregate.extensions.test_suite
:param regex: regex of tests to run, e.g., 'agg [ABC]\. '
:param title: title for blob
- :param fig_prefix: file name prefix for saved immage files (convenience)
+ :param filename: file name prefix for saved immage files (convenience)
+ :param browse: open browser to output file
:param fig_format: html or markdown (md); html uses svg output, markdown uses pdf
:param fig_size:
:param kwargs: passed to savefig
"""
- logger.warning(f'figure prefix = {fig_prefix}')
-
ans = []
- for n in cls.build.qshow(regex).index:
- a = cls.build(n)
+ for n in self.build.qshow(regex, tacit=False).index:
+ a = self.build(n)
ans.append(a.html_info_blob().replace('h3>', 'h2>'))
- ans.append(pprint_ex(a.program, 50, True, True))
- ans.append(cls.style_df(a.describe).to_html())
+ ans.append(pprint_ex(a.program, 50, True))
+ ans.append(self.style_df(a.describe).to_html())
ans.append('
')
- fn = cls.p / f'img/{fig_prefix}_tmp_{hash(a):0x}.{fig_format}'
+ fn = self.out_dir / f'img/{filename}_tmp_{hash(a):0x}.{fig_format}'
a.plot(figsize=fig_size)
a.figure.savefig(fn, **kwargs)
ans.append(f'})
')
plt.close(a.figure)
logger.warning(f'Created {n}, mean {a.agg_m:.2f}')
-
- blob = '\n'.join(ans)
- fn = cls.p / f'{fig_prefix}.html'
+ blob = '\n'.join([i if type(i)==str else i.data for i in ans])
+ fn = self.out_dir / f'{filename}.html'
fn.write_text(blob, encoding='utf-8')
- fn2 = cls.p / f'{fn.stem}_wrapped.html'
- fn3 = cls.build.template_dir / 'test_suite_template.html'
+ fn2 = self.out_dir / f'{fn.stem}_wrapped.html'
+ fn3 = self.build.template_dir / 'test_suite_template.html'
# TODO JINJA!
template = fn3.read_text()
template = template.replace('HEADING GOES HERE', title).replace(
'CONTENTHERE', blob)
fn2.write_text(template, encoding='utf-8')
+ logger.info(f'Output written to {fn2.resolve()}')
+ if browse:
+ import webbrowser
+ webbrowser.open(fn2.resolve().as_uri())
@staticmethod
def style_df(df):
@@ -166,7 +160,8 @@ def run_test_suite():
# run all the aggs
# TODO FIX for Portfolios
# t.run(regex=r'^C\.', title='C only', fig_prefix="auto", fig_format='png', dpi=300)
- t.run(regex=r'^[A-KNO]\.', title='Full Test Suite', fig_prefix="auto", fig_format='png', dpi=300)
+ t.run(regex=r'^[A-KNO]', title='Full Test Suite', filename='A_tests', browse=True,
+ fig_format='png', dpi=300)
if __name__ == '__main__':
diff --git a/aggregate/parser.py b/aggregate/parser.py
index 0d74ec4..02e5aee 100644
--- a/aggregate/parser.py
+++ b/aggregate/parser.py
@@ -199,6 +199,8 @@ class UnderwritingParser(Parser):
"""
+ expected_shift_reduce = 16 # Set this to the number of expected shift/reduce conflicts
+
debugfile = None
# uncomment to write detailed grammar rules
# debugfile = Path.home() / 'aggregate/parser/parser.out'
diff --git a/aggregate/underwriter.py b/aggregate/underwriter.py
index 46fb091..1986352 100644
--- a/aggregate/underwriter.py
+++ b/aggregate/underwriter.py
@@ -829,10 +829,11 @@ def qlist(self, regex):
"""
return self.show(regex, kind='', plot=False, describe=False, verbose=True)
- def qshow(self, regex):
+ def qshow(self, regex, tacit=True):
"""
Wrapper for show to just show (display) elements in knowledge that match ``regex``.
- No reutrn value.
+ No reutrn value if tacit, else returns a dataframe.
+
"""
def ff(x):
fs = '{x:120s}'
@@ -843,9 +844,12 @@ def ff(x):
r' note\{[^}]+\}', '').str.replace(' +', ' ') # , flags=re.MULTILINE)
# bit['program'] = bit['program'].str.replace(' ( +)', ' ') #, flags=re.MULTILINE)
# bit['program'] = bit['program'].str.replace(r' note\{[^}]+\}$| *', ' ' ) #, flags=re.MULTILINE)
- qd(bit,
- line_width=160, max_colwidth=130, col_space=15, justify='left',
- max_rows=200, formatters={'program': ff})
+ if tacit:
+ qd(bit,
+ line_width=160, max_colwidth=130, col_space=15, justify='left',
+ max_rows=200, formatters={'program': ff})
+ else:
+ return bit
def show(self, regex, kind='', plot=True, describe=True, logger_level=30, verbose=False, **kwargs):
"""
diff --git a/aggregate/utilities.py b/aggregate/utilities.py
index d271397..bb65e8f 100644
--- a/aggregate/utilities.py
+++ b/aggregate/utilities.py
@@ -2962,7 +2962,7 @@ def moms_analytic(fz, limit, attachment, n, analytic=True):
return ans
-def qd(*argv, accuracy=3, align=True, trim=True, **kwargs):
+def qd(*argv, accuracy=3, align=True, trim=True, ff=None, **kwargs):
"""
Endless quest for a robust display format!
@@ -2973,13 +2973,24 @@ def qd(*argv, accuracy=3, align=True, trim=True, **kwargs):
:param: argv: list of objects to print
:param: accuracy: number of decimal places to display
:param: align: if True, align columns at decimal point (sEngFormatter)
+ :param: trim: if True, trim trailing zeros (sEngFormatter)
+ :param: ff: if not None, use this function to format floats, or 'basic', or 'binary'
:kwargs: passed to pd.DataFrame.to_string for dataframes only. e.g., pass dict of formatters by column.
"""
from .distributions import Aggregate
from .portfolio import Portfolio
# ff = sEngFormatter(accuracy=accuracy - (2 if align else 0), min_prefix=0, max_prefix=12, align=align, trim=trim)
- ff = kwargs.pop('ff', lambda x: f'{x:.5g}')
+ if ff is None:
+ ff = lambda x: f'{x:.5g}'
+ elif ff == 'basic':
+ ff = lambda x: f'{x:.1%}' if x < 1 else f'{x:12,.0f}'
+ elif ff == 'int_ratio':
+ def format_function(x):
+ ir = np.round(x, 13).as_integer_ratio()
+ return f'{int(x)}' if x in [0, 1] else f' {ir[0]}/{ir[1]}'
+
+ ff = format_function
# split output
for x in argv:
if isinstance(x, (Aggregate, Portfolio)):
diff --git a/docs/conf.py b/docs/conf.py
index 2662e27..bc54576 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -14,6 +14,9 @@
sys.path.insert(0, os.path.abspath('../'))
import aggregate as agg
+# color graphs
+agg.knobble_fonts(True)
+
# graphics defaults - better res graphics
plt.rcParams['figure.dpi'] = 300
diff --git a/pipreq_requirements.txt b/pipreq_requirements.txt
new file mode 100644
index 0000000..2200f15
--- /dev/null
+++ b/pipreq_requirements.txt
@@ -0,0 +1,10 @@
+cycler>=0.12.1
+ipython>=8.17.2
+Jinja2>=3.1.2
+matplotlib>=3.8.2
+numpy>=1.26.3
+pandas>=2.1.4
+psutil>=5.9.6
+Pygments>=2.16.1
+scipy>=1.11.4
+titlecase>=2.4.1
diff --git a/pyproject.toml b/pyproject.toml
index aaea639..ea504bc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,10 +33,9 @@ dependencies = [
"numpy>=1.26",
"pandas>=2.1",
"psutil",
+ "Pygments",
"scipy>=1.11",
-# "sly",
"titlecase",
- "pygments"
]
license = {text = "BSD-3-Clause"}
requires-python = ">=3.10"
@@ -47,7 +46,7 @@ Documentation = "https://aggregate.readthedocs.io/en/latest/"
[project.optional-dependencies]
dev = [
- "docutils==0.16",
+ "docutils<0.17",
"jupyter-sphinx",
"nbsphinx",
"pickleshare",