From b4a94bbe8484a2b0fe3c5e94339928545a81499a Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Mon, 25 Sep 2023 15:31:07 -0700 Subject: [PATCH 01/14] add allocation package this has all the algorithms currently on portfolio tree the tests in the proprietary package depend on other code so I am just copying the implementation till I can refactor the tests to be open source friendly --- allocation/constant.go | 25 +++++ allocation/equal.go | 21 ++++ allocation/functions.go | 42 ++++++++ allocation/inverse_variance.go | 44 +++++++++ allocation/optimizer_internal.go | 95 +++++++++++++++++++ allocation/risk.go | 46 +++++++++ allocation/volatility.go | 75 +++++++++++++++ backtest/backtestconfig/weight_functions.go | 34 ------- .../backtestconfig/weight_functions_test.go | 12 --- backtest/run_benchmark_test.go | 5 +- backtest/run_test.go | 45 +++++---- fs_test.go | 5 +- go.mod | 3 +- go.sum | 2 + portfolio.go | 86 ++++++++--------- portfolio_test.go | 26 +++-- 16 files changed, 440 insertions(+), 126 deletions(-) create mode 100644 allocation/constant.go create mode 100644 allocation/equal.go create mode 100644 allocation/functions.go create mode 100644 allocation/inverse_variance.go create mode 100644 allocation/optimizer_internal.go create mode 100644 allocation/risk.go create mode 100644 allocation/volatility.go delete mode 100644 backtest/backtestconfig/weight_functions.go delete mode 100644 backtest/backtestconfig/weight_functions_test.go diff --git a/allocation/constant.go b/allocation/constant.go new file mode 100644 index 0000000..0ac66fb --- /dev/null +++ b/allocation/constant.go @@ -0,0 +1,25 @@ +package allocation + +import ( + "context" + "time" + + "github.com/portfoliotree/portfolio/returns" +) + +const ConstantWeightsAlgorithmName = "Constant Weights" + +type ConstantWeights struct { + weights []float64 +} + +func (cw *ConstantWeights) Name() string { return ConstantWeightsAlgorithmName } + +func (cw *ConstantWeights) PolicyWeights(_ context.Context, _ time.Time, _ returns.Table, ws []float64) ([]float64, error) { + copy(ws, cw.weights) + return ws, nil +} + +func (cw *ConstantWeights) SetWeights(in []float64) { + cw.weights = in +} diff --git a/allocation/equal.go b/allocation/equal.go new file mode 100644 index 0000000..5ddebbf --- /dev/null +++ b/allocation/equal.go @@ -0,0 +1,21 @@ +package allocation + +import ( + "context" + "time" + + "github.com/portfoliotree/portfolio/returns" +) + +const EqualWeightsAlgorithmName = "Equal Weights" + +type EqualWeights struct{} + +func (*EqualWeights) Name() string { return EqualWeightsAlgorithmName } + +func (*EqualWeights) PolicyWeights(_ context.Context, _ time.Time, _ returns.Table, ws []float64) ([]float64, error) { + for i := range ws { + ws[i] = 1.0 / float64(len(ws)) + } + return ws, nil +} diff --git a/allocation/functions.go b/allocation/functions.go new file mode 100644 index 0000000..110d0d4 --- /dev/null +++ b/allocation/functions.go @@ -0,0 +1,42 @@ +package allocation + +import ( + "golang.org/x/exp/slices" + + "github.com/portfoliotree/portfolio/backtest" +) + +type Algorithm interface { + backtest.PolicyWeightCalculator + Name() string +} + +func NewDefaultAlgorithmsList() []Algorithm { + return []Algorithm{ + new(ConstantWeights), + new(EqualWeights), + new(EqualInverseVariance), + new(EqualRiskContribution), + new(EqualVolatility), + new(EqualInverseVolatility), + } +} + +func AlgorithmNames(algorithmOptions []Algorithm) []string { + names := make([]string, 0, len(algorithmOptions)) + for _, alg := range algorithmOptions { + names = append(names, alg.Name()) + } + slices.Sort(names) + names = slices.Compact(names) + return names +} + +type WeightSetter interface { + SetWeights([]float64) +} + +func AlgorithmRequiresWeights(alg Algorithm) bool { + _, ok := alg.(WeightSetter) + return ok +} diff --git a/allocation/inverse_variance.go b/allocation/inverse_variance.go new file mode 100644 index 0000000..ed8c85c --- /dev/null +++ b/allocation/inverse_variance.go @@ -0,0 +1,44 @@ +package allocation + +import ( + "context" + "math" + "time" + + "github.com/portfoliotree/portfolio/returns" +) + +type EqualInverseVariance struct{} + +func (cw *EqualInverseVariance) Name() string { return "Equal Inverse Variance" } + +func (*EqualInverseVariance) PolicyWeights(_ context.Context, _ time.Time, assetReturns returns.Table, ws []float64) ([]float64, error) { + if isOnlyZeros(ws) { + for i := range ws { + ws[i] = 1.0 + } + scaleToUnitRange(ws) + } + + err := ensureEnoughReturns(assetReturns) + if err != nil { + return ws, err + } + + assetRisks := assetReturns.RisksFromStdDev() + for i := range assetRisks { + assetRisks[i] = 1.0 / math.Pow(assetRisks[i], 2) + } + + sumOfAssetRisks := 0.0 + for i := range assetRisks { + sumOfAssetRisks += assetRisks[i] + } + + newWeights := make([]float64, len(assetRisks)) + for i := range assetRisks { + newWeights[i] = assetRisks[i] / sumOfAssetRisks + } + + return newWeights, nil +} diff --git a/allocation/optimizer_internal.go b/allocation/optimizer_internal.go new file mode 100644 index 0000000..7c952ee --- /dev/null +++ b/allocation/optimizer_internal.go @@ -0,0 +1,95 @@ +package allocation + +import ( + "context" + "errors" + + "gonum.org/v1/gonum/optimize" + + "github.com/portfoliotree/portfolio/returns" +) + +const ( + maxTries = 50_000 + skipContextCheckCount = 500 + preCancelCheckTries = 10_000 +) + +func checkTries(ctx context.Context, try int) error { + switch { + case try > preCancelCheckTries && try%skipContextCheckCount == 0: + return ctx.Err() + case try > maxTries: + return errors.New("reached max tries to calculate policy") + default: + return nil + } +} + +func optWeights(ctx context.Context, weights []float64, fn func(ws []float64) float64) error { + var ( + try = 0 + m = &optimize.NelderMead{} + s = &optimize.Settings{ + Converger: &optimize.FunctionConverge{ + Absolute: 1e-10, + Relative: 1, + Iterations: 1000, + }, + } + ws = make([]float64, len(weights)) + p = optimize.Problem{ + Func: func(x []float64) float64 { + copy(ws, x) + scaleToUnitRange(ws) + return fn(ws) + }, + Status: func() (optimize.Status, error) { + err := checkTries(ctx, try) + if err != nil { + return optimize.RuntimeLimit, err + } + try++ + return optimize.NotTerminated, nil + }, + } + ) + optResult, err := optimize.Minimize(p, weights, s, m) + if err != nil { + return err + } + + copy(weights, optResult.X) + scaleToUnitRange(weights) + + return nil +} + +func ensureEnoughReturns(assetReturns returns.Table) error { + if assetReturns.NumberOfColumns() == 0 || assetReturns.NumberOfRows() < 2 { + return errors.New("not enough data") + } + return nil +} + +func isOnlyZeros(a []float64) bool { + for _, v := range a { + if v != 0 { + return false + } + } + return true +} + +func scaleToUnitRange(list []float64) { + sum := 0.0 + for _, v := range list { + sum += v + } + if sum == 0 { + return + } + for i := range list { + list[i] /= sum + } +} diff --git a/allocation/risk.go b/allocation/risk.go new file mode 100644 index 0000000..5df61fd --- /dev/null +++ b/allocation/risk.go @@ -0,0 +1,46 @@ +package allocation + +import ( + "context" + "math" + "time" + + "github.com/portfoliotree/portfolio/calculations" + "github.com/portfoliotree/portfolio/returns" +) + +type EqualRiskContribution struct{} + +func (*EqualRiskContribution) Name() string { return "Equal Risk Contribution" } + +func (*EqualRiskContribution) PolicyWeights(ctx context.Context, _ time.Time, assetReturns returns.Table, ws []float64) ([]float64, error) { + if isOnlyZeros(ws) { + for i := range ws { + ws[i] = 1.0 + } + scaleToUnitRange(ws) + } + + err := ensureEnoughReturns(assetReturns) + if err != nil { + return ws, err + } + + assetRisks := assetReturns.RisksFromStdDev() + + target := 1.0 / float64(len(assetRisks)) + + cm := assetReturns.CorrelationMatrix() + + weights := make([]float64, len(ws)) + copy(weights, ws) + + return weights, optWeights(ctx, weights, func(ws []float64) float64 { + _, _, riskWeights := calculations.RiskFromRiskContribution(assetRisks, ws, cm) + var diff float64 + for i := range riskWeights { + diff += math.Abs(target - riskWeights[i]) + } + return diff + }) +} diff --git a/allocation/volatility.go b/allocation/volatility.go new file mode 100644 index 0000000..3326d35 --- /dev/null +++ b/allocation/volatility.go @@ -0,0 +1,75 @@ +package allocation + +import ( + "context" + "time" + + "github.com/portfoliotree/portfolio/returns" +) + +type EqualVolatility struct{} + +func (*EqualVolatility) Name() string { return "Equal Volatility" } + +func (*EqualVolatility) PolicyWeights(_ context.Context, _ time.Time, assetReturns returns.Table, ws []float64) ([]float64, error) { + if isOnlyZeros(ws) { + for i := range ws { + ws[i] = 1.0 + } + scaleToUnitRange(ws) + } + + err := ensureEnoughReturns(assetReturns) + if err != nil { + return ws, err + } + + assetRisks := assetReturns.RisksFromStdDev() + + sumOfAssetRisks := 0.0 + for i := range assetRisks { + sumOfAssetRisks += assetRisks[i] + } + + newWeights := make([]float64, len(assetRisks)) + for i := range assetRisks { + newWeights[i] = assetRisks[i] / sumOfAssetRisks + } + + return newWeights, nil +} + +type EqualInverseVolatility struct{} + +func (*EqualInverseVolatility) Name() string { return "Equal Inverse Volatility" } + +func (*EqualInverseVolatility) PolicyWeights(_ context.Context, _ time.Time, assetReturns returns.Table, ws []float64) ([]float64, error) { + if isOnlyZeros(ws) { + for i := range ws { + ws[i] = 1.0 + } + scaleToUnitRange(ws) + } + + err := ensureEnoughReturns(assetReturns) + if err != nil { + return ws, err + } + + assetRisks := assetReturns.RisksFromStdDev() + for i := range assetRisks { + assetRisks[i] = 1.0 / assetRisks[i] + } + + sumOfAssetRisks := 0.0 + for i := range assetRisks { + sumOfAssetRisks += assetRisks[i] + } + + newWeights := make([]float64, len(assetRisks)) + for i := range assetRisks { + newWeights[i] = assetRisks[i] / sumOfAssetRisks + } + + return newWeights, nil +} diff --git a/backtest/backtestconfig/weight_functions.go b/backtest/backtestconfig/weight_functions.go deleted file mode 100644 index b3e631d..0000000 --- a/backtest/backtestconfig/weight_functions.go +++ /dev/null @@ -1,34 +0,0 @@ -package backtestconfig - -import ( - "context" - "time" - - "github.com/portfoliotree/portfolio/returns" -) - -// PolicyWeightCalculatorFunc can be used to wrap a function and pass it into Run as a PolicyWeightCalculator -type PolicyWeightCalculatorFunc func(ctx context.Context, today time.Time, assets returns.Table, currentWeights []float64) ([]float64, error) - -func (p PolicyWeightCalculatorFunc) PolicyWeights(ctx context.Context, today time.Time, assets returns.Table, currentWeights []float64) ([]float64, error) { - return p(ctx, today, assets, currentWeights) -} - -type ConstantWeights []float64 - -func (targetWeights ConstantWeights) PolicyWeights(_ context.Context, _ time.Time, _ returns.Table, ws []float64) ([]float64, error) { - copy(ws, targetWeights) - return ws, nil -} - -type EqualWeights struct{} - -func (EqualWeights) PolicyWeights(_ context.Context, _ time.Time, _ returns.Table, ws []float64) ([]float64, error) { - for i := range ws { - ws[i] = 1.0 / float64(len(ws)) - } - return ws, nil -} - -// Additional weight functions are maintained in portfoliotree.com proprietary code. -// If you'd like to read the code, feel free to ask us at support@portfoliotree.com, we are willing to share pseudocode. diff --git a/backtest/backtestconfig/weight_functions_test.go b/backtest/backtestconfig/weight_functions_test.go deleted file mode 100644 index 3824fad..0000000 --- a/backtest/backtestconfig/weight_functions_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package backtestconfig_test - -import ( - "github.com/portfoliotree/portfolio/backtest" - "github.com/portfoliotree/portfolio/backtest/backtestconfig" -) - -var ( - _ backtest.PolicyWeightCalculator = backtestconfig.ConstantWeights{} - _ backtest.PolicyWeightCalculator = backtestconfig.EqualWeights{} - _ backtest.PolicyWeightCalculator = backtestconfig.PolicyWeightCalculatorFunc(nil) -) diff --git a/backtest/run_benchmark_test.go b/backtest/run_benchmark_test.go index b2a80da..ada24e4 100644 --- a/backtest/run_benchmark_test.go +++ b/backtest/run_benchmark_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/portfoliotree/portfolio" + "github.com/portfoliotree/portfolio/allocation" "github.com/portfoliotree/portfolio/backtest" "github.com/portfoliotree/portfolio/backtest/backtestconfig" "github.com/portfoliotree/portfolio/portfoliotest" @@ -39,14 +40,14 @@ func benchmarkRun(b *testing.B, table returns.Table) { b.Helper() end := table.LastTime() start := table.FirstTime() - fn := backtestconfig.EqualWeights{} + alg := new(allocation.EqualWeights) lookback := backtestconfig.OneQuarterWindow.Function rebalance := backtestconfig.Daily() updatePolicyWeights := backtestconfig.Monthly() ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := backtest.Run(ctx, end, start, table, fn, lookback, rebalance, updatePolicyWeights) + _, err := backtest.Run(ctx, end, start, table, alg, lookback, rebalance, updatePolicyWeights) if err != nil { b.Fatal(err) } diff --git a/backtest/run_test.go b/backtest/run_test.go index 806e26e..583337c 100644 --- a/backtest/run_test.go +++ b/backtest/run_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/portfoliotree/portfolio/allocation" "github.com/portfoliotree/portfolio/backtest" "github.com/portfoliotree/portfolio/backtest/backtestconfig" "github.com/portfoliotree/portfolio/returns" @@ -32,7 +33,7 @@ func TestSpec_Run(t *testing.T) { {Time: date("2021-01-02"), Value: 0.2}, {Time: date("2021-01-01"), Value: 0.1}, }}) - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -62,7 +63,7 @@ func TestSpec_Run(t *testing.T) { policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() ws := []float64{.715, .315} - _, err := backtest.Run(context.Background(), date("2021-01-04"), date("2021-01-01"), assets, backtestconfig.PolicyWeightCalculatorFunc(func(_ context.Context, _ time.Time, _ returns.Table, currentWeights []float64) ([]float64, error) { + _, err := backtest.Run(context.Background(), date("2021-01-04"), date("2021-01-01"), assets, allocationFunction(func(_ context.Context, _ time.Time, _ returns.Table, currentWeights []float64) ([]float64, error) { return ws, nil }), windowFunc, rebalanceIntervalFunc, policyUpdateIntervalFunc) assert.NoError(t, err) @@ -78,7 +79,7 @@ func TestSpec_Run(t *testing.T) { assert.Error(t, err) }) t.Run("end date does not have a return", func(t *testing.T) { - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -96,7 +97,7 @@ func TestSpec_Run(t *testing.T) { t.Run("with no returns", func(t *testing.T) { assets := returns.Table{} - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -110,7 +111,7 @@ func TestSpec_Run(t *testing.T) { }) t.Run("when there is one asset", func(t *testing.T) { - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.OneDayWindow.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -132,7 +133,7 @@ func TestSpec_Run(t *testing.T) { }) t.Run("when called repeatedly", func(t *testing.T) { - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.OneDayWindow.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -178,7 +179,7 @@ func TestSpec_Run(t *testing.T) { <-c cancel() }() - alg := backtestconfig.PolicyWeightCalculatorFunc(func(ctx context.Context, _ time.Time, _ returns.Table, ws []float64) (targetWeights []float64, err error) { + alg := allocationFunction(func(ctx context.Context, _ time.Time, _ returns.Table, ws []float64) (targetWeights []float64, err error) { close(c) <-ctx.Done() return ws, ctx.Err() @@ -213,7 +214,7 @@ func TestSpec_Run(t *testing.T) { } assets := returns.NewTable([]returns.List{asset1, asset2}) - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.OneDayWindow.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -247,12 +248,12 @@ func TestSpec_Run(t *testing.T) { {Time: date("2021-04-15"), Value: 0}, } assets := returns.NewTable([]returns.List{asset1, asset2}) - - alg := backtestconfig.PolicyWeightCalculatorFunc(func(ctx context.Context, t time.Time, assetReturns returns.Table, currentWeights []float64) ([]float64, error) { + fallback := testAlgorithm() + alg := allocationFunction(func(ctx context.Context, t time.Time, assetReturns returns.Table, currentWeights []float64) ([]float64, error) { if t.Before(date("2021-04-20")) { return nil, backtest.ErrorNotEnoughData{} } - return backtestconfig.EqualWeights{}.PolicyWeights(ctx, t, assetReturns, currentWeights) + return fallback.PolicyWeights(ctx, t, assetReturns, currentWeights) }) windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -290,7 +291,7 @@ func TestSpec_Run(t *testing.T) { } assets := returns.NewTable([]returns.List{asset1, asset2}) - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.OneWeekWindow.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -327,7 +328,7 @@ func TestSpec_Run(t *testing.T) { } callCount := 0 - alg := backtestconfig.PolicyWeightCalculatorFunc(func(_ context.Context, tm time.Time, assetReturns returns.Table, currentWeights []float64) ([]float64, error) { + alg := allocationFunction(func(_ context.Context, tm time.Time, assetReturns returns.Table, currentWeights []float64) ([]float64, error) { callCount++ assert.Equalf(t, assetReturns.NumberOfColumns(), 1, "call count %d", callCount) for c := 0; c < assetReturns.NumberOfColumns(); c++ { @@ -360,7 +361,7 @@ func TestSpec_Run(t *testing.T) { } assert.Lenf(t, rs, 5, "call count %d", callCount) } - return backtestconfig.EqualWeights{}.PolicyWeights(context.Background(), tm, assetReturns, currentWeights) + return (&allocation.EqualWeights{}).PolicyWeights(context.Background(), tm, assetReturns, currentWeights) }) windowFunc := backtestconfig.OneWeekWindow.Function @@ -403,7 +404,7 @@ func TestSpec_Run_weightHistory(t *testing.T) { assets := returns.NewTable([]returns.List{asset}) - alg := backtestconfig.PolicyWeightCalculatorFunc(randomWeights) + alg := allocationFunction(randomWeights) windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() @@ -493,7 +494,7 @@ func TestSpec_Run_weightHistory(t *testing.T) { } assets := returns.NewTable([]returns.List{asset1, asset2}) - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalWeekly.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalMonthly.CheckFunction() @@ -549,7 +550,7 @@ func TestSpec_Run_weightHistory(t *testing.T) { } assets := returns.NewTable([]returns.List{asset1, asset2}) - alg := backtestconfig.EqualWeights{} + alg := testAlgorithm() windowFunc := backtestconfig.WindowNotSet.Function rebalanceIntervalFunc := backtestconfig.IntervalDaily.CheckFunction() policyUpdateIntervalFunc := backtestconfig.IntervalWeekly.CheckFunction() @@ -574,3 +575,13 @@ func date(str string) time.Time { d, _ := time.Parse(time.DateOnly, str) return d } + +type allocationFunction func(_ context.Context, _ time.Time, _ returns.Table, currentWeights []float64) (targetWeights []float64, err error) + +func (function allocationFunction) PolicyWeights(ctx context.Context, today time.Time, assets returns.Table, ws []float64) (targetWeights []float64, err error) { + return function(ctx, today, assets, ws) +} + +func testAlgorithm() allocation.Algorithm { + return new(allocation.EqualWeights) +} diff --git a/fs_test.go b/fs_test.go index 37d4865..b420069 100644 --- a/fs_test.go +++ b/fs_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/portfoliotree/portfolio" + "github.com/portfoliotree/portfolio/allocation" ) func TestParseSpecificationFile(t *testing.T) { @@ -36,7 +37,7 @@ func TestParseSpecificationFile(t *testing.T) { }, Policy: portfolio.Policy{ Weights: []float64{60, 40}, - WeightsAlgorithm: portfolio.PolicyAlgorithmConstantWeights, + WeightsAlgorithm: allocation.ConstantWeightsAlgorithmName, RebalancingInterval: "Quarterly", }, Filepath: "examples/60-40_portfolio.yml", @@ -59,7 +60,7 @@ func TestParseSpecificationFile(t *testing.T) { }, Policy: portfolio.Policy{ RebalancingInterval: "Quarterly", - WeightsAlgorithm: portfolio.PolicyAlgorithmEqualWeights, + WeightsAlgorithm: allocation.ConstantWeightsAlgorithmName, }, Filepath: "examples/maang_portfolio.yml", }, diff --git a/go.mod b/go.mod index 10360d0..a35e991 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/portfoliotree/round v0.0.0-20230629094931-8afd986aa2f1 github.com/stretchr/testify v1.8.4 go.mongodb.org/mongo-driver v1.12.1 + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb gonum.org/v1/gonum v0.13.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -15,6 +16,6 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect + golang.org/x/tools v0.7.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index b7b3951..41380c9 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= diff --git a/portfolio.go b/portfolio.go index b065364..d89fb2a 100644 --- a/portfolio.go +++ b/portfolio.go @@ -13,6 +13,7 @@ import ( "golang.org/x/exp/slices" "gopkg.in/yaml.v3" + "github.com/portfoliotree/portfolio/allocation" "github.com/portfoliotree/portfolio/backtest" "github.com/portfoliotree/portfolio/backtest/backtestconfig" "github.com/portfoliotree/portfolio/returns" @@ -74,11 +75,9 @@ func ParseSpecifications(r io.Reader) ([]Specification, error) { default: return result, fmt.Errorf("incorrect specification type got %q but expected %q", spec.Type, portfolioTypeName) } + pf := spec.Spec pf.setDefaultPolicyWeightAlgorithm() - if err := pf.ensureEqualNumberOfWeightsAndAssets(); err != nil { - return result, err - } result = append(result, pf) } } @@ -91,61 +90,27 @@ func (pf *Specification) RemoveAsset(index int) error { return nil } -func (pf *Specification) Backtest(ctx context.Context, assets returns.Table, weightsAlgorithm backtestconfig.PolicyWeightCalculatorFunc) (backtest.Result, error) { - return pf.BacktestWithStartAndEndTime(ctx, time.Time{}, time.Time{}, assets, weightsAlgorithm) +func (pf *Specification) Backtest(ctx context.Context, assets returns.Table, alg allocation.Algorithm) (backtest.Result, error) { + return pf.BacktestWithStartAndEndTime(ctx, time.Time{}, time.Time{}, assets, alg) } -const ( - PolicyAlgorithmEqualWeights = "EqualWeights" - PolicyAlgorithmConstantWeights = "ConstantWeights" -) - func (pf *Specification) setDefaultPolicyWeightAlgorithm() { - if pf.Policy.WeightsAlgorithm != "" { - return - } if len(pf.Policy.Weights) > 0 { - pf.Policy.WeightsAlgorithm = PolicyAlgorithmConstantWeights + pf.Policy.WeightsAlgorithm = (*allocation.ConstantWeights)(nil).Name() } else { - pf.Policy.WeightsAlgorithm = PolicyAlgorithmEqualWeights + pf.Policy.WeightsAlgorithm = (*allocation.EqualWeights)(nil).Name() } } -func (pf *Specification) ensureEqualNumberOfWeightsAndAssets() error { - switch pf.Policy.WeightsAlgorithm { - case PolicyAlgorithmConstantWeights: - if len(pf.Policy.Weights) != len(pf.Assets) { - return fmt.Errorf("the number of assets and number of weights must be equal: len(assets) is %d and len(weights) is %d", len(pf.Assets), len(pf.Policy.Weights)) +func (pf *Specification) BacktestWithStartAndEndTime(ctx context.Context, start, end time.Time, assets returns.Table, alg allocation.Algorithm) (backtest.Result, error) { + if alg == nil { + var err error + alg, err = pf.Algorithm(nil) + if err != nil { + return backtest.Result{}, err } } - return nil -} - -func (pf *Specification) policyWeightFunction(weights backtestconfig.PolicyWeightCalculatorFunc) (backtestconfig.PolicyWeightCalculatorFunc, error) { - switch pf.Policy.WeightsAlgorithm { - case PolicyAlgorithmEqualWeights: - return backtestconfig.EqualWeights{}.PolicyWeights, nil - case PolicyAlgorithmConstantWeights: - return backtestconfig.ConstantWeights(pf.Policy.Weights).PolicyWeights, nil - default: - if weights == nil { - return nil, fmt.Errorf("policy %q not supported by the backtest runner", pf.Policy.WeightsAlgorithm) - } - return weights, nil - } -} - -func (pf *Specification) BacktestWithStartAndEndTime(ctx context.Context, start, end time.Time, assets returns.Table, weightsFn backtestconfig.PolicyWeightCalculatorFunc) (backtest.Result, error) { - if err := pf.ensureEqualNumberOfWeightsAndAssets(); err != nil { - return backtest.Result{}, err - } - var err error - weightsFn, err = pf.policyWeightFunction(weightsFn) - if err != nil { - return backtest.Result{}, err - } - - return backtest.Run(ctx, end, start, assets, weightsFn, + return backtest.Run(ctx, end, start, assets, alg, pf.Policy.WeightsAlgorithmLookBack.Function, pf.Policy.WeightsUpdatingInterval.CheckFunction(), pf.Policy.RebalancingInterval.CheckFunction(), @@ -263,3 +228,28 @@ func (pf *Specification) filterEmptyAssetIDs() { } pf.Assets = filtered } + +func (pf *Specification) Algorithm(algorithmOptions []allocation.Algorithm) (allocation.Algorithm, error) { + if len(algorithmOptions) == 0 { + algorithmOptions = allocation.NewDefaultAlgorithmsList() + } + + for _, alg := range algorithmOptions { + if alg.Name() != pf.Policy.WeightsAlgorithm { + continue + } + if se, ok := alg.(allocation.WeightSetter); ok { + if len(pf.Policy.Weights) != len(pf.Assets) { + return nil, errAssetAndWeightsLenMismatch(pf) + } + se.SetWeights(slices.Clone(pf.Policy.Weights)) + } + return alg, nil // algorithm is known + } + + return nil, errors.New("unknown algorithm") +} + +func errAssetAndWeightsLenMismatch(spec *Specification) error { + return fmt.Errorf("expected the number of policy weights to be the same as the number of assets got %d but expected %d", len(spec.Policy.Weights), len(spec.Assets)) +} diff --git a/portfolio_test.go b/portfolio_test.go index 6925450..47f093b 100644 --- a/portfolio_test.go +++ b/portfolio_test.go @@ -67,7 +67,7 @@ spec: assets: [ACWI, AGG] policy: weights: [60, 40] - weights_algorithm: PolicyAlgorithmConstantWeights + weights_algorithm: Constant Weights rebalancing_interval: Quarterly ` @@ -80,7 +80,7 @@ spec: // Output: // Name: 60/40 - // Alg: PolicyAlgorithmConstantWeights + // Alg: Constant Weights } func ExampleOpen() { @@ -94,7 +94,7 @@ func ExampleOpen() { // Output: // Name: 60/40 - // Alg: ConstantWeights + // Alg: Constant Weights } func TestParse(t *testing.T) { @@ -119,7 +119,7 @@ func TestParse(t *testing.T) { Name: "the number of assets and policy weights do not match", // language=yaml SpecYAML: `{type: Portfolio, spec: {assets: ["a"], policy: {weights: [1, 2]}}}`, - ErrorStringContains: "the number of assets and number of weights must be equal:", + ErrorStringContains: "expected the number of policy weights to be the same as the number of assets", }, { Name: "component field is invalid", @@ -219,11 +219,11 @@ func TestPortfolio_Backtest(t *testing.T) { Assets: []portfolio.Component{{ID: "AAPL"}}, Policy: portfolio.Policy{ Weights: []float64{50, 50}, - WeightsAlgorithm: portfolio.PolicyAlgorithmConstantWeights, + WeightsAlgorithm: "Constant Weights", }, }, ctx: context.Background(), - ErrorSubstring: "the number of assets and number of weights must be equal:", + ErrorSubstring: "expected the number of policy weights to be the same as the number of assets", }, { Name: "unknown policy algorithm", @@ -235,7 +235,7 @@ func TestPortfolio_Backtest(t *testing.T) { }, }, ctx: context.Background(), - ErrorSubstring: `policy "unknown" not supported by the backtest runner`, + ErrorSubstring: `unknown algorithm`, }, } { t.Run(tt.Name, func(t *testing.T) { @@ -256,9 +256,7 @@ func TestPortfolio_Backtest_custom_function(t *testing.T) { {ID: "AAPL"}, {ID: "GOOG"}, }, - }).Backtest(context.Background(), returns.NewTable([]returns.List{{}}), func(ctx context.Context, today time.Time, assets returns.Table, currentWeights []float64) ([]float64, error) { - return nil, fmt.Errorf("lemon") - }) + }).Backtest(context.Background(), returns.NewTable([]returns.List{{}}), ErrorAlg{}) assert.EqualError(t, err, "lemon") } @@ -469,3 +467,11 @@ func TestPortfolio_RemoveAsset(t *testing.T) { require.Error(t, pf.RemoveAsset(-1)) }) } + +type ErrorAlg struct{} + +func (ErrorAlg) Name() string { return "" } + +func (ErrorAlg) PolicyWeights(ctx context.Context, today time.Time, assets returns.Table, currentWeights []float64) ([]float64, error) { + return nil, fmt.Errorf("lemon") +} From 690b7b26952e679d09e831f3087f9244dae5fe8c Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Mon, 25 Sep 2023 15:50:58 -0700 Subject: [PATCH 02/14] move weights len check to alg implementation --- allocation/constant.go | 4 ++++ portfolio.go | 3 --- portfolio_test.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/allocation/constant.go b/allocation/constant.go index 0ac66fb..3fb91fd 100644 --- a/allocation/constant.go +++ b/allocation/constant.go @@ -2,6 +2,7 @@ package allocation import ( "context" + "errors" "time" "github.com/portfoliotree/portfolio/returns" @@ -16,6 +17,9 @@ type ConstantWeights struct { func (cw *ConstantWeights) Name() string { return ConstantWeightsAlgorithmName } func (cw *ConstantWeights) PolicyWeights(_ context.Context, _ time.Time, _ returns.Table, ws []float64) ([]float64, error) { + if len(cw.weights) != len(ws) { + return nil, errors.New("expected the number of policy weights to be the same as the number of assets") + } copy(ws, cw.weights) return ws, nil } diff --git a/portfolio.go b/portfolio.go index d89fb2a..eb250ba 100644 --- a/portfolio.go +++ b/portfolio.go @@ -239,9 +239,6 @@ func (pf *Specification) Algorithm(algorithmOptions []allocation.Algorithm) (all continue } if se, ok := alg.(allocation.WeightSetter); ok { - if len(pf.Policy.Weights) != len(pf.Assets) { - return nil, errAssetAndWeightsLenMismatch(pf) - } se.SetWeights(slices.Clone(pf.Policy.Weights)) } return alg, nil // algorithm is known diff --git a/portfolio_test.go b/portfolio_test.go index 47f093b..dced264 100644 --- a/portfolio_test.go +++ b/portfolio_test.go @@ -240,7 +240,7 @@ func TestPortfolio_Backtest(t *testing.T) { } { t.Run(tt.Name, func(t *testing.T) { pf := tt.Portfolio - _, err := pf.Backtest(tt.ctx, returns.Table{}, nil) + _, err := pf.Backtest(tt.ctx, returns.NewTable([]returns.List{{}}), nil) if tt.ErrorSubstring == "" { assert.NoError(t, err) } else { From 2889189e52a8a7fa92d4ce0e0dc1fb6ebc6da5d5 Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Sat, 14 Oct 2023 17:52:43 -0700 Subject: [PATCH 03/14] fix lookback function when window is empty --- backtest/backtestconfig/window.go | 3 +++ backtest/backtestconfig/window_test.go | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/backtest/backtestconfig/window.go b/backtest/backtestconfig/window.go index 8f18a4c..949c8d9 100644 --- a/backtest/backtestconfig/window.go +++ b/backtest/backtestconfig/window.go @@ -97,5 +97,8 @@ func (dur Window) Sub(t time.Time) time.Time { } func (dur Window) Function(today time.Time, table returns.Table) returns.Table { + if !dur.IsSet() { + return table.Between(today, table.FirstTime()) + } return table.Between(today, dur.Sub(today)) } diff --git a/backtest/backtestconfig/window_test.go b/backtest/backtestconfig/window_test.go index ac7c0ea..1ee02ab 100644 --- a/backtest/backtestconfig/window_test.go +++ b/backtest/backtestconfig/window_test.go @@ -2,13 +2,16 @@ package backtestconfig_test import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/portfoliotree/portfolio/backtest/backtestconfig" + "github.com/portfoliotree/portfolio/internal/fixtures" + "github.com/portfoliotree/portfolio/returns" ) -func TestDurations_Validate(t *testing.T) { +func TestWindows_Validate(t *testing.T) { for _, d := range backtestconfig.Windows() { t.Run(d.String(), func(t *testing.T) { err := d.Validate() @@ -26,3 +29,20 @@ func TestDurations_Validate(t *testing.T) { assert.Error(t, err) }) } + +func TestWindow_Function(t *testing.T) { + t.Run("not set", func(t *testing.T) { + var zero backtestconfig.Window + + today := fixtures.T(t, fixtures.Day2) + table := returns.NewTable([]returns.List{{ + returns.New(fixtures.T(t, fixtures.Day3), .1), + returns.New(today, .1), + returns.New(fixtures.T(t, fixtures.Day1), .1), + returns.New(fixtures.T(t, fixtures.Day0), .1), + }}) + + result := zero.Function(today, table) + assert.Equal(t, result.FirstTime().Format(time.DateOnly), fixtures.Day0) + }) +} From 07a5441e69c0059aa9e63d201b9b74ea398322e2 Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Sat, 14 Oct 2023 20:38:25 -0700 Subject: [PATCH 04/14] add factors to Metadata --- portfolio.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/portfolio.go b/portfolio.go index eb250ba..8520dc5 100644 --- a/portfolio.go +++ b/portfolio.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "go.mongodb.org/mongo-driver/bson/primitive" "golang.org/x/exp/slices" "gopkg.in/yaml.v3" @@ -19,6 +20,23 @@ import ( "github.com/portfoliotree/portfolio/returns" ) +type Identifier = primitive.ObjectID + +type Document struct { + ID Identifier `json:"_id" yaml:"_id" bson:"_id"` + Type string `json:"type" yaml:"type" bson:"type"` + Metadata Metadata `json:"metadata" yaml:"metadata" bson:"metadata"` + Spec Specification `json:"spec" yaml:"spec" bson:"spec"` +} + +type Metadata struct { + Name string `json:"name,omitempty" yaml:"name,omitempty" bson:"name,omitempty"` + Benchmark Component `json:"benchmark,omitempty" yaml:"benchmark,omitempty" bson:"benchmark,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty" bson:"description,omitempty"` + Privacy string `json:"privacy,omitempty" yaml:"privacy,omitempty" bson:"privacy,omitempty"` + Factors []Identifier `json:"factors,omitempty" yaml:"factors,omitempty" bson:"factors,omitempty"` +} + // Specification models a portfolio. type Specification struct { Name string `yaml:"name"` From e98e6f3eb197424283d7251e6067ee1d9b2ca16a Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Thu, 19 Oct 2023 23:28:10 -0700 Subject: [PATCH 05/14] fix test setup the variable names were a bit weird too --- returns/table_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/returns/table_test.go b/returns/table_test.go index 186a0b1..f4e2faa 100644 --- a/returns/table_test.go +++ b/returns/table_test.go @@ -456,22 +456,26 @@ func TestTable_TimeBefore(t *testing.T) { assert.False(t, hasReturn) }) t.Run("on a Monday", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day2) + require.Equal(t, time.Monday, in.Weekday()) table := returns.NewTable([]returns.List{ {rtn(t, fixtures.LastDay, 0), rtn(t, fixtures.Day2, 0), rtn(t, fixtures.Day1, 0), rtn(t, fixtures.FirstDay, 0)}, {rtn(t, fixtures.LastDay, 0), rtn(t, fixtures.Day2, 0), rtn(t, fixtures.Day1, 0), rtn(t, fixtures.FirstDay, 0)}, }) - after, hasReturn := table.TimeBefore(fixtures.T(t, fixtures.Day2)) + result, hasReturn := table.TimeBefore(in) assert.True(t, hasReturn) - assert.Equal(t, after, fixtures.T(t, fixtures.Day1)) + assert.Equal(t, fixtures.T(t, fixtures.Day1), result) }) t.Run("on a Friday", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day1) + require.Equal(t, in.Weekday(), time.Friday) table := returns.NewTable([]returns.List{ {rtn(t, fixtures.LastDay, 0), rtn(t, fixtures.Day2, 0), rtn(t, fixtures.Day1, 0), rtn(t, fixtures.FirstDay, 0)}, {rtn(t, fixtures.LastDay, 0), rtn(t, fixtures.Day2, 0), rtn(t, fixtures.Day1, 0), rtn(t, fixtures.FirstDay, 0)}, }) - after, hasReturn := table.TimeBefore(fixtures.T(t, fixtures.Day3)) + result, hasReturn := table.TimeBefore(in) assert.True(t, hasReturn) - assert.Equal(t, after, fixtures.T(t, fixtures.Day2)) + assert.Equal(t, fixtures.T(t, fixtures.Day0), result) }) } From b56982fab8b042b6659d823882933bdc023a35be Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Thu, 19 Oct 2023 23:37:34 -0700 Subject: [PATCH 06/14] add table method "ClosestTimeOnOrBefore" --- returns/table.go | 6 +++++ returns/table_test.go | 58 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/returns/table.go b/returns/table.go index 6650ed4..1650c6d 100644 --- a/returns/table.go +++ b/returns/table.go @@ -245,6 +245,12 @@ func (table Table) TimeBefore(tm time.Time) (time.Time, bool) { return next, !next.IsZero() } +func (table Table) ClosestTimeOnOrBefore(tm time.Time) (time.Time, bool) { + index := indexOfClosest(table.times, identity[time.Time], tm) + next := indexOrEmpty(table.times, index) + return next, !next.IsZero() +} + func identity[T any](t T) T { return t } func (table Table) Lists() []List { diff --git a/returns/table_test.go b/returns/table_test.go index f4e2faa..75e75df 100644 --- a/returns/table_test.go +++ b/returns/table_test.go @@ -479,6 +479,64 @@ func TestTable_TimeBefore(t *testing.T) { }) } +func TestTable_ClosestTimeOnOrBefore(t *testing.T) { + t.Run("on a Friday", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day1) + require.Equal(t, in.Weekday(), time.Friday) + table := returns.NewTable([]returns.List{ + {rtn(t, fixtures.LastDay, 0), rtn(t, fixtures.Day2, 0), rtn(t, fixtures.Day1, 0), rtn(t, fixtures.FirstDay, 0)}, + }) + result, hasReturn := table.ClosestTimeOnOrBefore(in) + assert.True(t, hasReturn) + assert.Equal(t, fixtures.T(t, fixtures.Day1), result) + }) + t.Run("exactly between", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day2) + table := returns.NewTable([]returns.List{ + {rtn(t, fixtures.Day3, 0), rtn(t, fixtures.Day1, 0)}, + }) + result, hasReturn := table.ClosestTimeOnOrBefore(in) + assert.True(t, hasReturn) + assert.Equal(t, fixtures.T(t, fixtures.Day1), result) + }) + t.Run("between closer to final day", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day2) + table := returns.NewTable([]returns.List{ + {rtn(t, fixtures.Day3, 0), rtn(t, fixtures.Day0, 0)}, + }) + result, hasReturn := table.ClosestTimeOnOrBefore(in) + assert.True(t, hasReturn) + assert.Equal(t, fixtures.T(t, fixtures.Day0), result) + }) + t.Run("between closer to first day", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day1) + table := returns.NewTable([]returns.List{ + {rtn(t, fixtures.Day3, 0), rtn(t, fixtures.Day0, 0)}, + }) + result, hasReturn := table.ClosestTimeOnOrBefore(in) + assert.True(t, hasReturn) + assert.Equal(t, fixtures.T(t, fixtures.Day0), result) + }) + t.Run("exactly first", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day0) + table := returns.NewTable([]returns.List{ + {rtn(t, fixtures.Day1, 0), rtn(t, fixtures.Day0, 0)}, + }) + result, hasReturn := table.ClosestTimeOnOrBefore(in) + assert.True(t, hasReturn) + assert.Equal(t, fixtures.T(t, fixtures.Day0), result) + }) + t.Run("exactly last", func(t *testing.T) { + in := fixtures.T(t, fixtures.Day1) + table := returns.NewTable([]returns.List{ + {rtn(t, fixtures.Day1, 0), rtn(t, fixtures.Day0, 0)}, + }) + result, hasReturn := table.ClosestTimeOnOrBefore(in) + assert.True(t, hasReturn) + assert.Equal(t, fixtures.T(t, fixtures.Day1), result) + }) +} + func TestTable_Lists(t *testing.T) { table := returns.NewTable([]returns.List{ { /* ,*/ rtn(t, fixtures.Day2, 0.01), rtn(t, fixtures.Day1, -0.01), rtn(t, fixtures.Day0, 0.001)}, From 55dc958e7851f577e5bdfef117266a5067054a9b Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Fri, 20 Oct 2023 01:39:19 -0700 Subject: [PATCH 07/14] do not marshal nil table fields --- returns/table.go | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/returns/table.go b/returns/table.go index 1650c6d..4fa2565 100644 --- a/returns/table.go +++ b/returns/table.go @@ -48,10 +48,7 @@ func (table *Table) UnmarshalBSON(buf []byte) error { } func (table Table) MarshalBSON() ([]byte, error) { - return bson.Marshal(encodedTable{ - Times: table.times, - Values: table.values, - }) + return bson.Marshal(newEncodedTable(table.times, table.values)) } type encodedTable struct { @@ -59,6 +56,24 @@ type encodedTable struct { Values [][]float64 `json:"values" bson:"values"` } +func newEncodedTable(times []time.Time, values [][]float64) encodedTable { + if times == nil { + times = make([]time.Time, 0) + } + if values == nil { + values = make([][]float64, 0) + } + for i := range values { + if values[i] == nil { + values[i] = make([]float64, 0) + } + } + return encodedTable{ + Times: times, + Values: values, + } +} + func (table *Table) UnmarshalJSON(buf []byte) error { var enc encodedTable err := json.Unmarshal(buf, &enc) @@ -68,10 +83,7 @@ func (table *Table) UnmarshalJSON(buf []byte) error { } func (table Table) MarshalJSON() ([]byte, error) { - t := encodedTable{ - Times: table.times, - Values: table.values, - } + t := newEncodedTable(table.times, table.values) err := round.Recursive(t.Values, 6) if err != nil { return nil, err From a76917aa3aaf6af061f8a2e0a3da54b664839e5d Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Fri, 20 Oct 2023 02:29:33 -0700 Subject: [PATCH 08/14] add label field to Component --- component.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/component.go b/component.go index 4fd9863..de31c5f 100644 --- a/component.go +++ b/component.go @@ -10,8 +10,9 @@ import ( ) type Component struct { - Type string `yaml:"type,omitempty"` - ID string `yaml:"id,omitempty"` + Type string `yaml:"type,omitempty" json:"type,omitempty" bson:"type"` + ID string `yaml:"id,omitempty" json:"id,omitempty" bson:"id"` + Label string `yaml:"label,omitempty" json:"label,omitempty" bson:"label"` } var componentExpression = regexp.MustCompile(`^[a-zA-Z0-9.:s]{1,24}$`) From 41277e6d1a67d38f10ac25624ba230719c15e553 Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Sat, 21 Oct 2023 20:15:46 -0700 Subject: [PATCH 09/14] simplify implementation of AddColumn --- returns/table.go | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/returns/table.go b/returns/table.go index 4fa2565..b4b4130 100644 --- a/returns/table.go +++ b/returns/table.go @@ -196,39 +196,23 @@ func (table Table) addAdditionalColumn(list List) Table { list = list.Between(table.LastTime(), table.FirstTime()) updated := table.Between(list.LastTime(), list.FirstTime()) + newValues := make([]float64, len(updated.times)) for _, r := range list { - _, updated = updated.ensureRowForTime(r.Time) + i, found := updated.rowForTime(r.Time) + if !found { + continue + } + newValues[i] = r.Value } - newValues := make([]float64, len(updated.times)) - for i, tm := range updated.times { - value, _ := list.Value(tm) - newValues[i] = value - } updated.values = append(updated.values, newValues) return updated } -func (table Table) ensureRowForTime(tm time.Time) (index int, updated Table) { - for i, et := range table.times { - if et.Equal(tm) { - return i, table - } - if tm.After(et) { - index, updated = i, table - updated.times = append(updated.times[:i], append([]time.Time{tm}, updated.times[i:]...)...) - for j, values := range updated.values { - updated.values[j] = append(values[:i], append([]float64{0}, values[i:]...)...) - } - break - - //// an early return makes the coverage dip below 100% because the - //// empty block outside the loop would never execute. This break - //// is essentially like the following line - // return index, updated - } - } - return index, updated +func (table Table) rowForTime(tm time.Time) (index int, exists bool) { + return slices.BinarySearchFunc(table.times, tm, func(et time.Time, t time.Time) int { + return et.Compare(t) * -1 + }) } func (table Table) FirstTime() time.Time { return indexOrEmpty(table.times, firstIndex(table.times)) } From 5632ea2e2d1b866fa2cd3bafd58aba97b95b241d Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Sat, 21 Oct 2023 23:25:45 -0700 Subject: [PATCH 10/14] test: change in AddColumn behavior --- returns/table_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/returns/table_test.go b/returns/table_test.go index 75e75df..50416af 100644 --- a/returns/table_test.go +++ b/returns/table_test.go @@ -280,6 +280,9 @@ func TestTable_Between(t *testing.T) { func TestTable_AddColumn(t *testing.T) { t.Run("when adding list with an additional row", func(t *testing.T) { + t.Skip(` +AddColumn now does not add a column to the table if the table does not already have a row. +`) table := returns.NewTable([]returns.List{ {rtn(t, fixtures.Day3, .1), rtn(t, fixtures.Day1, .1), rtn(t, fixtures.Day0, .1)}, }) From affa8df52e88248217676c9459fdf05bfaecc106 Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:31:40 -0700 Subject: [PATCH 11/14] use Component for factor list --- portfolio.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/portfolio.go b/portfolio.go index 8520dc5..598090c 100644 --- a/portfolio.go +++ b/portfolio.go @@ -30,11 +30,11 @@ type Document struct { } type Metadata struct { - Name string `json:"name,omitempty" yaml:"name,omitempty" bson:"name,omitempty"` - Benchmark Component `json:"benchmark,omitempty" yaml:"benchmark,omitempty" bson:"benchmark,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty" bson:"description,omitempty"` - Privacy string `json:"privacy,omitempty" yaml:"privacy,omitempty" bson:"privacy,omitempty"` - Factors []Identifier `json:"factors,omitempty" yaml:"factors,omitempty" bson:"factors,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty" bson:"name,omitempty"` + Benchmark Component `json:"benchmark,omitempty" yaml:"benchmark,omitempty" bson:"benchmark,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty" bson:"description,omitempty"` + Privacy string `json:"privacy,omitempty" yaml:"privacy,omitempty" bson:"privacy,omitempty"` + Factors []Component `json:"factors,omitempty" yaml:"factors,omitempty" bson:"factors,omitempty"` } // Specification models a portfolio. From b216f08c89b3139e07db00fd53b1aaf276fc264d Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Sun, 17 Dec 2023 23:10:27 -0800 Subject: [PATCH 12/14] fix tests after rebase --- fs_test.go | 2 +- portfolio.go | 5 +++++ portfolio_test.go | 3 +-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fs_test.go b/fs_test.go index b420069..3102d13 100644 --- a/fs_test.go +++ b/fs_test.go @@ -60,7 +60,7 @@ func TestParseSpecificationFile(t *testing.T) { }, Policy: portfolio.Policy{ RebalancingInterval: "Quarterly", - WeightsAlgorithm: allocation.ConstantWeightsAlgorithmName, + WeightsAlgorithm: allocation.EqualWeightsAlgorithmName, }, Filepath: "examples/maang_portfolio.yml", }, diff --git a/portfolio.go b/portfolio.go index 598090c..fb91f95 100644 --- a/portfolio.go +++ b/portfolio.go @@ -96,6 +96,11 @@ func ParseSpecifications(r io.Reader) ([]Specification, error) { pf := spec.Spec pf.setDefaultPolicyWeightAlgorithm() + if pf.Policy.WeightsAlgorithm == allocation.ConstantWeightsAlgorithmName { + if len(pf.Policy.Weights) != len(pf.Assets) { + return result, fmt.Errorf("expected the number of policy weights to be the same as the number of assets got %d but expected %d", len(pf.Policy.Weights), len(pf.Assets)) + } + } result = append(result, pf) } } diff --git a/portfolio_test.go b/portfolio_test.go index dced264..51ab335 100644 --- a/portfolio_test.go +++ b/portfolio_test.go @@ -153,11 +153,10 @@ func TestParse(t *testing.T) { p, err := portfolio.ParseOneSpecification(tt.SpecYAML) if tt.ErrorStringContains == "" { assert.NoError(t, err) + assert.Equal(t, tt.Portfolio, p) } else { - assert.Error(t, err) assert.ErrorContains(t, err, tt.ErrorStringContains) } - assert.Equal(t, tt.Portfolio, p) }) } } From 74609681e5d45fdd9d7f27df1ab2d952e8b9205e Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Sun, 17 Dec 2023 23:15:49 -0800 Subject: [PATCH 13/14] use test constructor and remove Values encoding/decoding --- portfolio.go | 82 +--------------------------------------------------- 1 file changed, 1 insertion(+), 81 deletions(-) diff --git a/portfolio.go b/portfolio.go index fb91f95..056b11e 100644 --- a/portfolio.go +++ b/portfolio.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "io" - "net/url" - "strconv" "strings" "time" @@ -98,7 +96,7 @@ func ParseSpecifications(r io.Reader) ([]Specification, error) { pf.setDefaultPolicyWeightAlgorithm() if pf.Policy.WeightsAlgorithm == allocation.ConstantWeightsAlgorithmName { if len(pf.Policy.Weights) != len(pf.Assets) { - return result, fmt.Errorf("expected the number of policy weights to be the same as the number of assets got %d but expected %d", len(pf.Policy.Weights), len(pf.Assets)) + return result, errAssetAndWeightsLenMismatch(&spec.Spec) } } result = append(result, pf) @@ -149,84 +147,6 @@ type Policy struct { WeightsUpdatingInterval backtestconfig.Interval `yaml:"weights_updating_interval,omitempty"` } -func (pf *Specification) ParseValues(q url.Values) error { - if q.Has("asset-id") { - pf.Assets = pf.Assets[:0] - for _, assetID := range q["asset-id"] { - pf.Assets = append(pf.Assets, Component{ID: assetID}) - } - } - if q.Has("benchmark-id") { - pf.Benchmark.ID = q.Get("benchmark-id") - } - if q.Has("name") { - pf.Name = q.Get("name") - } - if q.Has("filepath") { - pf.Filepath = q.Get("filepath") - } - if q.Has("policy-rebalance") { - pf.Policy.RebalancingInterval = backtestconfig.Interval(q.Get("policy-rebalance")) - } - if q.Has("policy-weights-algorithm") { - pf.Policy.WeightsAlgorithm = q.Get("policy-weights-algorithm") - } - if q.Has("policy-weight") { - pf.Policy.Weights = pf.Policy.Weights[:0] - for i, weight := range q["policy-weight"] { - f, err := strconv.ParseFloat(weight, 64) - if err != nil { - return fmt.Errorf("failed to parse policy weight at indx %d: %w", i, err) - } - pf.Policy.Weights = append(pf.Policy.Weights, f) - } - } - if q.Has("policy-update-weights") { - pf.Policy.WeightsUpdatingInterval = backtestconfig.Interval(q.Get("policy-update-weights")) - } - if q.Has("policy-weight-algorithm-look-back") { - pf.Policy.WeightsAlgorithmLookBack = backtestconfig.Window(q.Get("policy-weight-algorithm-look-back")) - } - pf.filterEmptyAssetIDs() - return pf.Validate() -} - -func (pf *Specification) Values() url.Values { - q := make(url.Values) - if pf.Name != "" { - q.Set("name", pf.Name) - } - if pf.Benchmark.ID != "" { - q.Set("benchmark-id", pf.Benchmark.ID) - } - if pf.Filepath != "" { - q.Set("filepath", pf.Filepath) - } - if pf.Assets != nil { - for _, asset := range pf.Assets { - q.Add("asset-id", asset.ID) - } - } - if pf.Policy.RebalancingInterval != "" { - q.Set("policy-rebalance", pf.Policy.RebalancingInterval.String()) - } - if pf.Policy.WeightsAlgorithm != "" { - q.Set("policy-weights-algorithm", pf.Policy.WeightsAlgorithm) - } - if pf.Policy.Weights != nil { - for _, w := range pf.Policy.Weights { - q.Add("policy-weight", strconv.FormatFloat(w, 'f', 4, 64)) - } - } - if pf.Policy.WeightsUpdatingInterval != "" { - q.Set("policy-update-weights", string(pf.Policy.WeightsUpdatingInterval)) - } - if pf.Policy.WeightsAlgorithmLookBack != "" { - q.Set("policy-weight-algorithm-look-back", pf.Policy.WeightsAlgorithmLookBack.String()) - } - return q -} - // Validate does some simple validations. // Server you should do additional validations. func (pf *Specification) Validate() error { From 2f2305190a121ae5121aafad3653149a80ac7f35 Mon Sep 17 00:00:00 2001 From: Christopher Hunter <8398225+crhntr@users.noreply.github.com> Date: Mon, 18 Dec 2023 00:06:52 -0800 Subject: [PATCH 14/14] fix fake /api/returns endpoint --- api.go | 18 ++++++++ api_test.go | 24 ++++++++++ portfolio_test.go | 115 ++-------------------------------------------- 3 files changed, 46 insertions(+), 111 deletions(-) diff --git a/api.go b/api.go index 21a6432..eb08b5b 100644 --- a/api.go +++ b/api.go @@ -11,6 +11,8 @@ import ( "os" "strings" + "go.mongodb.org/mongo-driver/bson/primitive" + "github.com/portfoliotree/portfolio/returns" ) @@ -57,6 +59,22 @@ func (pf *Specification) AssetReturns(ctx context.Context) (returns.Table, error return doJSONRequest[returns.Table](http.DefaultClient.Do, req) } +func ParseComponentsFromURL(values url.Values, prefix string) ([]Component, error) { + assetValues, ok := values[prefix+"-id"] + if !ok { + return nil, errors.New("use asset-id parameters to specify asset returns") + } + components := make([]Component, 0, len(assetValues)) + for _, v := range assetValues { + if _, err := primitive.ObjectIDFromHex(v); err == nil { + components = append(components, Component{Type: "Portfolio", ID: v}) + continue + } + components = append(components, Component{Type: "Security", ID: v}) + } + return components, nil +} + func doJSONRequest[T any](do func(r *http.Request) (*http.Response, error), req *http.Request) (T, error) { var result T req.Header.Set("accept", "application/json") diff --git a/api_test.go b/api_test.go index 873e1fa..42136dd 100644 --- a/api_test.go +++ b/api_test.go @@ -2,6 +2,7 @@ package portfolio_test import ( "context" + "os" "testing" "github.com/stretchr/testify/assert" @@ -9,6 +10,29 @@ import ( "github.com/portfoliotree/portfolio" ) +func Test_APIEndpoints(t *testing.T) { + if value, found := os.LookupEnv("CI"); !found || value != "true" { + t.Skip("Skipping test in CI environment") + } + + t.Run("returns", func(t *testing.T) { + pf := portfolio.Specification{ + Assets: []portfolio.Component{ + {ID: "AAPL"}, + {ID: "GOOG"}, + }, + } + table, err := pf.AssetReturns(context.Background()) + assert.NoError(t, err) + if table.NumberOfColumns() != 2 { + t.Errorf("Expected 2 columns, got %d", table.NumberOfColumns()) + } + if table.NumberOfRows() < 10 { + t.Errorf("Expected at least 10 rows, got %d", table.NumberOfRows()) + } + }) +} + func TestSpecification_AssetReturns(t *testing.T) { for _, tt := range []struct { Name string diff --git a/portfolio_test.go b/portfolio_test.go index 51ab335..8e894fc 100644 --- a/portfolio_test.go +++ b/portfolio_test.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "net/url" "os" "path/filepath" "testing" @@ -32,12 +31,12 @@ func TestMain(m *testing.M) { func testdataAssetReturns(crp portfolio.ComponentReturnsProvider) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - var pf portfolio.Specification - if err := pf.ParseValues(req.URL.Query()); err != nil { - http.Error(res, err.Error(), http.StatusInternalServerError) + assets, err := portfolio.ParseComponentsFromURL(req.URL.Query(), "asset") + if err != nil { + http.Error(res, err.Error(), http.StatusBadRequest) return } - table, err := crp.ComponentReturnsTable(req.Context(), pf.Assets...) + table, err := crp.ComponentReturnsTable(req.Context(), assets...) if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) return @@ -294,112 +293,6 @@ func Test_Portfolio_Validate(t *testing.T) { } } -func Test_Portfolio_ParseValues(t *testing.T) { - for _, tt := range []struct { - Name string - Values url.Values - In, Out portfolio.Specification - ExpectErr bool - }{ - { - Name: "set everything", - Values: url.Values{ - "name": []string{"X"}, - "asset-id": []string{"y", "z"}, - "benchmark-id": []string{"b"}, - "filepath": []string{"f"}, - "policy-weight": []string{".5", ".5"}, - "policy-rebalance": []string{"Daily"}, - "policy-weights-algorithm": []string{"Static"}, - "policy-update-weights": []string{"Daily"}, - "policy-weight-algorithm-look-back": []string{"1 Week"}, - }, - Out: portfolio.Specification{ - Name: "X", - Assets: []portfolio.Component{ - {ID: "y"}, - {ID: "z"}, - }, - Benchmark: portfolio.Component{ - ID: "b", - }, - Filepath: "f", - Policy: portfolio.Policy{ - RebalancingInterval: "Daily", - WeightsAlgorithm: "Static", - Weights: []float64{0.5, 0.5}, - WeightsUpdatingInterval: "Daily", - WeightsAlgorithmLookBack: "1 Week", - }, - }, - }, - { - Name: "empty values do not override", - Values: url.Values{}, - In: portfolio.Specification{ - Name: "no change", - Benchmark: portfolio.Component{ID: "b"}, - Assets: []portfolio.Component{{ID: "a1"}}, - Filepath: "f", - }, - Out: portfolio.Specification{ - Name: "no change", - Benchmark: portfolio.Component{ID: "b"}, - Assets: []portfolio.Component{{ID: "a1"}}, - Filepath: "f", - }, - }, - } { - t.Run(tt.Name, func(t *testing.T) { - pf := &tt.In - err := pf.ParseValues(tt.Values) - if tt.ExpectErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.Out, *pf) - }) - } -} - -func Test_Portfolio_Values(t *testing.T) { - t.Run("encode and decode", func(t *testing.T) { - pf := portfolio.Specification{ - Name: "X", - Assets: []portfolio.Component{ - {ID: "y"}, - {ID: "z"}, - }, - Benchmark: portfolio.Component{ - ID: "b", - }, - Filepath: "f", - Policy: portfolio.Policy{ - RebalancingInterval: "Daily", - WeightsAlgorithm: "Static", - Weights: []float64{0.5, 0.5}, - WeightsUpdatingInterval: "Daily", - WeightsAlgorithmLookBack: "1 Week", - }, - } - - var update portfolio.Specification - e := pf.Values().Encode() - q, err := url.ParseQuery(e) - require.NoError(t, err) - assert.NoError(t, update.ParseValues(q)) - assert.Equal(t, pf, update) - }) - - t.Run("fail to parse float", func(t *testing.T) { - values, err := url.ParseQuery(`policy-weight=x`) - require.NoError(t, err) - var pf portfolio.Specification - assert.Error(t, pf.ParseValues(values)) - }) -} - func TestPortfolio_RemoveAsset(t *testing.T) { t.Run("nil", func(t *testing.T) { var zero portfolio.Specification