version 0.10.1

Author: Robert Carver

docs/userguide.md

@@ -3249,19 +3249,19 @@ forecast_weight_estimate: ## can also be applied to instrument weights
 
 I recommend using weekly data, since it speeds things up and doesn't affect out of sample performance. 
 
-### Removing expensive assets (forecasts only)
+### Removing expensive assets (forecast weights only)
 
 Again I recommend you check out this [blog post](http://qoppac.blogspot.co.uk/2016/05/optimising-weights-with-costs.html). 
 
 ```
 forecast_weight_estimate:
-   ceiling_cost_SR: 0.13 ## Max cost to allow for assets, annual SR units. Defaults to 999 for instrument weight estimation since I recommend you don't use it
+   ceiling_cost_SR: 0.13 ## Max cost to allow for assets, annual SR units.  
 ```
 
 See ['costs'](#costs) to see how to configure pooling when estimating the costs of forecasts.
 
 
-### Pooling gross returns
+### Pooling gross returns (forecast weights only)
 
 Pooling across instruments is only available when calculating forecast weights. Again I recommend you check out this [blog post](http://qoppac.blogspot.co.uk/2016/05/optimising-weights-with-costs.html). Only instruments whose rules have survived the application of a ceiling cost will be included in the pooling process.
 
@@ -3270,14 +3270,14 @@ Pooling across instruments is only available when calculating forecast weights.
 forecast_weight_estimate:
    pool_gross_returns: True ## pool gross returns for estimation
 forecast_cost_estimate:
-   use_pooled_costs: False  ### use weighted average of SR cost * turnover across instruments with the same set of trading rules
+   use_pooled_costs: False  ### use weighted average of [SR cost * turnover] across instruments with the same set of trading rules
    use_pooled_turnover: True ### Use weighted average of turnover across instruments with the same set of trading rules
 ```
 
 See ['costs'](#costs) to see how to configure pooling when estimating the costs of forecasts. Notice if pool_gross_returns is True, and use_pooled_costs is True, then a single optimisation will be run across all instruments with a common set of trading rules. Otherwise each instrument is optimised individually, which is slower.
 
 
-### Working out net costs
+### Working out net costs (both instrument and forecast weights)
 
 Again I recommend you check out this [blog post](http://qoppac.blogspot.co.uk/2016/05/optimising-weights-with-costs.html). 
 

examples/breakout/breakout.py

@@ -141,22 +141,22 @@
 print(system.combForecast.get_forecast_weights("V2X").irow(-1))
 
 ## now include other rules
-"""
+
 
 my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml")
 #my_config.forecast_weight_estimate["method"]="bootstrap"
 
 system = futures_system(config=my_config, log_level="on")
 bvariations=["breakout"+str(ws) for ws in [10, 20, 40, 80, 160, 320]]
-
+evariations=[
+            "ewmac%d_%d" % (fast, fast*4) for fast in [2,4,8,16,32, 64]]
 
 #cProfile.run("system.accounts.pandl_for_all_trading_rules_unweighted().to_frame()","restats")
 system.accounts.pandl_for_all_trading_rules_unweighted().to_frame().loc[:, bvariations].cumsum().plot()
 show()
 
-"""
-variations=["breakout"+str(ws) for ws in [10, 20, 40, 80, 160, 320]]+[
-            "ewmac%d_%d" % (fast, fast*4) for fast in [2,4,8,16,32, 64]]+["carry"]
+
+variations=bvariations+evariation+["carry"]
 
 corr_result=system.combForecast.get_forecast_correlation_matrices("US10")
 matrix=corr_result.corr_list[-1]
@@ -208,13 +208,30 @@
 allpandl.cumsum().sum(axis=1).plot()
 show()
 
+"""
+### show grouped courves
+my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml")
+my_config.forecast_weight_estimate["method"]="equal_weights"
+system=futures_system(config=my_config, log_level="on")
 
+allrulespandl=system.accounts.pandl_for_all_trading_rules()
+
+## 
+ewmac_all=allrulespandl.to_frame().loc[:,evariations].sum(axis=1)
+break_all=allrulespandl.to_frame().loc[:,evariations].sum(axis=1)
+
+both_plot=pd.concat(ewmac_all, break_all)
+both_plot.corr()
+both_plot.plot()
+show()
+
+"""
 ## full backtest 37 instruments
 
 my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml")
 
 ## will do all instruments we have data for
-del(my_config.instruments)
+#del(my_config.instruments)
 
 ## temporarily remove breakout rules
 my_config.rule_variations=["ewmac%d_%d" % (fast, fast*4) for fast in [2,4,8,16,32, 64]]+["carry"]
@@ -223,7 +240,7 @@
 
 ## new system has all trading rules
 new_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml")
-del(new_config.instruments)
+#del(new_config.instruments)
 
 system_new = futures_system(config=new_config, log_level="on")
 

examples/optimisation/optimisationwithcosts.py

@@ -255,6 +255,7 @@
 
 system.config.rule_variations=rule_variations
 system.config.forecast_weight_estimate['method']="shrinkage"
+system.config.forecast_weight_estimate['equalise_gross']=False
 
 
 print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market
@@ -266,6 +267,24 @@
 show()
 
 ## equal weights
+system=futures_system()
+system.set_logging_level("on")
+del(system.config.rule_variations)
+
+system=futures_system()
+system.set_logging_level("on")
+
+system.config.rule_variations=rule_variations
+system.config.forecast_weight_estimate['method']="equal_weights"
+
+
+print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market
+system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh")
+show()
+
+print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market
+system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh")
+show()
 
 
 
@@ -278,13 +297,7 @@
 system.set_logging_level("on")
 
 system.config.rule_variations=rule_variations
-system.config.forecast_weight_estimate['apply_cost_weight']=True
-system.config.forecast_cost_estimates['use_pooled_costs']=False
-system.config.forecast_weight_estimate['pool_gross_returns']=True
-system.config.forecast_weight_estimate['cost_multiplier']=0.0
-system.config.forecast_weight_estimate['ceiling_cost_SR']=0.13
-system.config.forecast_weight_estimate['method']="bootstrap"
-system.config.forecast_weight_estimate['equalise_gross']=False
+system.config.forecast_weight_estimate['method']="equal_weights"
 
 system.config.instrument_weight_estimate['method']="bootstrap"
 system.config.instrument_weight_estimate['apply_cost_weight']=True

syscore/accounting.py

@@ -189,8 +189,6 @@ def pandl_with_data(price, trades=None, marktomarket=True, positions=None,
 
     instr_ccy_returns = cum_trades.shift(1)* price_returns * value_of_price_point
 
-    instr_ccy_returns=instr_ccy_returns.resample("1B", how="sum")
-
     instr_ccy_returns=instr_ccy_returns.cumsum().ffill().reindex(price.index).diff()
     base_ccy_returns = instr_ccy_returns * fx
 

systems/account.py

@@ -592,9 +592,14 @@ def get_forecast_scaling_factor(self, instrument_code, rule_variation_name):
 
         fdm = self.get_forecast_diversification_multiplier(instrument_code)
         forecast_weights = self.get_forecast_weights(instrument_code)
-
-        fcast_weight_this_code = forecast_weights[
-            rule_variation_name]
+        
+        if rule_variation_name in forecast_weights.columns:
+            
+            fcast_weight_this_code = forecast_weights[
+                rule_variation_name]
+        else:
+            fcast_weight_this_code = self.get_aligned_forecast(instrument_code, rule_variation_name)
+            fcast_weight_this_code[:]=0.0
 
         multiplier = fcast_weight_this_code* fdm
 

systems/forecast_combine.py

@@ -648,7 +648,7 @@ def _get_forecast_correlation_matrices(system, NotUsed1, NotUsed2, this_stage,
                                                codes_to_use, corr_func, **corr_params):
             this_stage.log.terse("Calculating forecast correlations over %s" % ", ".join(codes_to_use))
 
-            forecast_data=[this_stage.get_all_forecasts(code) for code in codes_to_use]
+            forecast_data=[this_stage.get_all_forecasts(instrument_code, this_stage.apply_cost_weighting(code)) for code in codes_to_use]
 
             ## if we're not pooling passes a list of one
             forecast_data=[forecast_ts.ffill() for forecast_ts in forecast_data]
@@ -666,7 +666,7 @@ def _get_forecast_correlation_matrices(system, NotUsed1, NotUsed2, this_stage,
 
         if pooling:
             ## find set of instruments with same trading rules as I have
-            codes_to_use=self.has_same_rules_as_code(instrument_code)
+            codes_to_use=self.has_same_cheap_rules_as_code(instrument_code)
             instrument_code_ref=ALL_KEYNAME
 
             ## We