update monash

Author: shikai

data/last_val_mae.csv

@@ -1,19 +1,25 @@
 dataset,mae
+bitcoin,7.777284173521224e+17
+wind_4_seconds,0.0
+tourism_quarterly,15845.100306204946
+traffic_weekly,1.1855844384623289
+us_births,1152.6666666666667
+sunspot,3.933333396911621
 covid_deaths,353.70939849624057
-pedestrian_counts,170.8838383838384
+weather,2.362190193902301
+nn5_daily,8.262752532958984
+solar_10_minutes,2.7269221451758905
+fred_md,2825.672461360778
+tourism_yearly,99456.0540551959
+oikolab_weather,120.03774162319314
+cif_2016,386526.36704240675
+solar_4_seconds,0.0
 australian_electricity_demand,659.600688770839
-tourism_monthly,5636.83029361023
-solar_weekly,1729.4092503457175
 traffic_hourly,0.026246391982575817
-us_births,1152.6666666666667
-tourism_quarterly,15845.100306204946
-tourism_yearly,99456.0540551959
-fred_md,2825.672461360778
-hospital,24.06573229030856
+solar_weekly,1729.4092503457175
+tourism_monthly,5636.83029361023
 nn5_weekly,16.708553516113007
-weather,2.362190193902301
-solar_10_minutes,2.7269221451758905
-traffic_weekly,1.1855844384623289
+pedestrian_counts,170.8838383838384
 kaggle_web_traffic_weekly,2081.781183003247
+hospital,24.06573229030856
 saugeenday,21.496667098999023
-cif_2016,386526.36704240675

data/monash.py

@@ -13,7 +13,6 @@
     'traffic_hourly': 168,    
 }
 
-
 def get_benchmark_test_sets():
     test_set_dir = "datasets/monash"
     if not os.path.exists(test_set_dir):
@@ -27,7 +26,6 @@ def get_benchmark_test_sets():
 
     benchmarks = {
         "monash_tsf": datasets.get_dataset_config_names("monash_tsf"),
-        # "ett": datasets.get_dataset_config_names("ett"),
     }
 
     test_sets = defaultdict(list)
@@ -93,7 +91,6 @@ def get_datasets():
         test = [test[i] for i in ind]
         benchmarks[k] = [list(train), list(test)]
 
-    # df = pd.read_csv('data/last_value_results.csv')
     df = pd.read_csv('data/last_val_mae.csv')
     df.sort_values(by='mae')
 
@@ -110,12 +107,11 @@ def get_datasets():
     # lower case and repalce spaces with underscores
     datasets = [d.lower().replace(' ', '_') for d in datasets]
     df_paper['Dataset'] = datasets
-    # remove from df_paper datasets in df_paper but not in df
-    df_paper = df_paper[df_paper['Dataset'].isin(df['dataset'])]
     df_paper = df_paper.reset_index(drop=True)
     # for each dataset, add last value mae to df_paper
     for dataset in df_paper['Dataset']:
-        df_paper.loc[df_paper['Dataset'] == dataset, 'Last Value'] = df[df['dataset'] == dataset]['mae'].values[0]
+        if dataset in df['dataset'].values:
+            df_paper.loc[df_paper['Dataset'] == dataset, 'Last Value'] = df[df['dataset'] == dataset]['mae'].values[0]
     # turn '-' into np.nan
     df_paper = df_paper.replace('-', np.nan)
     # convert all values to float
@@ -132,9 +128,7 @@ def get_datasets():
     df_paper = df_paper.reset_index(drop=True)
     # save as csv
     df_paper.to_csv('data/paper_mae_normalized.csv', index=False)
-    selected_datasets = df_paper.head(20)['Dataset']
-    datasets = {k: benchmarks[k] for k in selected_datasets}
-    return datasets
+    return benchmarks
 
 def main():
     get_datasets()

data/paper_mae.csv

@@ -4,16 +4,28 @@ tourism_quarterly,15014.19,7656.49,9972.42,8925.52,10475.47,9092.58,10267.97,898
 tourism_monthly,5302.1,2069.96,2940.08,2004.51,2536.77,2187.28,2537.04,2022.21,1871.69,2003.02,2095.13,2146.98,5636.83029361023
 cif_2016,581875.97,714818.58,855578.4,642421.42,469059.49,563205.57,603551.3,1495923.44,3200418.0,679034.8,5998224.62,4057973.04,386526.3670424068
 australian_electricity_demand,659.6,665.04,370.74,1282.99,1045.92,247.18,241.77,258.76,302.41,213.83,227.5,231.45,659.600688770839
+dominick,5.7,5.86,7.08,5.81,7.1,8.19,8.09,5.85,5.23,8.28,5.1,5.18,
+bitcoin,5.33e+18,5.33e+18,9.9e+17,1.1e+18,3.62e+18,6.66e+17,1.93e+18,1.45e+18,1.95e+18,1.06e+18,2.46e+18,2.61e+18,7.777284173521224e+17
 pedestrian_counts,170.87,170.94,222.38,216.5,635.16,44.18,43.41,46.41,44.78,66.84,46.46,47.29,170.8838383838384
+vehicle_trips,29.98,30.76,21.21,30.95,30.07,27.24,22.61,22.93,22.0,28.16,24.15,28.01,
+kdd_cup,42.04,42.06,39.2,44.88,52.2,36.85,34.82,37.16,48.98,49.1,37.08,44.46,
 weather,2.24,2.51,2.3,2.35,2.45,8.17,2.51,2.09,2.02,2.34,2.29,2.03,2.362190193902301
+nn5_daily,6.63,3.8,3.7,3.72,4.41,5.47,4.22,4.06,3.94,4.92,3.97,4.16,8.262752532958984
 nn5_weekly,15.66,15.3,14.98,15.7,15.38,14.94,15.29,15.02,14.69,14.19,19.34,20.34,16.708553516113007
+kaggle_daily,363.43,358.73,415.4,403.23,340.36,,,,,,,,
 kaggle_web_traffic_weekly,2337.11,2373.98,2241.84,2668.28,3115.03,4051.75,10715.36,2025.23,2272.58,2051.3,2025.5,3100.32,2081.781183003247
 solar_10_minutes,3.28,3.29,8.77,3.28,2.37,3.28,5.69,3.28,3.28,3.52,,3.28,2.7269221451758905
 solar_weekly,1202.39,1210.83,908.65,1131.01,839.88,1044.98,1513.49,1050.84,721.59,1172.64,1996.89,576.35,1729.4092503457175
+electricity_hourly,845.97,846.03,574.3,1344.61,868.2,537.38,407.14,354.39,329.75,350.37,286.56,398.8,
+electricity_weekly,74149.18,74111.14,24347.24,67737.82,28457.18,44882.52,34518.43,27451.83,50312.05,32991.72,61429.32,76382.47,
+carparts,0.55,0.53,0.58,0.56,0.56,0.41,0.53,0.39,0.39,0.98,0.4,0.39,
 fred_md,2798.22,3492.84,1989.97,2041.42,2957.11,8921.94,2475.68,2339.57,4264.36,2557.8,2508.4,4666.04,2825.672461360778
 traffic_hourly,0.03,0.03,0.04,0.03,0.04,0.02,0.02,0.01,0.01,0.02,0.02,0.01,0.0262463919825758
 traffic_weekly,1.12,1.13,1.17,1.14,1.22,1.13,1.17,1.15,1.18,1.11,1.2,1.42,1.1855844384623289
+rideshare,6.29,7.62,6.45,6.29,3.37,6.3,6.07,6.59,6.28,5.55,2.75,6.29,
 hospital,21.76,18.54,17.43,17.97,19.6,19.24,19.17,22.86,18.25,20.18,19.35,36.19,24.06573229030856
 covid_deaths,353.71,321.32,96.29,85.59,85.77,347.98,475.15,144.14,201.98,158.81,1049.48,408.66,353.70939849624057
+temperature_rain,8.18,8.22,7.14,8.21,7.19,6.13,6.76,5.56,5.37,7.28,5.81,5.24,
+sunspot,4.93,4.93,2.57,4.93,2.57,3.83,2.27,7.97,0.77,14.47,0.17,0.13,3.933333396911621
 saugeenday,21.5,21.49,22.26,30.69,22.38,25.24,21.28,22.98,23.51,27.92,22.17,28.06,21.496667098999023
 us_births,1192.2,586.93,399.0,419.73,526.33,574.93,441.7,557.87,424.93,422.0,504.4,452.87,1152.6666666666667

data/paper_mae_normalized.csv

@@ -1,19 +1,31 @@
 Dataset,SES,Theta,TBATS,ETS,(DHR-)ARIMA,PR,CatBoost,FFNN,DeepAR,N-BEATS,WaveNet,Transformer,Last Value,normalized_min,normalized_median
+sunspot,1.2533898102487173,1.2533898102487173,0.6533898199471001,1.2533898102487173,0.6533898199471001,0.9737287978199975,0.5771186347392675,2.026271153688089,0.19576270870010395,3.6788134998577977,0.04322033828443854,0.033050846923394175,3.933333396911621,0.033050846923394175,0.9737287978199975
 covid_deaths,1.000001700559165,0.9084293529266092,0.2722291248391112,0.24197830299075218,0.24248719532091148,0.9838019613824273,1.34333439263998,0.4075096692731279,0.5710337380309863,0.44898439418111175,2.96706845919774,1.1553552202383548,353.70939849624057,0.24197830299075218,0.9084293529266092
 pedestrian_counts,0.9999190187675484,1.0003286537608984,1.3013518545884437,1.266942515147037,3.7169108910891087,0.25853820008866557,0.2540322151618147,0.27158800059110383,0.26204935717452343,0.3911428993645633,0.27188059701492534,0.27673769765036205,170.8838383838384,0.2540322151618147,0.3911428993645633
 australian_electricity_demand,0.9999989557760464,1.0082463698443023,0.5620673330267002,1.9451010616602633,1.58568664012323,0.3747418767263844,0.36653994472100476,0.3922979529966794,0.458474354481859,0.32418098349543967,0.34490564347945807,0.35089411509151897,659.600688770839,0.32418098349543967,0.458474354481859
 tourism_monthly,0.9406172837969468,0.3672205640724105,0.5215839127413151,0.3556094286308854,0.4500348365775033,0.3880336795804277,0.4500827358375371,0.3587494912331008,0.33204654078759493,0.3553450956773656,0.37168583953556084,0.38088427150871706,5636.83029361023,0.33204654078759493,0.38088427150871706
 solar_weekly,0.6952605346361114,0.7001408138403036,0.5254106278304898,0.6539863249684281,0.48564560403045365,0.6042410145493922,0.8751485512740526,0.6076294548499331,0.4172465249944457,0.6780581286734667,1.1546659644620334,0.3332640899687479,1729.4092503457175,0.3332640899687479,0.6539863249684281
 us_births,1.0342972816657028,0.5091931752458068,0.34615384615384615,0.3641382301908618,0.45661943319838055,0.4987825332562174,0.3831983805668016,0.4839820705610179,0.36864950838635047,0.3661075766338924,0.437593984962406,0.39288895315211103,1152.6666666666667,0.34615384615384615,0.437593984962406
 traffic_hourly,1.1430142482028047,1.1430142482028047,1.5240189976037397,1.1430142482028047,1.5240189976037397,0.7620094988018699,0.7620094988018699,0.38100474940093493,0.38100474940093493,0.7620094988018699,0.7620094988018699,0.38100474940093493,0.0262463919825758,0.38100474940093493,0.7620094988018699
+nn5_daily,0.802396050656708,0.4598951723220951,0.44779266778730314,0.45021316869426153,0.5337204499843262,0.6620069980531211,0.5107256913682214,0.4913616841125542,0.4768386786708039,0.5954432231117652,0.4804694300312415,0.5034641886473462,8.262752532958984,0.44779266778730314,0.5034641886473462
 tourism_quarterly,0.9475604262423281,0.4832086797835995,0.6293693196814159,0.5632984220683516,0.6611173042494279,0.5738417444059564,0.6480217733919336,0.5668023443488726,0.6002719967809447,0.5453143137640065,0.5766527079933916,0.6009220399994131,15845.100306204946,0.4832086797835995,0.6002719967809447
 tourism_yearly,0.961019727838344,0.9114940348396415,0.9463584785674776,0.9533747432547206,0.9555299665041875,0.8313518044272377,0.800023897548177,0.800285319542514,0.7186218142169105,0.7133985022231359,0.702877976248726,0.7472297257918155,99456.0540551959,0.702877976248726,0.8313518044272377
 fred_md,0.9902846272042592,1.2361092970831906,0.7042465208588531,0.7224545760045026,1.046515489829959,3.1574572502658014,0.876138347191086,0.8279692823538782,1.5091487277143167,0.9052004558122859,0.8877178916879889,1.651302500132285,2825.672461360778,0.7042465208588531,0.9902846272042592
 hospital,0.9041902293894837,0.7703900208125471,0.7242663464273299,0.746704890722841,0.8144360522074393,0.7994770226770985,0.7965683224906435,0.9498983751766358,0.7583396914686615,0.8385367108952104,0.8040478372558139,1.5037979963973074,24.06573229030856,0.7242663464273299,0.8040478372558139
 nn5_weekly,0.937244506826888,0.9156986560952354,0.8965467887782109,0.939638490241516,0.9204866229244916,0.8941528053635828,0.9151001602415783,0.8989407721928389,0.8791904090221573,0.8492656163393065,1.1574909809726701,1.2173405663383716,16.708553516113007,0.8492656163393065,0.9156986560952354
 weather,0.9482724997260088,1.0625732028179828,0.9736726559686696,0.9948394528375538,1.0371730465753222,3.4586546083756655,1.0625732028179828,0.8847721091193562,0.8551385935029185,0.9906060934637768,0.9694392965948928,0.8593719528766953,2.362190193902301,0.8551385935029185,0.9906060934637768
+bitcoin,6.853292076103736,6.853292076103736,1.2729379278316508,1.414375475368501,4.654581109849067,0.8563400605412924,2.48158606132837,1.864404035713024,2.5073019790623428,1.3629436399005554,3.1630578812786476,3.355927264283443,7.777284173521224e+17,0.8563400605412924,2.5073019790623428
 solar_10_minutes,1.2028212854564042,1.20648842352182,3.216080083369715,1.2028212854564042,0.8691117215035604,1.2028212854564042,2.086601559221628,1.2028212854564042,1.2028212854564042,1.2908325990263851,,1.2028212854564042,2.7269221451758905,0.8691117215035604,1.2028212854564042
 traffic_weekly,0.9446817650985787,0.9531164237155302,0.9868550581833365,0.9615510823324818,1.0290283512680944,0.9531164237155302,0.9868550581833365,0.9699857409494334,0.9952897168002881,0.9362471064816271,1.0121590340341913,1.1977215236071264,1.1855844384623289,0.9362471064816271,0.9868550581833365
 kaggle_web_traffic_weekly,1.1226492097639231,1.1403600048758329,1.0768855143391425,1.2817293295689465,1.4963292133835864,1.9462900486759183,5.147207635214411,0.9728351935040241,1.0916517156339651,0.9853581234895814,0.9729648901321829,1.489263148938341,2081.781183003247,0.9728351935040241,1.1403600048758329
 saugeenday,1.0001550426857162,0.9996898542937693,1.0355093604736765,1.4276631748848667,1.0410916211770385,1.1741355012738361,0.9899208980628857,1.0690029246938493,1.0936579094670322,1.2988059903155906,1.0313226649461549,1.3053186278028464,21.496667098999023,0.9899208980628857,1.0690029246938493
 cif_2016,1.5053978709197886,1.8493397629496653,2.2135059156420556,1.6620377670885214,1.2135252080967047,1.4570948272157827,1.5614751061310725,3.870171785294736,8.279947431500512,1.7567619130249434,15.51828059207645,10.498567202673625,386526.3670424068,1.2135252080967047,1.8493397629496653
+dominick,,,,,,,,,,,,,,,
+vehicle_trips,,,,,,,,,,,,,,,
+kdd_cup,,,,,,,,,,,,,,,
+kaggle_daily,,,,,,,,,,,,,,,
+electricity_hourly,,,,,,,,,,,,,,,
+electricity_weekly,,,,,,,,,,,,,,,
+carparts,,,,,,,,,,,,,,,
+rideshare,,,,,,,,,,,,,,,
+temperature_rain,,,,,,,,,,,,,,,

visualize.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "42145ff5",
    "metadata": {},
    "outputs": [],
@@ -109,7 +109,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "e6ff231f",
    "metadata": {},
    "outputs": [],