2.0.0 initial

v1/experiments/baselines/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

v1/experiments/baselines/timegpt_pipeline.py

@@ -0,0 +1,259 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from time import time
+from typing import List, Optional, Tuple
+from dotenv import load_dotenv
+from gluonts.time_feature.seasonality import get_seasonality as _get_seasonality
+from nixtla import NixtlaClient
+import pandas as pd
+from tqdm import tqdm
+from utilsforecast.processing import (
+    backtest_splits,
+    drop_index_if_pandas,
+    join,
+    maybe_compute_sort_indices,
+    take_rows,
+    vertical_concat,
+)
+
+
+def get_seasonality(freq: str) -> int:
+  return _get_seasonality(freq, seasonalities={"D": 7})
+
+
+def maybe_convert_col_to_datetime(
+    df: pd.DataFrame, col_name: str
+) -> pd.DataFrame:
+  if not pd.api.types.is_datetime64_any_dtype(df[col_name]):
+    df = df.copy()
+    df[col_name] = pd.to_datetime(df[col_name])
+  return df
+
+
+def zero_pad_time_series(df, freq, min_length=36):
+  """If time_series length is less than min_length, front pad it with zeros."""
+  # 1. Calculate required padding for each unique_id
+  value_counts = df["unique_id"].value_counts()
+  to_pad = value_counts[value_counts < min_length].index
+
+  # 2. Create a new DataFrame to hold padded data
+  padded_data = []
+
+  for unique_id in to_pad:
+    # 2a. Filter data for the specific unique_id
+    subset = df[df["unique_id"] == unique_id]
+    if len(subset) > min_length:
+      padded_data.append(subset)
+    else:
+      # 2b. Determine earliest date and calculate padding dates
+      start_date = subset["ds"].min()
+      padding_dates = pd.date_range(
+          end=start_date,
+          periods=min_length - len(subset) + 1,
+          freq=freq,  # 'MS' for month start
+      )[
+          :-1
+      ]  # Exclude the start_date itself
+
+      # 2c. Create padding data
+      padding_df = pd.DataFrame(
+          {"ds": padding_dates, "unique_id": unique_id, "y": 0}  # Zero padding
+      )
+
+      # 2d. Combine original and padding data, and append to the list
+      padded_data.append(pd.concat([padding_df, subset]).sort_values("ds"))
+
+  # 3. Combine all padded data and original data (unchanged)
+  result_df = pd.concat(padded_data + [df[~df["unique_id"].isin(to_pad)]])
+  return result_df
+
+
+class Forecaster:
+  """Borrowed from
+
+  https://github.com/Nixtla/nixtla/tree/main/experiments/foundation-time-series-arena/xiuhmolpilli/models.
+  """
+
+  def forecast(
+      self,
+      df: pd.DataFrame,
+      h: int,
+      freq: str,
+  ) -> pd.DataFrame:
+    raise NotImplementedError
+
+  def cross_validation(
+      self,
+      df: pd.DataFrame,
+      h: int,
+      freq: str,
+      n_windows: int = 1,
+      step_size: int | None = None,
+  ) -> pd.DataFrame:
+    df = maybe_convert_col_to_datetime(df, "ds")
+    # mlforecast cv code
+    results = []
+    sort_idxs = maybe_compute_sort_indices(df, "unique_id", "ds")
+    if sort_idxs is not None:
+      df = take_rows(df, sort_idxs)
+    splits = backtest_splits(
+        df,
+        n_windows=n_windows,
+        h=h,
+        id_col="unique_id",
+        time_col="ds",
+        freq=pd.tseries.frequencies.to_offset(freq),
+        step_size=h if step_size is None else step_size,
+    )
+    for _, (cutoffs, train, valid) in tqdm(enumerate(splits)):
+      if len(valid.columns) > 3:
+        raise NotImplementedError(
+            "Cross validation with exogenous variables is not yet supported."
+        )
+      y_pred = self.forecast(
+          df=train,
+          h=h,
+          freq=freq,
+      )
+      y_pred = join(y_pred, cutoffs, on="unique_id", how="left")
+      result = join(
+          valid[["unique_id", "ds", "y"]],
+          y_pred,
+          on=["unique_id", "ds"],
+      )
+      if result.shape[0] < valid.shape[0]:
+        raise ValueError(
+            "Cross validation result produced less results than expected."
+            " Please verify that the frequency parameter (freq) matches your"
+            " series' and that there aren't any missing periods."
+        )
+      results.append(result)
+    out = vertical_concat(results)
+    out = drop_index_if_pandas(out)
+    first_out_cols = ["unique_id", "ds", "cutoff", "y"]
+    remaining_cols = [c for c in out.columns if c not in first_out_cols]
+    fcst_cv_df = out[first_out_cols + remaining_cols]
+    return fcst_cv_df
+
+
+class TimeGPT(Forecaster):
+  """Borrowed from
+
+  https://github.com/Nixtla/nixtla/tree/main/experiments/foundation-time-series-arena/xiuhmolpilli/models.
+  We modify the class to take care of edge cases.
+  """
+
+  def __init__(
+      self,
+      api_key: str | None = None,
+      base_url: Optional[str] = None,
+      max_retries: int = 1,
+      model: str = "timegpt-1",
+      alias: str = "TimeGPT",
+  ):
+    self.api_key = api_key
+    self.base_url = base_url
+    self.max_retries = max_retries
+    self.model = model
+    self.alias = alias
+
+  def _get_client(self) -> NixtlaClient:
+    if self.api_key is None:
+      api_key = os.environ["NIXTLA_API_KEY"]
+    else:
+      api_key = self.api_key
+    return NixtlaClient(
+        api_key=api_key,
+        base_url=self.base_url,
+        max_retries=self.max_retries,
+    )
+
+  def forecast(
+      self,
+      df: pd.DataFrame,
+      h: int,
+      freq: str,
+      level: List = [90.0],
+      chunk_size: Optional[int] = None,
+  ) -> pd.DataFrame:
+    client = self._get_client()
+    fcst_df = None
+    if chunk_size is None:
+      fcst_df = client.forecast(
+          df=df,
+          h=h,
+          freq=freq,
+          level=level,
+          model=self.model,
+      )
+    else:
+      all_unique_ids = df["unique_id"].unique()
+      all_fcst_df = []
+      for i in range(0, len(all_unique_ids), chunk_size):
+        chunk_ids = all_unique_ids[i : i + chunk_size]
+        chunk_df = df[df["unique_id"].isin(chunk_ids)]
+        fct_chunk_df = client.forecast(
+            df=chunk_df,
+            h=h,
+            freq=freq,
+            level=level,
+        )
+        all_fcst_df.append(fct_chunk_df)
+      fcst_df = pd.concat(all_fcst_df)
+    fcst_df["ds"] = pd.to_datetime(fcst_df["ds"])
+    replace_dict = {}
+    for col in fcst_df.columns:
+      if col.startswith("TimeGPT"):
+        replace_dict[col] = col.replace("TimeGPT", self.alias)
+    fcst_df = fcst_df.rename(columns=replace_dict)
+    return fcst_df
+
+
+def run_timegpt(
+    train_df: pd.DataFrame,
+    horizon: int,
+    freq: str,
+    seasonality: int,
+    level: List[int],
+    dataset: str,
+    model: str = "timegpt-1",
+) -> Tuple[pd.DataFrame, float, str]:
+  os.environ["NIXTLA_ID_AS_COL"] = "true"
+  model = TimeGPT(model="timegpt-1", alias=model)
+  padded_train_df = zero_pad_time_series(train_df, freq)
+  init_time = time()
+  # For these datasets the API fails if we do not chunk.
+  if dataset in ["m5", "m4_quarterly"]:
+    chunk_size = 5000
+  else:
+    chunk_size = None
+  fcsts_df = model.forecast(
+      df=padded_train_df,
+      h=horizon,
+      level=level,
+      freq=freq,
+      chunk_size=chunk_size,
+  )
+  total_time = time() - init_time
+  # In case levels are not returned we replace the levels with the mean predictions.
+  # Note that this does not affect the results table as we only compare on point
+  # forecastign metrics.
+  for lvl in level:
+    if f"{model.alias}-lo-{lvl}" not in fcsts_df.columns:
+      fcsts_df[f"{model.alias}-lo-{lvl}"] = fcsts_df[model.alias]
+    if f"{model.alias}-hi-{lvl}" not in fcsts_df.columns:
+      fcsts_df[f"{model.alias}-hi-{lvl}"] = fcsts_df[model.alias]
+  return fcsts_df, total_time, model.alias

v1/experiments/extended_benchmarks/README.md

@@ -0,0 +1,35 @@
+# Extended Benchmarks
+
+The benchmark setting has been borrowed from Nixtla's original [benchmarking](https://github.com/AzulGarza/nixtla/tree/main/experiments/amazon-chronos) of time-series foundation models against a strong statistical ensemble. Later more datasets were added by the Chronos team in this [pull request](https://github.com/shchur/nixtla/tree/chronos-full-eval/experiments/amazon-chronos). We compare on all the datasets in this extended benchmarks.
+
+
+## Running TimesFM on the benchmark
+
+We need to add the following packages for running these benchmarks. Follow the installation instructions till before `poetry lock`. Then,
+
+```
+poetry add git+https://github.com/awslabs/gluon-ts.git
+poetry lock
+poetry install --only <pax or pytorch>
+```
+
+To run the timesfm on the benchmark do:
+
+```
+poetry run python3 -m experiments.extended_benchmarks.run_timesfm --model_path=google/timesfm-1.0-200m(-pytorch) --backend="gpu"
+```
+
+
+Note: In the current version of TimesFM we focus on point forecasts and therefore the mase, smape have been calculated using the quantile head corresponding to the median i.e 0.5 quantile. We do offer 10 quantile heads but they have not been calibrated after pretraining. We recommend using them with caution or calibrate/conformalize them on a hold out for your applications. More to follow on later versions.
+
+## Benchmark Results for TimesFM-1.0
+
+![Benchmark Results Table](./tfm_extended_new.png)
+
+__Update:__ We have added TimeGPT-1 to the benchmark results. We had to remove the Dominick dataset as we were not able to run TimeGPT-1 on this benchmark. Note that the previous results including Dominick remain available at `./tfm_results.png`. In order to reproduce the results for TimeGPT-1, please run `run_timegpt.py`.
+
+_Remark:_ All baselines except the ones involving TimeGPT were run performed on a [g2-standard-32](https://cloud.google.com/compute/docs/gpus). Since TimeGPT-1 can only be accessed by an API, the time column might not reflect the true speed of the model as it also includes the communication cost. Moreover, we are not sure about the exact backend hardware for TimeGPT. The TimesFM latency numbers are from the PAX version.
+
+We can see that TimesFM performs the best in terms of both mase and smape. More importantly it is much faster than the other methods, in particular it is more than 600x faster than StatisticalEnsemble and 80x faster than Chronos (Large).
+
+Note: This benchmark only compares on `one` small horizon window for long horizon datasets like ETT hourly and 15 minutes. More in depth comparison on longer horizon rolling validation tasks are presented in our long horizon benchmarks.

v1/experiments/extended_benchmarks/run_timegpt.py

@@ -0,0 +1,108 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Evaluation script for timegpt."""
+
+import os
+import sys
+import time
+
+from absl import flags
+import numpy as np
+import pandas as pd
+
+from ..baselines.timegpt_pipeline import run_timegpt
+from .utils import ExperimentHandler
+
+
+dataset_names = [
+    "m1_monthly",
+    "m1_quarterly",
+    "m1_yearly",
+    "m3_monthly",
+    "m3_other",
+    "m3_quarterly",
+    "m3_yearly",
+    "m4_quarterly",
+    "m4_yearly",
+    "tourism_monthly",
+    "tourism_quarterly",
+    "tourism_yearly",
+    "nn5_daily_without_missing",
+    "m5",
+    "nn5_weekly",
+    "traffic",
+    "weather",
+    "australian_electricity_demand",
+    "car_parts_without_missing",
+    "cif_2016",
+    "covid_deaths",
+    "ercot",
+    "ett_small_15min",
+    "ett_small_1h",
+    "exchange_rate",
+    "fred_md",
+    "hospital",
+]
+
+_MODEL_NAME = flags.DEFINE_string(
+    "model_name",
+    "timegpt-1-long-horizon",
+    "Path to model, can also be set to timegpt-1",
+)
+_SAVE_DIR = flags.DEFINE_string("save_dir", "./results", "Save directory")
+
+
+QUANTILES = list(np.arange(1, 10) / 10.0)
+
+
+def main():
+  results_list = []
+  run_id = np.random.randint(100000)
+  model_name = _MODEL_NAME.value
+  for dataset in dataset_names:
+    print(f"Evaluating model {model_name} on dataset {dataset}", flush=True)
+    exp = ExperimentHandler(dataset, quantiles=QUANTILES)
+    train_df = exp.train_df
+    horizon = exp.horizon
+    seasonality = exp.seasonality
+    freq = exp.freq
+    level = exp.level
+    fcsts_df, total_time, model_name = run_timegpt(
+        train_df=train_df,
+        horizon=exp.horizon,
+        model=model_name,
+        seasonality=seasonality,
+        freq=freq,
+        dataset=dataset,
+        level=level,
+    )
+    time_df = pd.DataFrame({"time": [total_time], "model": model_name})
+    fcsts_df = exp.fcst_from_level_to_quantiles(fcsts_df, model_name)
+    results = exp.evaluate_from_predictions(
+        models=[model_name], fcsts_df=fcsts_df, times_df=time_df
+    )
+    print(results, flush=True)
+    results_list.append(results)
+    results_full = pd.concat(results_list)
+    save_path = os.path.join(_SAVE_DIR.value, str(run_id))
+    print(f"Saving results to {save_path}", flush=True)
+    os.makedirs(save_path, exist_ok=True)
+    results_full.to_csv(f"{save_path}/results.csv")
+
+
+if __name__ == "__main__":
+  FLAGS = flags.FLAGS
+  FLAGS(sys.argv)
+  main()

v1/experiments/extended_benchmarks/run_timesfm.py

@@ -0,0 +1,152 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Evaluation script for timesfm."""
+
+import os
+import sys
+import time
+
+from absl import flags
+import numpy as np
+import pandas as pd
+import timesfm
+
+from .utils import ExperimentHandler
+
+dataset_names = [
+    "m1_monthly",
+    "m1_quarterly",
+    "m1_yearly",
+    "m3_monthly",
+    "m3_other",
+    "m3_quarterly",
+    "m3_yearly",
+    "m4_quarterly",
+    "m4_yearly",
+    "tourism_monthly",
+    "tourism_quarterly",
+    "tourism_yearly",
+    "nn5_daily_without_missing",
+    "m5",
+    "nn5_weekly",
+    "traffic",
+    "weather",
+    "australian_electricity_demand",
+    "car_parts_without_missing",
+    "cif_2016",
+    "covid_deaths",
+    "ercot",
+    "ett_small_15min",
+    "ett_small_1h",
+    "exchange_rate",
+    "fred_md",
+    "hospital",
+]
+
+
+context_dict_v2 = {}
+
+context_dict_v1 = {
+    "cif_2016": 32,
+    "tourism_yearly": 64,
+    "covid_deaths": 64,
+    "tourism_quarterly": 64,
+    "tourism_monthly": 64,
+    "m1_monthly": 64,
+    "m1_quarterly": 64,
+    "m1_yearly": 64,
+    "m3_monthly": 64,
+    "m3_other": 64,
+    "m3_quarterly": 64,
+    "m3_yearly": 64,
+    "m4_quarterly": 64,
+    "m4_yearly": 64,
+}
+
+_MODEL_PATH = flags.DEFINE_string("model_path", "google/timesfm-2.0-500m-jax",
+                                  "Path to model")
+_BATCH_SIZE = flags.DEFINE_integer("batch_size", 64, "Batch size")
+_HORIZON = flags.DEFINE_integer("horizon", 128, "Horizon")
+_BACKEND = flags.DEFINE_string("backend", "gpu", "Backend")
+_NUM_JOBS = flags.DEFINE_integer("num_jobs", 1, "Number of jobs")
+_SAVE_DIR = flags.DEFINE_string("save_dir", "./results", "Save directory")
+
+QUANTILES = list(np.arange(1, 10) / 10.0)
+
+
+def main():
+  results_list = []
+  model_path = _MODEL_PATH.value
+  num_layers = 20
+  max_context_len = 512
+  use_positional_embedding = True
+  context_dict = context_dict_v1
+  if "2.0" in model_path:
+    num_layers = 50
+    use_positional_embedding = False
+    max_context_len = 2048
+    context_dict = context_dict_v2
+
+  tfm = timesfm.TimesFm(
+      hparams=timesfm.TimesFmHparams(
+          backend="gpu",
+          per_core_batch_size=32,
+          horizon_len=128,
+          num_layers=num_layers,
+          context_len=max_context_len,
+          use_positional_embedding=use_positional_embedding,
+      ),
+      checkpoint=timesfm.TimesFmCheckpoint(huggingface_repo_id=model_path),
+  )
+  run_id = np.random.randint(100000)
+  model_name = "timesfm"
+  for dataset in dataset_names:
+    print(f"Evaluating model {model_name} on dataset {dataset}", flush=True)
+    exp = ExperimentHandler(dataset, quantiles=QUANTILES)
+
+    if dataset in context_dict:
+      context_len = context_dict[dataset]
+    else:
+      context_len = max_context_len
+
+    train_df = exp.train_df
+    freq = exp.freq
+    init_time = time.time()
+    fcsts_df = tfm.forecast_on_df(
+        inputs=train_df,
+        freq=freq,
+        value_name="y",
+        model_name=model_name,
+        forecast_context_len=context_len,
+        num_jobs=_NUM_JOBS.value,
+        normalize=True,
+    )
+    total_time = time.time() - init_time
+    time_df = pd.DataFrame({"time": [total_time], "model": model_name})
+    results = exp.evaluate_from_predictions(models=[model_name],
+                                            fcsts_df=fcsts_df,
+                                            times_df=time_df)
+    print(results, flush=True)
+    results_list.append(results)
+    results_full = pd.concat(results_list)
+    save_path = os.path.join(_SAVE_DIR.value, str(run_id))
+    print(f"Saving results to {save_path}", flush=True)
+    os.makedirs(save_path, exist_ok=True)
+    results_full.to_csv(f"{save_path}/results.csv")
+
+
+if __name__ == "__main__":
+  FLAGS = flags.FLAGS
+  FLAGS(sys.argv)
+  main()

v1/experiments/extended_benchmarks/utils.py

@@ -0,0 +1,278 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Forked from https://github.com/Nixtla/nixtla/blob/main/experiments/amazon-chronos/src/utils.py."""
+
+from functools import partial
+from itertools import repeat
+import multiprocessing
+import os
+from pathlib import Path
+from typing import List
+
+from gluonts.dataset import Dataset
+from gluonts.dataset.repository.datasets import (
+    dataset_names as gluonts_datasets,
+    get_dataset,
+)
+from gluonts.time_feature.seasonality import get_seasonality
+import numpy as np
+import pandas as pd
+from utilsforecast.evaluation import evaluate
+from utilsforecast.losses import mae, mase, smape
+
+
+def parallel_transform(inp):
+  ts, last_n = inp[0], inp[1]
+  return ExperimentHandler._transform_gluonts_instance_to_df(ts, last_n=last_n)
+
+
+def quantile_loss(
+    df: pd.DataFrame,
+    models: list,
+    q: float = 0.5,
+    id_col: str = "unique_id",
+    target_col: str = "y",
+) -> pd.DataFrame:
+  delta_y = df[models].sub(df[target_col], axis=0)
+  res = (
+      np.maximum(q * delta_y, (q - 1) * delta_y)
+      .groupby(df[id_col], observed=True)
+      .mean()
+  )
+  res.index.name = id_col
+  res = res.reset_index()
+  return res
+
+
+class ExperimentHandler:
+
+  def __init__(
+      self,
+      dataset: str,
+      quantiles: List[float] = list(np.arange(1, 10) / 10.0),
+      results_dir: str = "./results",
+      models_dir: str = "./models",
+  ):
+    if dataset not in gluonts_datasets:
+      raise Exception(
+          f"dataset {dataset} not found in gluonts "
+          f"available datasets: {', '.join(gluonts_datasets)}"
+      )
+    self.dataset = dataset
+    self.quantiles = quantiles
+    self.level = self._transform_quantiles_to_levels(quantiles)
+    self.results_dir = results_dir
+    self.models_dir = models_dir
+    # defining datasets
+    self._maybe_download_m3_or_m5_file(self.dataset)
+    gluonts_dataset = get_dataset(self.dataset)
+    self.horizon = gluonts_dataset.metadata.prediction_length
+    if self.horizon is None:
+      raise Exception(
+          f"horizon not found for dataset {self.dataset} "
+          "experiment cannot be run"
+      )
+    self.freq = gluonts_dataset.metadata.freq
+    # get_seasonality() returns 1 for freq='D', override this to 7. This significantly improves the accuracy of
+    # statistical models on datasets like m5/nn5_daily. The models like AutoARIMA/AutoETS can still set
+    # seasonality=1 internally on datasets like weather by choosing non-seasonal models during model selection.
+    if self.freq == "D":
+      self.seasonality = 7
+    else:
+      self.seasonality = get_seasonality(self.freq)
+    self.gluonts_train_dataset = gluonts_dataset.train
+    self.gluonts_test_dataset = gluonts_dataset.test
+    self._create_dir_if_not_exists(self.results_dir)
+    try:
+      multiprocessing.set_start_method("spawn")
+    except RuntimeError:
+      print("Multiprocessing context has already been set.")
+
+  @staticmethod
+  def _maybe_download_m3_or_m5_file(dataset: str):
+    if dataset[:2] == "m3":
+      m3_file = Path.home() / ".gluonts" / "datasets" / "M3C.xls"
+      if not m3_file.exists():
+        from datasetsforecast.m3 import M3
+        from datasetsforecast.utils import download_file
+
+        download_file(m3_file.parent, M3.source_url)
+    elif dataset == "m5":
+      m5_raw_dir = Path.home() / ".gluonts" / "m5"
+      if not m5_raw_dir.exists():
+        import zipfile
+        from datasetsforecast.m5 import M5
+        from datasetsforecast.utils import download_file
+
+        download_file(m5_raw_dir, M5.source_url)
+        with zipfile.ZipFile(m5_raw_dir / "m5.zip", "r") as zip_ref:
+          zip_ref.extractall(m5_raw_dir)
+
+  @staticmethod
+  def _transform_quantiles_to_levels(quantiles: List[float]) -> List[int]:
+    level = [
+        int(100 - 200 * q) for q in quantiles if q < 0.5
+    ]  # in this case mean=mediain
+    level = sorted(list(set(level)))
+    return level
+
+  @staticmethod
+  def _create_dir_if_not_exists(directory: str):
+    Path(directory).mkdir(parents=True, exist_ok=True)
+
+  @staticmethod
+  def _transform_gluonts_instance_to_df(
+      ts: dict,
+      last_n: int | None = None,
+  ) -> pd.DataFrame:
+    start_period = ts["start"]
+    start_ds, freq = start_period.to_timestamp(), start_period.freq
+    target = ts["target"]
+    ds = pd.date_range(start=start_ds, freq=freq, periods=len(target))
+    if last_n is not None:
+      target = target[-last_n:]
+      ds = ds[-last_n:]
+    ts_df = pd.DataFrame({"unique_id": ts["item_id"], "ds": ds, "y": target})
+    return ts_df
+
+  @staticmethod
+  def _transform_gluonts_dataset_to_df(
+      gluonts_dataset: Dataset,
+      last_n: int | None = None,
+  ) -> pd.DataFrame:
+    with multiprocessing.Pool(os.cpu_count()) as pool:  # Create a process pool
+      results = pool.map(
+          parallel_transform, zip(gluonts_dataset, repeat(last_n))
+      )
+    df = pd.concat(results)
+    df = df.reset_index(drop=True)
+    return df
+
+  @property
+  def train_df(self) -> pd.DataFrame:
+    train_df = self._transform_gluonts_dataset_to_df(self.gluonts_train_dataset)
+    return train_df
+
+  @property
+  def test_df(self) -> pd.DataFrame:
+    test_df = self._transform_gluonts_dataset_to_df(
+        self.gluonts_test_dataset,
+        last_n=self.horizon,
+    )
+    # Make sure that only the first backtest window is used for evaluation on `traffic` / `exchange_rate` datasets
+    return test_df.groupby("unique_id", sort=False).head(self.horizon)
+
+  def save_dataframe(self, df: pd.DataFrame, file_name: str):
+    df.to_csv(f"{self.results_dir}/{file_name}", index=False)
+
+  def save_results(
+      self, fcst_df: pd.DataFrame, total_time: float, model_name: str
+  ):
+    self.save_dataframe(
+        fcst_df,
+        f"{model_name}-{self.dataset}-fcst.csv",
+    )
+    time_df = pd.DataFrame({"time": [total_time], "model": model_name})
+    self.save_dataframe(
+        time_df,
+        f"{model_name}-{self.dataset}-time.csv",
+    )
+
+  def fcst_from_level_to_quantiles(
+      self,
+      fcst_df: pd.DataFrame,
+      model_name: str,
+  ) -> pd.DataFrame:
+    fcst_df = fcst_df.copy()
+    cols = ["unique_id", "ds", model_name]
+    for q in self.quantiles:
+      if q == 0.5:
+        col = f"{model_name}"
+      else:
+        lv = int(100 - 200 * q)
+        hi_or_lo = "lo" if lv > 0 else "hi"
+        lv = abs(lv)
+        col = f"{model_name}-{hi_or_lo}-{lv}"
+      q_col = f"{model_name}-q-{q}"
+      fcst_df[q_col] = fcst_df[col].values
+      cols.append(q_col)
+    return fcst_df[cols]
+
+  def evaluate_models(self, models: List[str]) -> pd.DataFrame:
+    fcsts_df = []
+    times_df = []
+    for model in models:
+      fcst_method_df = pd.read_csv(
+          f"{self.results_dir}/{model}-{self.dataset}-fcst.csv"
+      ).set_index(["unique_id", "ds"])
+      fcsts_df.append(fcst_method_df)
+      time_method_df = pd.read_csv(
+          f"{self.results_dir}/{model}-{self.dataset}-time.csv"
+      )
+      times_df.append(time_method_df)
+    fcsts_df = pd.concat(fcsts_df, axis=1).reset_index()
+    fcsts_df["ds"] = pd.to_datetime(fcsts_df["ds"])
+    times_df = pd.concat(times_df)
+    return self.evaluate_from_predictions(
+        models=models, fcsts_df=fcsts_df, times_df=times_df
+    )
+
+  def evaluate_from_predictions(
+      self, models: List[str], fcsts_df: pd.DataFrame, times_df: pd.DataFrame
+  ) -> pd.DataFrame:
+    test_df = self.test_df
+    train_df = self.train_df
+    test_df = test_df.merge(fcsts_df, how="left")
+    assert test_df.isna().sum().sum() == 0, "merge contains nas"
+    # point evaluation
+    point_fcsts_cols = ["unique_id", "ds", "y"] + models
+    test_df["unique_id"] = test_df["unique_id"].astype(str)
+    train_df["unique_id"] = train_df["unique_id"].astype(str)
+    mase_seas = partial(mase, seasonality=self.seasonality)
+    eval_df = evaluate(
+        test_df[point_fcsts_cols],
+        train_df=train_df,
+        metrics=[smape, mase_seas, mae],
+    )
+    # probabilistic evaluation
+    eval_prob_df = []
+    for q in self.quantiles:
+      prob_cols = [f"{model}-q-{q}" for model in models]
+      eval_q_df = quantile_loss(test_df, models=prob_cols, q=q)
+      eval_q_df[prob_cols] = eval_q_df[prob_cols] * self.horizon
+      eval_q_df = eval_q_df.rename(columns=dict(zip(prob_cols, models)))
+      eval_q_df["metric"] = f"quantile-loss-{q}"
+      eval_prob_df.append(eval_q_df)
+    eval_prob_df = pd.concat(eval_prob_df)
+    eval_prob_df = eval_prob_df.groupby("metric").sum().reset_index()
+    total_y = test_df["y"].sum()
+    eval_prob_df[models] = eval_prob_df[models] / total_y
+    eval_prob_df["metric"] = "scaled_crps"
+    eval_df = pd.concat([eval_df, eval_prob_df]).reset_index(drop=True)
+    eval_df = eval_df.groupby("metric").mean(numeric_only=True).reset_index()
+    eval_df = eval_df.melt(
+        id_vars="metric", value_name="value", var_name="model"
+    )
+    times_df.insert(0, "metric", "time")
+    times_df = times_df.rename(columns={"time": "value"})
+    eval_df = pd.concat([eval_df, times_df])
+    eval_df.insert(0, "dataset", self.dataset)
+    eval_df = eval_df.sort_values(["dataset", "metric", "model"])
+    eval_df = eval_df.reset_index(drop=True)
+    return eval_df
+
+
+if __name__ == "__main__":
+  multiprocessing.set_start_method("spawn")

v1/experiments/long_horizon_benchmarks/README.md

@@ -0,0 +1,45 @@
+# Extended Benchmarks
+
+We benchmark on the original test set for ETT datasets as per long horizon benchmark papers (see [here](https://openreview.net/forum?id=pCbC3aQB5W) for example.) In the original benchmark, rolling validation task on all test windows (with a stride of 1) is considered. While we can easily run our method on this task, the baselines can take a very long time to run. Therefore we present results on a modified task with stride between windows set to Horizon length i.e all disjoint horizons in the test period is considered.
+
+All experiments were performed on a [g2-standard-32](https://cloud.google.com/compute/docs/gpus). We compare TimesFM with [Amazon-Chronos](https://github.com/amazon-science/chronos-forecasting).
+
+## Running TimesFM on the benchmark
+
+We need to add the following packages for running these benchmarks. Follow the installation instructions till before `poetry lock`. Then,
+
+```
+poetry add git+https://github.com/awslabs/gluon-ts.git
+poetry add git+https://github.com/amazon-science/chronos-forecasting.git
+poetry lock
+poetry install --only pax
+```
+Note that for now only the pax version runs on this benchmark, because we had to remove the old tf dependency from the pytorch version. We will fix this issue soon.
+
+To run the timesfm on the benchmark do:
+
+```
+poetry run python3 -m experiments.long_horizon_benchmarks.run_eval \
+--model_path=google/timesfm-1.0-200m --backend="gpu" \
+--pred_len=96 --context_len=512 --dataset=etth1
+```
+
+In the above, `<model_path>` should point to the checkpoint directory that can be downloaded from HuggingFace. 
+
+For running chronos on the same benchmark you can run the command,
+
+```
+poetry run python3 -m experiments.long_horizon_benchmarks.run_eval \
+--model_path=amazon/chronos-t5-mini --backend="gpu" \
+--pred_len=96 --context_len=512 --dataset=etth1
+```
+
+You can change the model size from "mini" to "large" as required. The datasets we benchmark on are etth1, etth2, ettm1 and ettm2.
+
+## Benchmark Results for TimesFM-1.0
+
+![Benchmark Results Table](./tfm_long_horizon.png)
+
+We compare the performance on horizon lengths of 96, 192 and 336, while context length is held fixed at 512.
+
+We can see that TimesFM performs the best in terms of both wape and smape. More importantly it is much faster than the other methods, in particular it is more than 1000x faster than Chronos (Large).

v1/experiments/long_horizon_benchmarks/run_eval.py

@@ -0,0 +1,238 @@
+# Copyright 2024 The Google Research Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Eval pipeline."""
+
+import json
+import os
+import sys
+import time
+from absl import flags
+import chronos
+import numpy as np
+import pandas as pd
+import timesfm
+from timesfm import data_loader
+import torch
+import tqdm
+
+FLAGS = flags.FLAGS
+
+_BATCH_SIZE = flags.DEFINE_integer("batch_size", 64,
+                                   "Batch size for the randomly sampled batch")
+_DATASET = flags.DEFINE_string("dataset", "etth1", "The name of the dataset.")
+
+_MODEL_PATH = flags.DEFINE_string("model_path", "google/timesfm-2.0-500m-jax",
+                                  "The name of the model.")
+_DATETIME_COL = flags.DEFINE_string("datetime_col", "date",
+                                    "Column having datetime.")
+_NUM_COV_COLS = flags.DEFINE_list("num_cov_cols", None,
+                                  "Column having numerical features.")
+_CAT_COV_COLS = flags.DEFINE_list("cat_cov_cols", None,
+                                  "Column having categorical features.")
+_TS_COLS = flags.DEFINE_list("ts_cols", None, "Columns of time-series features")
+_NORMALIZE = flags.DEFINE_bool("normalize", True,
+                               "normalize data for eval or not")
+_CONTEXT_LEN = flags.DEFINE_integer("context_len", 2048,
+                                    "Length of the context window")
+_PRED_LEN = flags.DEFINE_integer("pred_len", 96, "prediction length.")
+_BACKEND = flags.DEFINE_string("backend", "gpu", "backend to use")
+_RESULTS_DIR = flags.DEFINE_string("results_dir", "./results/long_horizon",
+                                   "results directory")
+
+DATA_DICT = {
+    "ettm2": {
+        "boundaries": [34560, 46080, 57600],
+        "data_path": "./datasets/ETT-small/ETTm2.csv",
+        "freq": "15min",
+    },
+    "ettm1": {
+        "boundaries": [34560, 46080, 57600],
+        "data_path": "./datasets/ETT-small/ETTm1.csv",
+        "freq": "15min",
+    },
+    "etth2": {
+        "boundaries": [8640, 11520, 14400],
+        "data_path": "./datasets/ETT-small/ETTh2.csv",
+        "freq": "H",
+    },
+    "etth1": {
+        "boundaries": [8640, 11520, 14400],
+        "data_path": "./datasets/ETT-small/ETTh1.csv",
+        "freq": "H",
+    },
+    "elec": {
+        "boundaries": [18413, 21044, 26304],
+        "data_path": "./datasets/electricity/electricity.csv",
+        "freq": "H",
+    },
+    "traffic": {
+        "boundaries": [12280, 14036, 17544],
+        "data_path": "./datasets/traffic/traffic.csv",
+        "freq": "H",
+    },
+    "weather": {
+        "boundaries": [36887, 42157, 52696],
+        "data_path": "./datasets/weather/weather.csv",
+        "freq": "10min",
+    },
+}
+
+QUANTILES = list(np.arange(1, 10) / 10.0)
+EPS = 1e-7
+
+
+def get_forecasts(model_path, model, past, freq, pred_len):
+  """Get forecasts."""
+  if model_path.startswith("amazon"):
+    out = model.predict(
+        torch.tensor(past),
+        prediction_length=pred_len,
+        limit_prediction_length=False,
+    )
+    out = out.numpy()
+    out = np.median(out, axis=1)
+  else:
+    lfreq = [freq] * past.shape[0]
+    _, out = model.forecast(list(past), lfreq)
+    out = out[:, :, 5]
+  return out
+
+
+def _mse(y_pred, y_true):
+  """mse loss."""
+  return np.square(y_pred - y_true)
+
+
+def _mae(y_pred, y_true):
+  """mae loss."""
+  return np.abs(y_pred - y_true)
+
+
+def _smape(y_pred, y_true):
+  """_smape loss."""
+  abs_diff = np.abs(y_pred - y_true)
+  abs_val = (np.abs(y_true) + np.abs(y_pred)) / 2
+  abs_val = np.where(abs_val > EPS, abs_val, 1.0)
+  abs_diff = np.where(abs_val > EPS, abs_diff, 0.0)
+  return abs_diff / abs_val
+
+
+def eval():
+  """Eval pipeline."""
+  dataset = _DATASET.value
+  data_path = DATA_DICT[dataset]["data_path"]
+  freq = DATA_DICT[dataset]["freq"]
+  int_freq = timesfm.freq_map(freq)
+  boundaries = DATA_DICT[dataset]["boundaries"]
+
+  data_df = pd.read_csv(open(data_path, "r"))
+
+  if _TS_COLS.value is not None:
+    ts_cols = DATA_DICT[dataset]["ts_cols"]
+    num_cov_cols = DATA_DICT[dataset]["num_cov_cols"]
+    cat_cov_cols = DATA_DICT[dataset]["cat_cov_cols"]
+  else:
+    ts_cols = [col for col in data_df.columns if col != _DATETIME_COL.value]
+    num_cov_cols = None
+    cat_cov_cols = None
+  batch_size = min(_BATCH_SIZE.value, len(ts_cols))
+  dtl = data_loader.TimeSeriesdata(
+      data_path=data_path,
+      datetime_col=_DATETIME_COL.value,
+      num_cov_cols=num_cov_cols,
+      cat_cov_cols=cat_cov_cols,
+      ts_cols=np.array(ts_cols),
+      train_range=[0, boundaries[0]],
+      val_range=[boundaries[0], boundaries[1]],
+      test_range=[boundaries[1], boundaries[2]],
+      hist_len=_CONTEXT_LEN.value,
+      pred_len=_PRED_LEN.value,
+      batch_size=batch_size,
+      freq=freq,
+      normalize=_NORMALIZE.value,
+      epoch_len=None,
+      holiday=False,
+      permute=False,
+  )
+  eval_itr = dtl.tf_dataset(mode="test",
+                            shift=_PRED_LEN.value).as_numpy_iterator()
+  model_path = _MODEL_PATH.value
+  if model_path.startswith("amazon"):
+    model = chronos.ChronosPipeline.from_pretrained(
+        model_path,
+        device_map="auto",
+        torch_dtype=torch.bfloat16,
+    )
+  else:
+    model = timesfm.TimesFm(
+        hparams=timesfm.TimesFmHparams(
+            backend="gpu",
+            per_core_batch_size=32,
+            horizon_len=128,
+            num_layers=50,
+            context_len=_CONTEXT_LEN.value,
+            use_positional_embedding=False,
+        ),
+        checkpoint=timesfm.TimesFmCheckpoint(huggingface_repo_id=model_path),
+    )
+  smape_run_losses = []
+  mse_run_losses = []
+  mae_run_losses = []
+
+  num_elements = 0
+  abs_sum = 0
+  start_time = time.time()
+
+  for batch in tqdm.tqdm(eval_itr):
+    past = batch[0]
+    actuals = batch[3]
+    forecasts = get_forecasts(model_path, model, past, int_freq,
+                              _PRED_LEN.value)
+    forecasts = forecasts[:, 0:actuals.shape[1]]
+    mae_run_losses.append(_mae(forecasts, actuals).sum())
+    mse_run_losses.append(_mse(forecasts, actuals).sum())
+    smape_run_losses.append(_smape(forecasts, actuals).sum())
+    num_elements += actuals.shape[0] * actuals.shape[1]
+    abs_sum += np.abs(actuals).sum()
+
+  mse_val = np.sum(mse_run_losses) / num_elements
+
+  result_dict = {
+      "mse": mse_val,
+      "smape": np.sum(smape_run_losses) / num_elements,
+      "mae": np.sum(mae_run_losses) / num_elements,
+      "wape": np.sum(mae_run_losses) / abs_sum,
+      "nrmse": np.sqrt(mse_val) / (abs_sum / num_elements),
+      "num_elements": num_elements,
+      "abs_sum": abs_sum,
+      "total_time": time.time() - start_time,
+      "model_path": model_path,
+      "dataset": dataset,
+      "freq": freq,
+      "pred_len": _PRED_LEN.value,
+      "context_len": _CONTEXT_LEN.value,
+  }
+  run_id = np.random.randint(10000)
+  save_path = os.path.join(_RESULTS_DIR.value, str(run_id))
+  print(f"Saving results to {save_path}", flush=True)
+  os.makedirs(save_path, exist_ok=True)
+  with open(os.path.join(save_path, "results.json"), "w") as f:
+    json.dump(result_dict, f)
+  print(result_dict, flush=True)
+
+
+if __name__ == "__main__":
+  FLAGS = flags.FLAGS
+  FLAGS(sys.argv)
+  eval()