Merge pull request #49 from Nixtla/docs/vn1-benchmark

docs: VN1 Forecasting Competition
Nixtla · Dec 13, 2024 · d0fcff1 · d0fcff1
2 parents fb582fd + 41d6845
commit d0fcff1
Show file tree

Hide file tree

Showing 7 changed files with 221 additions and 6 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -10,3 +10,4 @@
 ^codecov\.yml$
 ^cran-comments\.md$
 ^CRAN-SUBMISSION$
+^experiments$
diff --git a/CRAN-SUBMISSION b/CRAN-SUBMISSION
@@ -1,3 +1,3 @@
-Version: 0.6.1
-Date: 2024-10-10 03:09:51 UTC
-SHA: c1a9515794c08f1a3a83f5c744486b35b8a5f7c4
+Version: 0.6.2
+Date: 2024-10-28 23:00:07 UTC
+SHA: fb582fdb5029c70a03f648237f5954fe52425263
diff --git a/_pkgdown.yml b/_pkgdown.yml
@@ -11,8 +11,6 @@ navbar:
       href: articles/get-started.html
     - text: "Reference"
       href: reference/index.html
-    - text: "Nixtla"
-      href: https://docs.nixtla.io/
     - text: "News"
       href: news/index.html
     - text: "Articles"
@@ -39,10 +37,15 @@ navbar:
           href: articles/setting-up-your-api-key.html
         - text: "Special Topics"
           href: articles/special-topics.html
-
+    - text: "Experiments"
+      menu:
+        - text: "VN1 Competition"
+          href: articles/vn1-forecasting-competition.html
   right:
     - icon: "fa-github"
       href: https://github.com/Nixtla/nixtlar
+    - text: "Nixtla"
+      href: https://docs.nixtla.io/
 
 home:
   links:

diff --git a/experiments/vn1-forecasting-competition/functions.R b/experiments/vn1-forecasting-competition/functions.R
@@ -0,0 +1,78 @@
+
+# Functions for VN1 Forecasting Competition ---- 
+
+read_and_prepare_data <- function(dataset){
+  # Reads data in wide format and returns it in long format with columns `unique_id`, `ds`, and `y`
+  url <- get_dataset_url(dataset)
+  df_wide <- fread(url)
+  df_wide <- df_wide |> 
+    mutate(unique_id = paste0(Client, "/", Warehouse, "/", Product)) |> 
+    select(c(unique_id, everything())) |> 
+    select(-c(Client, Warehouse, Product))
+
+  df <- pivot_longer(
+    data = df_wide, 
+    cols = -unique_id, 
+    names_to = "ds", 
+    values_to = "y"
+  )
+
+  if(startsWith(dataset, "winners")){
+    names(df)[which(names(df) == "y")] <- dataset
+  }
+
+  return(df)
+}
+
+get_train_data <- function(df0, df1){
+  # Merges training data from phase 0 and phase 1 and removes leading zeros 
+  df <- rbind(df0, df1) |> 
+    arrange(unique_id, ds)
+
+  df_clean <- df |> 
+    group_by(unique_id) |> 
+    mutate(cumsum = cumsum(y)) |>
+    filter(cumsum > 0) |> 
+    select(-cumsum) |> 
+    ungroup()
+
+  return(df_clean)
+}
+
+vn1_competition_evaluation <- function(test, forecast, model){
+  # Computes competition evaluation 
+  if(!is.character(forecast$ds)){
+    forecast$ds <- as.character(forecast$ds) # nixtlar returns timestamps for plotting 
+  }
+
+  res <- merge(forecast, test, by=c("unique_id", "ds"))
+
+  res <- res |> 
+    mutate(abs_err = abs(res[[model]]-res$y)) |> 
+    mutate(err = res[[model]]-res$y) 
+
+  abs_err = sum(res$abs_err, na.rm = TRUE)
+  err = sum(res$err, na.rm = TRUE)    
+  score = abs_err+abs(err)
+  score = score/sum(res$y)
+  score = round(score, 4)
+
+  return(score)
+}
+
+get_dataset_url <- function(dataset){
+  # Returns the url of the given competition dataset 
+  urls <- list(
+    phase0_sales = "https://www.datasource.ai/attachments/eyJpZCI6Ijk4NDYxNjE2NmZmZjM0MGRmNmE4MTczOGMyMzI2ZWI2LmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoiUGhhc2UgMCAtIFNhbGVzLmNzdiIsInNpemUiOjEwODA0NjU0LCJtaW1lX3R5cGUiOiJ0ZXh0L2NzdiJ9fQ", 
+    phase1_sales = "https://www.datasource.ai/attachments/eyJpZCI6ImM2OGQxNGNmNTJkZDQ1MTUyZTg0M2FkMDAyMjVlN2NlLmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoiUGhhc2UgMSAtIFNhbGVzLmNzdiIsInNpemUiOjEwMTgzOTYsIm1pbWVfdHlwZSI6InRleHQvY3N2In19",
+    phase2_sales = "https://www.datasource.ai/attachments/eyJpZCI6IjhlNmJmNmU3ZTlhNWQ4NTcyNGVhNTI4YjAwNTk3OWE1LmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoiUGhhc2UgMiAtIFNhbGVzLmNzdiIsInNpemUiOjEwMTI0MzcsIm1pbWVfdHlwZSI6InRleHQvY3N2In19", 
+    winners1 = "https://www.datasource.ai/attachments/eyJpZCI6IjI1NDQxYmMyMTQ3MTA0MjJhMDcyYjllODcwZjEyNmY4LmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoicGhhc2UgMiBzdWJtaXNzaW9uIGV4YW1pbmUgc21vb3RoZWQgMjAyNDEwMTcgRklOQUwuY3N2Iiwic2l6ZSI6MTk5MzAzNCwibWltZV90eXBlIjoidGV4dC9jc3YifX0",
+    winners2 = "https://www.datasource.ai/attachments/eyJpZCI6IjU3ODhjZTUwYTU3MTg3NjFlYzMzOWU0ZTg3MWUzNjQxLmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoidm4xX3N1Ym1pc3Npb25fanVzdGluX2Z1cmxvdHRlLmNzdiIsInNpemUiOjM5MDkzNzksIm1pbWVfdHlwZSI6InRleHQvY3N2In19",
+    winners3 = "https://www.datasource.ai/attachments/eyJpZCI6ImE5NzcwNTZhMzhhMTc2ZWJjODFkMDMwMTM2Y2U2MTdlLmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoiYXJzYW5pa3phZF9zdWIuY3N2Iiwic2l6ZSI6Mzg4OTcyNCwibWltZV90eXBlIjoidGV4dC9jc3YifX0",
+    winners4 = "https://www.datasource.ai/attachments/eyJpZCI6ImVlZmUxYWY2NDFjOWMwM2IxMzRhZTc2MzI1Nzg3NzIxLmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoiVEZUX3R1bmVkX1YyX3NlZWRfNDIuY3N2Iiwic2l6ZSI6NjA3NDgzLCJtaW1lX3R5cGUiOiJ0ZXh0L2NzdiJ9fQ",
+    winners5 = "https://www.datasource.ai/attachments/eyJpZCI6IjMwMDEwMmY3NTNhMzlhN2YxNTk3ODYxZTI1N2Q2NzRmLmNzdiIsInN0b3JhZ2UiOiJzdG9yZSIsIm1ldGFkYXRhIjp7ImZpbGVuYW1lIjoiZGl2aW5lb3B0aW1pemVkd2VpZ2h0c2Vuc2VtYmxlLmNzdiIsInNpemUiOjE3OTU0NzgsIm1pbWVfdHlwZSI6InRleHQvY3N2In19"
+  )
+
+  return(urls[[dataset]])
+}
+
diff --git a/experiments/vn1-forecasting-competition/main.R b/experiments/vn1-forecasting-competition/main.R
@@ -0,0 +1,44 @@
+
+# VN1 Forecasting Competition Solution with nixtlar ---- 
+
+install.packages(c("nixtlar", "tidyverse", "data.table"))
+
+library(nixtlar)
+library(tidyverse)
+library(data.table)
+
+source("functions.R") # same directory as main.R
+
+## Load Data ---- 
+sales0 <- read_and_prepare_data("phase0_sales")
+sales1 <- read_and_prepare_data("phase1_sales")
+test_df <- read_and_prepare_data("phase2_sales")
+
+## Prepare Training Dataset ---- 
+train_df <- get_train_data(sales0, sales1)
+
+## Generate TimeGPT Forecast  ----
+
+# nixtla_client_setup(api_key = "Your API key here") 
+# Learn how to set up your API key here: https://nixtla.github.io/nixtlar/articles/setting-up-your-api-key.html
+
+fc <- nixtla_client_forecast(train_df, h=13, model="timegpt-1-long-horizon")
+
+## Visualize TimeGPT Forecast ----
+nixtla_client_plot(train_df, fc)
+
+## Evaluate TimeGPT & Top 5 Competition Solutions ----
+timegpt_score <- vn1_competition_evaluation(test_df, fc, "TimeGPT")
+
+scores <- lapply(1:5, function(i){ # Top 5 
+  winner_df <- read_and_prepare_data(paste0("winners", i))
+  vn1_competition_evaluation(test_df, winner_df, model = paste0("winners", i))
+})
+
+scores_df <- data.frame(
+  "Result" = c(paste0("Place #", 1:5), "TimeGPT"), 
+  "Score" = c(as.numeric(scores), timegpt_score)
+)
+
+scores_df <- scores_df |> arrange(Score)
+print(scores_df) # TimeGPT places 2nd! 
diff --git a/man/figures/vn1.png b/man/figures/vn1.png
diff --git a/vignettes/vn1-forecasting-competition.Rmd b/vignettes/vn1-forecasting-competition.Rmd
@@ -0,0 +1,89 @@
+---
+title: "VN1 Forecasting Competition"
+output: 
+  rmarkdown::html_vignette:
+    toc: true 
+    toc_depth: 2
+vignette: >
+  %\VignetteIndexEntry{VN1 Forecasting Competition}
+  %\VignetteEngine{knitr::rmarkdown}
+  %\VignetteEncoding{UTF-8}
+---
+```{r setup, include=FALSE}
+knitr::opts_chunk$set(
+  collapse = TRUE,
+  comment = "#>", 
+  fig.width = 7, 
+  fig.height = 4
+)
+```
+
+## Introduction 
+
+The [VN1 Forecasting Accuracy Challenge](https://www.datasource.ai/en/home/data-science-competitions-for-startups/phase-2-vn1-forecasting-accuracy-challenge/description) was a forecasting competition sponsored by SupChains, Syrup, and Flieber on the DataSource.ai platform. The competition ran from September 12 to October 17, 2024. Participants were tasked with predicting the next 13 weeks of sales for different products across multiple clients and warehouses. Submissions were evaluated based on their accuracy and bias against actual sales. 
+
+## TimeGPT 2nd Place Submission
+
+Using TimeGPT via `nixtlar`, it is possible to achieve **2nd place in the competition** with a score of **0.4651**. This result can be obtained with a zero-shot approach and the long-horizon model. Unlike the top five solutions, there is no need for fine-tuning or manually adjusting the results. The only preprocessing required is transforming the data from a wide to a long format and removing the leading zeros of each series, which represent a product-client-warehouse combination. 
+
+The competition provided prices as exogenous variables, but TimeGPT can achieve second place without using them.
+
+The official competition results and TimeGPT's score are shown below. 
+
+<div style="display: flex; justify-content: center;">
+  <table style="border-collapse: collapse; width: 60%;">
+    <thead>
+      <tr>
+        <th style="text-align: center; padding: 8px; border: 1px solid #ddd; width: 66.67%;">Model</th>
+        <th style="text-align: center; padding: 8px; border: 1px solid #ddd; width: 33.33%;">Score</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">1st</td>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">0.4637</td>
+      </tr>
+      <tr>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd; font-weight: bold;">TimeGPT</td>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd; font-weight: bold;">0.4651</td>
+      </tr>
+      <tr>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">2nd</td>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">0.4657</td>
+      </tr>
+      <tr>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">3rd</td>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">0.4758</td>
+      </tr>
+      <tr>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">4th</td>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">0.4774</td>
+      </tr>
+      <tr>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">5th</td>
+        <td style="text-align: center; padding: 8px; border: 1px solid #ddd;">0.4808</td>
+      </tr>
+    </tbody>
+  </table>
+</div>
+
+![](../man/figures/vn1.png) 
+
+TimeGPT was not an official entry in the competition as the rules required fully open-source solutions, and TimeGPT works through an API. However, with the same evaluation metric as in the competition, we can see that TimeGPT with a zero-shot approach using the long-horizon model and no exogenous variables would have placed 2nd overall. Furthermore, TimeGPT via the `nixtlar` package only requires a few lines of code, in contrast to the top five solutions that used multiple models and carefully crafted features.
+
+
+## Try It Yourself! 
+
+To reproduce TimeGPT's 2nd place submission, download the `main.R` and the `functions.R` scripts found in the `experiments/vn1-forecasting-competition` section of the `nixtlar` [GitHub repository](https://github.com/Nixtla/nixtlar) repository. Then run the `main.R` script. Make sure the `functions.R` script is in the same directory as the `main.R` script. 
+
+The output of the `main.R` script is the table shown in the previous section, which includes TimeGPT's result alongside the top five solutions.
+
+This experiment is independent of the `nixtlar` package and is not included as part of its code. 
+
+## References 
+
+- Vandeput, Nicolas. “VN1 Forecasting - Accuracy Challenge.” DataSource.ai, DataSource, 3 Oct. 2024, [https://www.datasource.ai/en/home/data-science-competitions-for-startups/phase-2-vn1-forecasting-accuracy-challenge/description](https://www.datasource.ai/en/home/data-science-competitions-for-startups/phase-2-vn1-forecasting-accuracy-challenge/description)
+
+- Garza, A., & Mergenthaler-Canseco, M. (2023). TimeGPT-1. [arXiv preprint arXiv:2310.03589](https://arxiv.org/abs/2310.03589).
+
+