{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 330 }, "id": "KmKX79qPmrao", "outputId": "6fe4b7ad-4cfe-4ef4-86f7-3abea192ee17" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: openpyxl in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (3.1.5)\n", "Requirement already satisfied: pandas in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (2.3.3)\n", "Requirement already satisfied: et-xmlfile in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from openpyxl) (2.0.0)\n", "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from pandas) (2025.3)\n", "Requirement already satisfied: numpy>=1.22.4 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from pandas) (2.2.6)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\pmarv\\appdata\\roaming\\python\\python310\\site-packages (from pandas) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from pandas) (2025.2)\n", "Requirement already satisfied: six>=1.5 in c:\\users\\pmarv\\appdata\\roaming\\python\\python310\\site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING: You are using pip version 21.2.3; however, version 25.3 is available.\n", "You should consider upgrading via the 'c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\python.exe -m pip install --upgrade pip' command.\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
MarketSale TypeDealer CodeSale DateSale MonthSale YearDummy IDPart Qty.Dealer PricePRC_UNIT_WEIGHTPRC_UNIT_CUBE
0NZLOTC225982023-09-0592023PartNo1218.37920.020.000616
1NZLRO345612025-07-1672025PartNo21150.71100.190.001029
2NZLRO345612025-07-1672025PartNo21150.71100.190.001029
3NZLRO345612025-07-1672025PartNo21150.71100.190.001029
4NZLRO456112025-07-0972025PartNo21150.71100.190.001029
\n", "
" ], "text/plain": [ " Market Sale Type Dealer Code Sale Date Sale Month Sale Year Dummy ID \\\n", "0 NZL OTC 22598 2023-09-05 9 2023 PartNo1 \n", "1 NZL RO 34561 2025-07-16 7 2025 PartNo2 \n", "2 NZL RO 34561 2025-07-16 7 2025 PartNo2 \n", "3 NZL RO 34561 2025-07-16 7 2025 PartNo2 \n", "4 NZL RO 45611 2025-07-09 7 2025 PartNo2 \n", "\n", " Part Qty. Dealer Price PRC_UNIT_WEIGHT PRC_UNIT_CUBE \n", "0 2 18.3792 0.02 0.000616 \n", "1 1 150.7110 0.19 0.001029 \n", "2 1 150.7110 0.19 0.001029 \n", "3 1 150.7110 0.19 0.001029 \n", "4 1 150.7110 0.19 0.001029 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%pip install openpyxl pandas\n", "import pandas as pd\n", "\n", "file_path = \"D:\\\\TimePassProjects\\\\col\\\\tr1\\\\data_without_key.xlsx\"\n", "df = pd.read_excel(file_path)\n", "display(df.head())" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "DIb7ntFmmz6H", "outputId": "f714191e-ff6f-4ab1-ee9a-d0617f001cc6" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of duplicate rows found: 706142\n", "Number of rows after removing duplicates: 342433\n" ] } ], "source": [ "num_duplicates = df.duplicated().sum()\n", "print(f\"Number of duplicate rows found: {num_duplicates}\")\n", "\n", "df_cleaned = df.drop_duplicates()\n", "print(f\"Number of rows after removing duplicates: {len(df_cleaned)}\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 660 }, "id": "rMYr7Pp9m1rC", "outputId": "a7407183-395a-40ad-f3cd-5c3facb6ccbb" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of rows after filtering: 319045\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
MarketSale TypeDealer CodeSale DateSale MonthSale YearDummy IDPart Qty.Dealer PricePRC_UNIT_WEIGHTPRC_UNIT_CUBE
0NZLOTC225982023-09-0592023PartNo1218.37920.020.000616
1NZLRO345612025-07-1672025PartNo21150.71100.190.001029
4NZLRO456112025-07-0972025PartNo21150.71100.190.001029
7NZLRO442422025-03-2632025PartNo21150.71100.190.001029
10NZLRO225912025-03-2532025PartNo21150.71100.190.001029
\n", "
" ], "text/plain": [ " Market Sale Type Dealer Code Sale Date Sale Month Sale Year Dummy ID \\\n", "0 NZL OTC 22598 2023-09-05 9 2023 PartNo1 \n", "1 NZL RO 34561 2025-07-16 7 2025 PartNo2 \n", "4 NZL RO 45611 2025-07-09 7 2025 PartNo2 \n", "7 NZL RO 44242 2025-03-26 3 2025 PartNo2 \n", "10 NZL RO 22591 2025-03-25 3 2025 PartNo2 \n", "\n", " Part Qty. Dealer Price PRC_UNIT_WEIGHT PRC_UNIT_CUBE \n", "0 2 18.3792 0.02 0.000616 \n", "1 1 150.7110 0.19 0.001029 \n", "4 1 150.7110 0.19 0.001029 \n", "7 1 150.7110 0.19 0.001029 \n", "10 1 150.7110 0.19 0.001029 " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Index: 319045 entries, 0 to 1048487\n", "Data columns (total 11 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 Market 319045 non-null object \n", " 1 Sale Type 319045 non-null object \n", " 2 Dealer Code 319045 non-null int64 \n", " 3 Sale Date 319045 non-null datetime64[ns]\n", " 4 Sale Month 319045 non-null int64 \n", " 5 Sale Year 319045 non-null int64 \n", " 6 Dummy ID 319045 non-null object \n", " 7 Part Qty. 319045 non-null int64 \n", " 8 Dealer Price 319045 non-null float64 \n", " 9 PRC_UNIT_WEIGHT 319045 non-null float64 \n", " 10 PRC_UNIT_CUBE 319045 non-null float64 \n", "dtypes: datetime64[ns](1), float64(3), int64(4), object(3)\n", "memory usage: 29.2+ MB\n" ] } ], "source": [ "df_filtered = df_cleaned[(df_cleaned['Part Qty.'] >= 0) &\n", " (df_cleaned['PRC_UNIT_WEIGHT'] > 0) &\n", " (df_cleaned['PRC_UNIT_CUBE'] > 0) &\n", " (df_cleaned['Dealer Price'].notna())]\n", "\n", "print(f\"Number of rows after filtering: {len(df_filtered)}\")\n", "display(df_filtered.head())\n", "df_filtered.info()" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Success! LightGBM version: 4.6.0\n" ] } ], "source": [ "try:\n", " import lightgbm as lgb\n", " print(f\"Success! LightGBM version: {lgb.__version__}\")\n", "except ImportError:\n", " print(\"LightGBM is NOT installed in this environment.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import math\n", "import pickle\n", "import warnings\n", "import time\n", "from typing import Optional, Dict, Any, Tuple, List\n", "warnings.filterwarnings(\"ignore\")\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.decomposition import TruncatedSVD\n", "from sklearn.calibration import CalibratedClassifierCV\n", "from sklearn.linear_model import ElasticNet, Ridge, LogisticRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.model_selection import TimeSeriesSplit\n", "from sklearn.metrics import(\n", " mean_squared_error, mean_absolute_error, r2_score\n", ")\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from scipy.fft import fft, fftfreq\n", "from scipy.signal import correlate\n", "from scipy.stats import pearsonr\n", "from sklearn.metrics import mean_absolute_error\n", "from sklearn.preprocessing import StandardScaler\n", "import lightgbm as lgb\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.preprocessing import MinMaxScaler\n", "import tensorflow as tf\n", "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import LSTM, Dense, Dropout, Input\n", "from tensorflow.keras.optimizers import Adam\n", "from tensorflow.keras.callbacks import EarlyStopping\n", "\n", "# Optional SHAP library\n", "try:\n", " import shap\n", " SHAP_AVAILABLE = True\n", " print(\"[INFO] SHAP is available.\")\n", "except ImportError:\n", " SHAP_AVAILABLE = False\n", " print(\"[INFO] SHAP is NOT available.\")\n", "\n", "# ----------------------------- Utilities -----------------------------\n", "def safe_mkdir(path: str):\n", " os.makedirs(path, exist_ok=True)\n", "\n", "def rmse(y_true, y_pred):\n", " return math.sqrt(mean_squared_error(y_true, y_pred))\n", "\n", "def wmape(y_true, y_pred):\n", " denom = np.sum(np.abs(y_true))\n", " return np.sum(np.abs(y_true - y_pred)) / denom * 100.0 if denom != 0 else np.nan\n", "\n", "def mape(y_true, y_pred):\n", " mask = y_true != 0\n", " return np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask])) * 100.0 if mask.sum() > 0 else np.nan\n", "\n", "# ----------------------------- Padding with Inactive Filtering (Moved to Top) -----------------------------\n", "def pad_monthly_series_and_filter_inactive(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Padding series to monthly grid and filtering inactive pairs...\")\n", " df = df.copy()\n", " df[\"MonthStart\"] = pd.to_datetime(df[\"MonthStart\"])\n", " min_date = df[\"MonthStart\"].min()\n", " max_date = df[\"MonthStart\"].max()\n", " all_months = pd.date_range(start=min_date, end=max_date, freq=\"MS\")\n", " print(f\"[INFO] Date range: {min_date.strftime('%Y-%m')} to {max_date.strftime('%Y-%m')}. Total months: {len(all_months)}\")\n", " df_active = df.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0)\n", " print(f\"[INFO] Filtered from {df.shape[0]} to {df_active.shape[0]} records after removing inactive pairs.\")\n", " if df_active.empty:\n", " print(\"[WARNING] No active series found after filtering.\")\n", " return df_active\n", " full_index = pd.MultiIndex.from_product(\n", " [df_active[\"DealerCode\"].unique(), df_active[\"DummyID\"].unique(), all_months],\n", " names=[\"DealerCode\", \"DummyID\", \"MonthStart\"]\n", " )\n", " df_indexed = df_active.set_index([\"DealerCode\", \"DummyID\", \"MonthStart\"])\n", " df_padded = df_indexed.reindex(full_index, fill_value=0).reset_index()\n", " df_padded[\"OTC_Flag\"] = (df_padded[\"OTC_Qty\"] > 0).astype(int)\n", " for col in [\"PartQty\", \"OTC_Qty\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]:\n", " df_padded[col] = pd.to_numeric(df_padded[col], errors=\"coerce\").fillna(0)\n", " df_final = df_padded.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0).reset_index(drop=True)\n", " print(f\"[INFO] Padding complete. Final shape: {df_final.shape}\")\n", " return df_final\n", "\n", "# ----------------------------- Data helpers (Same as before) -----------------------------\n", "def aggregate_monthly(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Aggregating data to monthly level...\")\n", " df = df.copy()\n", " colmap = {}\n", " for c in df.columns:\n", " lc = c.lower().strip()\n", " if lc in (\"dealer code\", \"dealercode\", \"dealer_code\", \"dealercode\"):\n", " colmap[c] = \"DealerCode\"\n", " if lc in (\"dummy id\", \"dummyid\", \"partno\", \"part_no\", \"dummy_id\"):\n", " colmap[c] = \"DummyID\"\n", " if lc in (\"sale date\", \"saledate\", \"sale_date\"):\n", " colmap[c] = \"SaleDate\"\n", " if lc in (\"part qty.\", \"part qty\", \"partqty\", \"qty\", \"quantity\"):\n", " colmap[c] = \"PartQty\"\n", " if lc in (\"sale type\", \"saletype\", \"sale_type\"):\n", " colmap[c] = \"SaleType\"\n", " if lc in (\"dealer price\", \"dealerprice\", \"price\"):\n", " colmap[c] = \"DealerPrice\"\n", " if lc in (\"prc_unit_weight\", \"weight\", \"prc unit weight\"):\n", " colmap[c] = \"PRC_UNIT_WEIGHT\"\n", " if lc in (\"prc_unit_cube\", \"cube\", \"prc unit cube\"):\n", " colmap[c] = \"PRC_UNIT_CUBE\"\n", " if lc in (\"otc_qty\",\"otc qty\",\"otcqty\"):\n", " colmap[c] = \"OTC_Qty\"\n", " df = df.rename(columns=colmap)\n", " if \"MonthStart\" in df.columns and \"PartQty\" in df.columns:\n", " out = df.copy()\n", " out[\"MonthStart\"] = pd.to_datetime(out[\"MonthStart\"])\n", " if \"OTC_Qty\" not in out.columns:\n", " if \"SaleType\" in out.columns:\n", " out[\"OTC_Qty\"] = out[\"PartQty\"] * 0\n", " mask = out[\"SaleType\"].astype(str).str.upper() == \"OTC\"\n", " out.loc[mask, \"OTC_Qty\"] = out.loc[mask, \"PartQty\"]\n", " else:\n", " out[\"OTC_Qty\"] = 0\n", " out[\"OTC_Flag\"] = (out[\"OTC_Qty\"] > 0).astype(int)\n", " out[\"DealerPrice\"] = out.get(\"DealerPrice\", 0.0)\n", " out[\"PRC_UNIT_WEIGHT\"] = out.get(\"PRC_UNIT_WEIGHT\", 0.0)\n", " out[\"PRC_UNIT_CUBE\"] = out.get(\"PRC_UNIT_CUBE\", 0.0)\n", " return out[[\"DealerCode\", \"DummyID\", \"MonthStart\", \"PartQty\", \"OTC_Qty\", \"OTC_Flag\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]]\n", " if \"SaleDate\" not in df.columns:\n", " raise ValueError(\"Input must have 'SaleDate' or 'MonthStart' + 'PartQty'.\")\n", " df[\"SaleDate\"] = pd.to_datetime(df[\"SaleDate\"])\n", " df[\"MonthStart\"] = df[\"SaleDate\"].dt.to_period(\"M\").dt.to_timestamp()\n", " agg = df.groupby([\"DealerCode\", \"DummyID\", \"MonthStart\"], as_index=False).agg(\n", " PartQty=(\"PartQty\", \"sum\"),\n", " OTC_Qty=(\"PartQty\", lambda x: x[df.loc[x.index, \"SaleType\"].astype(str).str.upper() == \"OTC\"].sum() if \"SaleType\" in df.columns else 0),\n", " DealerPrice=(\"DealerPrice\", \"mean\"),\n", " PRC_UNIT_WEIGHT=(\"PRC_UNIT_WEIGHT\", \"mean\"),\n", " PRC_UNIT_CUBE=(\"PRC_UNIT_CUBE\", \"mean\"),\n", " )\n", " agg[\"OTC_Qty\"] = agg[\"OTC_Qty\"].fillna(0).astype(int)\n", " agg[\"OTC_Flag\"] = (agg[\"OTC_Qty\"] > 0).astype(int)\n", " agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]] = agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]].fillna(0.0)\n", " print(f\"[INFO] Aggregated to {len(agg)} monthly records.\")\n", " return agg\n", "\n", "def add_time_series_features(df: pd.DataFrame, max_lag: int = 12) -> pd.DataFrame:\n", " print(f\"[INFO] Adding time series features (max_lag={max_lag})...\")\n", " df = df.sort_values([\"DealerCode\", \"DummyID\", \"MonthStart\"]).copy()\n", " rows = []\n", " grp = df.groupby([\"DealerCode\", \"DummyID\"])\n", " for (d, p), g in grp:\n", " g = g.sort_values(\"MonthStart\").copy()\n", " for lag in range(1, max_lag+1):\n", " g[f\"qty_lag_{lag}\"] = g[\"PartQty\"].shift(lag).fillna(0)\n", " g[f\"otc_lag_{lag}\"] = g[\"OTC_Flag\"].shift(lag).fillna(0)\n", " g[\"qty_sum_1_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"qty_sum_1_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"nonzero_count_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).apply(lambda x: np.sum(x>0)).shift(1).fillna(0)\n", " last_pos = -1\n", " t_since = []\n", " for idx, val in enumerate(g[\"PartQty\"].values):\n", " if val > 0:\n", " last_pos = idx\n", " t_since.append(0)\n", " else:\n", " t_since.append(idx - last_pos if last_pos != -1 else g.shape[0]+1)\n", " g[\"time_since_last_sale\"] = pd.Series(t_since).astype(float)\n", " g[\"qty_ma_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"qty_ma_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"otc_share_slope_3\"] = g[\"OTC_Flag\"].rolling(3, min_periods=1).apply(lambda x: np.polyfit(np.arange(len(x)), x, 1)[0] if len(x)>1 else 0).shift(1).fillna(0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added {len([c for c in out.columns if c.startswith(('qty_lag_', 'otc_lag_'))])} lag features.\")\n", " return out\n", "\n", "def croston_forecast(series: np.ndarray, alpha: float = 0.3) -> np.ndarray:\n", " n = len(series)\n", " if n == 0:\n", " return np.zeros(0)\n", " demand = np.asarray(series, dtype=float)\n", " forecast = np.zeros(n)\n", " nz_idx = np.where(demand > 0)[0]\n", " if len(nz_idx) == 0:\n", " return forecast\n", " last = nz_idx[0]\n", " z = demand[last]\n", " p = 1.0\n", " for t in range(last+1, n):\n", " if demand[t] > 0:\n", " interval = t - last\n", " z = alpha * demand[t] + (1-alpha) * z\n", " p = alpha * interval + (1-alpha) * p\n", " last = t\n", " forecast[t] = z / max(1e-9, p)\n", " forecast[:last+1] = forecast[last+1] if last+1 < n else forecast[last]\n", " return forecast\n", "\n", "def add_croston_features(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Adding Croston features...\")\n", " rows = []\n", " for (d,p), g in df.groupby([\"DealerCode\",\"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " g[\"croston_forecast\"] = croston_forecast(g[\"PartQty\"].values, alpha=0.3)\n", " g[\"croston_ratio\"] = np.where(g[\"qty_sum_1_3\"]!=0, g[\"croston_forecast\"]/g[\"qty_sum_1_3\"], 0.0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added Croston features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- SHOS with Train-Only Sparsity (Same as before) -----------------------------\n", "def compute_shos_for_series(y: np.ndarray, alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5,\n", " n_train: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n", " n = len(y)\n", " if n == 0:\n", " return np.zeros(0), np.zeros(0), np.zeros(0)\n", " if n_train is not None and n_train > 0:\n", " y_sparsity = y[:n_train]\n", " else:\n", " y_sparsity = y\n", " nonzero_count = np.sum(y_sparsity > 0)\n", " sparsity = 1.0 - (nonzero_count / max(1, len(y_sparsity)))\n", " if adaptivity:\n", " alpha = alpha_base * (1.0 - sparsity_k * sparsity)\n", " beta = beta_base * (1.0 - sparsity_k * sparsity)\n", " last_pos_all = np.where(y > 0)[0]\n", " if len(last_pos_all) > 0 and (n - 1 - last_pos_all[-1]) <= 3:\n", " alpha = min(0.9, alpha * 1.2)\n", " beta = min(0.9, beta * 1.2)\n", " else:\n", " alpha, beta = alpha_base, beta_base\n", " # --- IMPROVED INITIALIZATION (Point 6) ---\n", " if n >= 3:\n", " init_window = 3\n", " y_init_window = y[:init_window]\n", " q0 = float(np.sum(y_init_window > 0)) / max(1, init_window)\n", " positive_y_init = y_init_window[y_init_window > 0]\n", " if len(positive_y_init) > 0:\n", " z0 = float(np.mean(positive_y_init))\n", " else:\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " else:\n", " q0 = float(np.sum(y > 0)) / max(1, n)\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " # --- END IMPROVED INITIALIZATION ---\n", " q_prev, z_prev = q0, z0\n", " q, z, f = np.zeros(n), np.zeros(n), np.zeros(n)\n", " for t in range(n):\n", " yt = y[t]\n", " if yt > 0:\n", " z_curr = alpha * yt + (1 - alpha) * z_prev\n", " q_curr = beta * 1.0 + (1 - beta) * q_prev\n", " else:\n", " z_curr = z_prev\n", " q_curr = beta * 0.0 + (1 - beta) * q_prev\n", " q[t], z[t], f[t] = q_curr, z_curr, q_curr * z_curr\n", " q_prev, z_prev = q_curr, z_curr\n", " return q, z, f\n", "\n", "def add_shos_features_with_ntrain(df: pd.DataFrame, n_train_per_series: Dict[Tuple, int],\n", " alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5) -> pd.DataFrame:\n", " print(f\"[INFO] Adding SHOS features with train-only sparsity (k={sparsity_k})...\")\n", " rows = []\n", " for (d, p), g in df.groupby([\"DealerCode\", \"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " y = g[\"PartQty\"].values.astype(float)\n", " n_train = n_train_per_series.get((d, p), len(y))\n", " q, z, f = compute_shos_for_series(y, alpha_base, beta_base, adaptivity, sparsity_k, n_train)\n", " g[\"q_shos\"] = q\n", " g[\"z_shos\"] = z\n", " g[\"shos_forecast\"] = f\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True).fillna(0)\n", " print(f\"[INFO] Added SHOS features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- Embeddings (Same as before) -----------------------------\n", "def build_low_rank_embeddings(df: pd.DataFrame, n_components: int = 16):\n", " print(f\"[INFO] Building low-rank embeddings (n_components={n_components})...\")\n", " occ = df.groupby([\"DealerCode\", \"DummyID\"]).agg(months_active=(\"PartQty\", lambda x: (x>0).sum())).reset_index()\n", " pivot = occ.pivot(index=\"DealerCode\", columns=\"DummyID\", values=\"months_active\").fillna(0)\n", " if pivot.shape[0] < 2 or pivot.shape[1] < 2:\n", " print(\"[INFO] Pivot too small for SVD, creating empty embeddings.\")\n", " dealer_embed = pd.DataFrame({\"DealerCode\": sorted(df[\"DealerCode\"].unique())})\n", " part_embed = pd.DataFrame({\"DummyID\": sorted(df[\"DummyID\"].unique())})\n", " return dealer_embed, part_embed, []\n", " k = min(n_components, min(pivot.shape)-1)\n", " svd = TruncatedSVD(n_components=k, random_state=0)\n", " U = svd.fit_transform(pivot)\n", " V = svd.components_.T\n", " dealer_index = list(pivot.index)\n", " part_index = list(pivot.columns)\n", " dealer_embed = pd.DataFrame(U, index=dealer_index).reset_index().rename(columns={\"index\":\"DealerCode\"})\n", " part_embed = pd.DataFrame(V, index=part_index).reset_index().rename(columns={\"index\":\"DummyID\"})\n", " emb_cols = []\n", " for i in range(U.shape[1]):\n", " dealer_embed.rename(columns={i: f\"emb_dealer_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_dealer_{i}\")\n", " for i in range(V.shape[1]):\n", " part_embed.rename(columns={i: f\"emb_part_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_part_{i}\")\n", " print(f\"[INFO] Created {len(emb_cols)} embedding features for {U.shape[0]} dealers and {V.shape[0]} parts.\")\n", " return dealer_embed, part_embed, emb_cols\n", "\n", "# ----------------------------- Feature Sets (NO CROSTON in ML Baselines, YES in Ablation) -----------------------------\n", "def build_base_features(df: pd.DataFrame, max_lag: int = 6):\n", " print(\"[INFO] Building base feature matrix...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " # Croston features are NOT added here for fairness in ML baselines\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " print(f\"[INFO] Base features include {len(base_cols)} columns.\")\n", " return df2, base_cols\n", "\n", "def build_fold_features(train_df: pd.DataFrame, test_df: pd.DataFrame, max_lag: int = 6, svd_components: int = 12, sparsity_k: float = 0.5):\n", " print(f\"[INFO] Building fold features with sparsity_k={sparsity_k}...\")\n", " train_base, base_cols = build_base_features(train_df, max_lag)\n", " test_base, _ = build_base_features(test_df, max_lag)\n", " # Add Croston features to both train and test folds BEFORE SHOS\n", " train_croston = add_croston_features(train_base)\n", " test_croston = add_croston_features(test_base)\n", " # Pass the Croston-augmented base to SHOS calculation\n", " n_train_map = train_df.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " train_shos = add_shos_features_with_ntrain(train_croston, n_train_map, sparsity_k=sparsity_k) # Pass train_croston\n", " test_shos = add_shos_features_with_ntrain(test_croston, n_train_map, sparsity_k=sparsity_k) # Pass test_croston\n", " # Calculate embeddings based on the SHOS-augmented training data\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(train_shos, svd_components)\n", " if emb_cols:\n", " train_shos = train_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " train_shos = train_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " test_shos = test_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " test_shos = test_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " train_shos[c] = train_shos[c].fillna(0)\n", " test_shos[c] = test_shos[c].fillna(0)\n", " # Define feature lists\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Fold features: {len(base_cols)} base, {len(emb_cols)} embeddings, {len(full_cols)} total.\")\n", " return train_shos, test_shos, base_cols, full_cols\n", "\n", "# ----------------------------- Intermittency Analysis (Same as before) -----------------------------\n", "def compute_intermittency(df_processed):\n", " print(\"[INFO] Computing intermittency metrics (ADI, CV²)...\")\n", " stats = []\n", " for (d, p), g in df_processed.groupby([\"DealerCode\", \"DummyID\"]):\n", " y = g[\"PartQty\"].values\n", " if np.sum(y > 0) == 0:\n", " continue\n", " intervals = np.diff(np.where(y > 0)[0])\n", " adi = np.mean(intervals) if len(intervals) > 0 else len(y)\n", " mean_y = np.mean(y)\n", " cv2 = (np.std(y) / mean_y) ** 2 if mean_y > 0 else np.inf\n", " stats.append({\"ADI\": adi, \"CV2\": cv2})\n", " stats_df = pd.DataFrame(stats)\n", " print(f\"[INFO] Computed metrics for {len(stats_df)} active series. Median ADI: {stats_df['ADI'].median():.2f}, Median CV²: {stats_df['CV2'].median():.2f}\")\n", " return stats_df\n", "\n", "# ----------------------------- Build Feature Matrix Once (Optimized for CV) -----------------------------\n", "def build_feature_matrix_optimized(df: pd.DataFrame, max_lag=6, svd_components=12, shos_k=0.5):\n", " print(\"[INFO] Building feature matrix ONCE for all folds...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " df2 = add_croston_features(df2)\n", " # Calculate n_train_map for the *entire* dataset (will be used later if needed per fold, but we calculate SHOS once here)\n", " # For this optimized version, we calculate SHOS once on the full series, using the provided k.\n", " # This means the sparsity used for alpha/beta is based on the *full* series length for each (d, p).\n", " # This is a trade-off for speed. For strict train-only sparsity per fold, calculation must happen inside the loop.\n", " # Using full-series sparsity is often a reasonable approximation.\n", " n_full_map = df2.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " df2 = add_shos_features_with_ntrain(df2, n_full_map, sparsity_k=shos_k) # Calculate SHOS once using full series length\n", " df2.fillna(0, inplace=True)\n", " # Define baseline features (excluding Croston/SHOS for fairness in base ML models)\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " # Calculate embeddings based on the full processed dataset (including SHOS features for context)\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(df2, n_components=svd_components)\n", " if emb_cols:\n", " df2 = df2.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " df2[c] = df2[c].fillna(0)\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Final feature matrix has {len(full_cols)} columns and shape {df2.shape}.\")\n", " return df2, base_cols, full_cols\n", "\n", "# ----------------------------- Train Hybrid Models (Statistical + ML/DL) -----------------------------\n", "def train_sba_lightgbm(X_train, y_train, X_test, y_test):\n", " \"\"\"Train SBA (Simple Moving Average) + LightGBM model.\"\"\"\n", " print(\"[INFO] Training SBA+LightGBM model...\")\n", " \n", " # Calculate SMA as a feature\n", " sma_train = pd.Series(y_train).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " sma_test = pd.Series(y_test).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " \n", " # Add SMA as a feature to X\n", " X_train_aug = np.column_stack([X_train, sma_train])\n", " X_test_aug = np.column_stack([X_test, sma_test])\n", " \n", " model = lgb.LGBMRegressor(n_estimators=100, learning_rate=0.05, num_leaves=31, verbose=-1)\n", " model.fit(X_train_aug, y_train)\n", " y_pred = model.predict(X_test_aug)\n", " return model, y_pred\n", "\n", "def train_croston_lightgbm(X_train, y_train, X_test, y_test):\n", " \"\"\"Train Croston + LightGBM model.\"\"\"\n", " print(\"[INFO] Training Croston+LightGBM model...\")\n", " \n", " # Calculate Croston as a feature\n", " croston_train = croston_forecast(y_train, alpha=0.3)\n", " croston_test = croston_forecast(y_test, alpha=0.3)\n", " \n", " # Add Croston as a feature to X\n", " X_train_aug = np.column_stack([X_train, croston_train])\n", " X_test_aug = np.column_stack([X_test, croston_test])\n", " \n", " model = lgb.LGBMRegressor(n_estimators=100, learning_rate=0.05, num_leaves=31, verbose=-1)\n", " model.fit(X_train_aug, y_train)\n", " y_pred = model.predict(X_test_aug)\n", " return model, y_pred\n", "\n", "def train_sba_xgboost(X_train, y_train, X_test, y_test):\n", " \"\"\"Train SBA + XGBoost model.\"\"\"\n", " print(\"[INFO] Training SBA+XGBoost model...\")\n", " \n", " # Calculate SMA as a feature\n", " sma_train = pd.Series(y_train).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " sma_test = pd.Series(y_test).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " \n", " # Add SMA as a feature to X\n", " X_train_aug = np.column_stack([X_train, sma_train])\n", " X_test_aug = np.column_stack([X_test, sma_test])\n", " \n", " import xgboost as xgb\n", " model = xgb.XGBRegressor(n_estimators=100, learning_rate=0.05, max_depth=6, verbose=0)\n", " model.fit(X_train_aug, y_train)\n", " y_pred = model.predict(X_test_aug)\n", " return model, y_pred\n", "\n", "def train_croston_xgboost(X_train, y_train, X_test, y_test):\n", " \"\"\"Train Croston + XGBoost model.\"\"\"\n", " print(\"[INFO] Training Croston+XGBoost model...\")\n", " \n", " # Calculate Croston as a feature\n", " croston_train = croston_forecast(y_train, alpha=0.3)\n", " croston_test = croston_forecast(y_test, alpha=0.3)\n", " \n", " # Add Croston as a feature to X\n", " X_train_aug = np.column_stack([X_train, croston_train])\n", " X_test_aug = np.column_stack([X_test, croston_test])\n", " \n", " import xgboost as xgb\n", " model = xgb.XGBRegressor(n_estimators=100, learning_rate=0.05, max_depth=6, verbose=0)\n", " model.fit(X_train_aug, y_train)\n", " y_pred = model.predict(X_test_aug)\n", " return model, y_pred\n", "\n", "def prepare_sequences(X, y, sequence_length=3):\n", " \"\"\"Convert tabular data into sequences for deep learning models.\"\"\"\n", " X_seq, y_seq = [], []\n", " for i in range(len(X) - sequence_length + 1):\n", " X_seq.append(X[i:i+sequence_length])\n", " y_seq.append(y[i+sequence_length-1])\n", " return np.array(X_seq), np.array(y_seq)\n", "\n", "def create_lstm_model(input_shape, n_outputs=1):\n", " model = Sequential([\n", " Input(shape=input_shape),\n", " LSTM(50, return_sequences=True),\n", " Dropout(0.2),\n", " LSTM(50, return_sequences=False),\n", " Dropout(0.2),\n", " Dense(25, activation='relu'),\n", " Dense(n_outputs)\n", " ])\n", " model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')\n", " return model\n", "\n", "def train_sba_lstm(X_train, y_train, X_test, y_test):\n", " \"\"\"Train SBA + LSTM model.\"\"\"\n", " print(\"[INFO] Training SBA+LSTM model...\")\n", " \n", " # Calculate SMA as a feature\n", " sma_train = pd.Series(y_train).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " sma_test = pd.Series(y_test).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " \n", " # Add SMA as a feature to X\n", " X_train_aug = np.column_stack([X_train, sma_train])\n", " X_test_aug = np.column_stack([X_test, sma_test])\n", " \n", " # Prepare sequences\n", " X_train_seq, y_train_seq = prepare_sequences(X_train_aug, y_train, 3)\n", " X_test_seq, y_test_seq = prepare_sequences(X_test_aug, y_test, 3)\n", " \n", " if len(X_train_seq) == 0 or len(X_test_seq) == 0:\n", " # If not enough data for sequences, return zeros\n", " return None, np.zeros(len(y_test))\n", " \n", " # Split train data for validation\n", " split_idx = int(0.8 * len(X_train_seq))\n", " X_train_split = X_train_seq[:split_idx]\n", " X_val_split = X_train_seq[split_idx:]\n", " y_train_split = y_train_seq[:split_idx]\n", " y_val_split = y_train_seq[split_idx:]\n", " \n", " model = create_lstm_model(input_shape=(3, X_train_aug.shape[1]))\n", " early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n", " \n", " model.fit(\n", " X_train_split, y_train_split,\n", " validation_data=(X_val_split, y_val_split),\n", " epochs=50,\n", " callbacks=[early_stop],\n", " verbose=0\n", " )\n", " \n", " y_pred = model.predict(X_test_seq, verbose=0).flatten()\n", " return model, y_pred\n", "\n", "def train_croston_lstm(X_train, y_train, X_test, y_test):\n", " \"\"\"Train Croston + LSTM model.\"\"\"\n", " print(\"[INFO] Training Croston+LSTM model...\")\n", " \n", " # Calculate Croston as a feature\n", " croston_train = croston_forecast(y_train, alpha=0.3)\n", " croston_test = croston_forecast(y_test, alpha=0.3)\n", " \n", " # Add Croston as a feature to X\n", " X_train_aug = np.column_stack([X_train, croston_train])\n", " X_test_aug = np.column_stack([X_test, croston_test])\n", " \n", " # Prepare sequences\n", " X_train_seq, y_train_seq = prepare_sequences(X_train_aug, y_train, 3)\n", " X_test_seq, y_test_seq = prepare_sequences(X_test_aug, y_test, 3)\n", " \n", " if len(X_train_seq) == 0 or len(X_test_seq) == 0:\n", " # If not enough data for sequences, return zeros\n", " return None, np.zeros(len(y_test))\n", " \n", " # Split train data for validation\n", " split_idx = int(0.8 * len(X_train_seq))\n", " X_train_split = X_train_seq[:split_idx]\n", " X_val_split = X_train_seq[split_idx:]\n", " y_train_split = y_train_seq[:split_idx]\n", " y_val_split = y_train_seq[split_idx:]\n", " \n", " model = create_lstm_model(input_shape=(3, X_train_aug.shape[1]))\n", " early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n", " \n", " model.fit(\n", " X_train_split, y_train_split,\n", " validation_data=(X_val_split, y_val_split),\n", " epochs=50,\n", " callbacks=[early_stop],\n", " verbose=0\n", " )\n", " \n", " y_pred = model.predict(X_test_seq, verbose=0).flatten()\n", " return model, y_pred\n", "\n", "# ----------------------------- Cross-validation with Hybrid Models -----------------------------\n", "def run_cross_validation_optimized(df_processed: pd.DataFrame, base_cols: List[str], full_cols: List[str], train_window=12, test_window=3, shos_k=0.1, shos_w=0.7):\n", " print(f\"[INFO] Starting optimized cross-validation with {train_window}-month train, {test_window}-month test windows...\")\n", " print(f\"[INFO] Using SHOS k={shos_k}, w={shos_w}\")\n", " df_processed = df_processed.sort_values([\"MonthStart\", \"DealerCode\", \"DummyID\"]).reset_index(drop=True)\n", " all_months = sorted(df_processed[\"MonthStart\"].unique())\n", " \n", " # Define models to evaluate (only the 6 specified hybrid models)\n", " model_names = [\n", " \"SBA-LightGBM\", \"Croston-LightGBM\", \n", " \"SBA-XGBoost\", \"Croston-XGBoost\", \n", " \"Croston-LSTM\", \"SBA-LSTM\"\n", " ]\n", " \n", " all_names = model_names + [\"Croston\", \"TSB\"]\n", " metrics = {name: {\"MAE\": [], \"RMSE\": [], \"WMAPE\": [], \"MAPE\": [], \"R2\": []} for name in all_names}\n", " fold_train_metrics = {name: {\"MAE\": [], \"RMSE\": [], \"WMAPE\": [], \"MAPE\": [], \"R2\": []} for name in all_names} # NEW: Store train metrics\n", " fold1_actual, fold1_preds = None, {}\n", " \n", " total_folds = len(all_months) - train_window - test_window + 1\n", " print(f\"[INFO] Total possible folds: {total_folds}\")\n", " \n", " for i in range(total_folds):\n", " train_months = all_months[i:i + train_window]\n", " test_months = all_months[i + train_window:i + train_window + test_window]\n", " \n", " # --- OPTIMIZATION: Use pre-computed features and slice them ---\n", " train_df_fold = df_processed[df_processed[\"MonthStart\"].isin(train_months)].copy()\n", " test_df_fold = df_processed[df_processed[\"MonthStart\"].isin(test_months)].copy()\n", " \n", " if train_df_fold.empty or test_df_fold.empty:\n", " print(f\"[INFO] Skipping fold {i+1} due to empty train/test set.\")\n", " continue\n", " \n", " print(f\"[INFO] Processing Fold {i+1}: Train {train_months[0].strftime('%Y-%m')} to {train_months[-1].strftime('%Y-%m')}, Test {test_months[0].strftime('%Y-%m')} to {test_months[-1].strftime('%Y-%m')}\")\n", " \n", " # Slice pre-computed features\n", " train_f = df_processed[df_processed[\"MonthStart\"].isin(train_months)].copy()\n", " test_f = df_processed[df_processed[\"MonthStart\"].isin(test_months)].copy()\n", " \n", " # Ensure features are aligned\n", " X_train_base = train_f[base_cols].fillna(0).values\n", " X_test_base = test_f[base_cols].fillna(0).values\n", " X_train_full = train_f[full_cols].fillna(0).values\n", " X_test_full = test_f[full_cols].fillna(0).values\n", " y_train_actual = train_f[\"PartQty\"].values\n", " y_test = test_f[\"PartQty\"].values\n", " \n", " # --- MODEL TRAINING AND PREDICTION ---\n", " model_predictions = {}\n", " model_test_scores = {}\n", " model_train_scores = {} # NEW: Store train scores for this fold\n", " \n", " # Baselines: Croston & TSB (predictions are based on training logic, so calculated per fold)\n", " def compute_croston_on_train_for_test(train_series, test_len):\n", " full_series = np.concatenate([train_series, np.zeros(test_len)])\n", " forecast = croston_forecast(full_series, alpha=0.3)\n", " return forecast[-test_len:]\n", " \n", " croston_preds_list = []\n", " for (d, p), test_g in test_df_fold.groupby([\"DealerCode\", \"DummyID\"]):\n", " train_g = train_df_fold[(train_df_fold[\"DealerCode\"]==d) & (train_df_fold[\"DummyID\"]==p)]\n", " train_y = train_g[\"PartQty\"].values if not train_g.empty else np.array([])\n", " test_len = len(test_g)\n", " if len(train_y) > 0:\n", " croston_for_test = compute_croston_on_train_for_test(train_y, test_len)\n", " else:\n", " croston_for_test = np.zeros(test_len)\n", " croston_preds_list.append(croston_for_test)\n", " croston_pred = np.concatenate(croston_preds_list)\n", " tsb_pred = test_f[\"shos_forecast\"].values # This is the pre-computed SHOS forecast for the test period\n", " model_predictions[\"Croston\"] = croston_pred\n", " model_predictions[\"TSB\"] = tsb_pred\n", " \n", " # Calculate TEST scores for baselines\n", " for name in [\"Croston\", \"TSB\"]:\n", " y_pred = model_predictions[name]\n", " try:\n", " test_mae = mean_absolute_error(y_test, y_pred)\n", " test_rmse = rmse(y_test, y_pred)\n", " test_wmape = wmape(y_test, y_pred)\n", " test_mape = mape(y_test, y_pred)\n", " test_r2 = r2_score(y_test, y_pred)\n", " model_test_scores[name] = {\n", " \"MAE\": test_mae,\n", " \"RMSE\": test_rmse,\n", " \"WMAPE\": test_wmape,\n", " \"MAPE\": test_mape,\n", " \"R2\": test_r2\n", " }\n", " # Baselines don't have a meaningful train prediction in this context for *these specific metrics*\n", " # So we assign NaN for train metrics for Croston/TSB.\n", " model_train_scores[name] = {k: np.nan for k in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]}\n", " except Exception as e:\n", " print(f\"[WARNING] Error calculating test metrics for {name}: {e}\")\n", " model_test_scores[name] = {k: np.nan for k in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]}\n", " model_train_scores[name] = {k: np.nan for k in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]}\n", " \n", " # --- HYBRID MODELS ---\n", " for name in model_names:\n", " try:\n", " if name == \"SBA-LightGBM\":\n", " model, y_pred_test = train_sba_lightgbm(X_train_base, y_train_actual, X_test_base, y_test)\n", " # For train predictions\n", " _, y_pred_train = train_sba_lightgbm(X_train_base, y_train_actual, X_train_base, y_train_actual)\n", " \n", " elif name == \"Croston-LightGBM\":\n", " model, y_pred_test = train_croston_lightgbm(X_train_base, y_train_actual, X_test_base, y_test)\n", " # For train predictions\n", " _, y_pred_train = train_croston_lightgbm(X_train_base, y_train_actual, X_train_base, y_train_actual)\n", " \n", " elif name == \"SBA-XGBoost\":\n", " model, y_pred_test = train_sba_xgboost(X_train_base, y_train_actual, X_test_base, y_test)\n", " # For train predictions\n", " _, y_pred_train = train_sba_xgboost(X_train_base, y_train_actual, X_train_base, y_train_actual)\n", " \n", " elif name == \"Croston-XGBoost\":\n", " model, y_pred_test = train_croston_xgboost(X_train_base, y_train_actual, X_test_base, y_test)\n", " # For train predictions\n", " _, y_pred_train = train_croston_xgboost(X_train_base, y_train_actual, X_train_base, y_train_actual)\n", " \n", " elif name == \"SBA-LSTM\":\n", " model, y_pred_test = train_sba_lstm(X_train_base, y_train_actual, X_test_base, y_test)\n", " # For train predictions\n", " _, y_pred_train = train_sba_lstm(X_train_base, y_train_actual, X_train_base, y_train_actual)\n", " \n", " elif name == \"Croston-LSTM\":\n", " model, y_pred_test = train_croston_lstm(X_train_base, y_train_actual, X_test_base, y_test)\n", " # For train predictions\n", " _, y_pred_train = train_croston_lstm(X_train_base, y_train_actual, X_train_base, y_train_actual)\n", " \n", " # Adjust predictions to match test length\n", " if len(y_pred_test) > len(y_test):\n", " y_pred_test = y_pred_test[:len(y_test)]\n", " elif len(y_pred_test) < len(y_test):\n", " # Pad with last value\n", " padding = np.full(len(y_test) - len(y_pred_test), y_pred_test[-1] if len(y_pred_test) > 0 else 0)\n", " y_pred_test = np.concatenate([y_pred_test, padding])\n", " \n", " if len(y_pred_train) > len(y_train_actual):\n", " y_pred_train = y_pred_train[:len(y_train_actual)]\n", " elif len(y_pred_train) < len(y_train_actual):\n", " # Pad with last value\n", " padding = np.full(len(y_train_actual) - len(y_pred_train), y_pred_train[-1] if len(y_pred_train) > 0 else 0)\n", " y_pred_train = np.concatenate([y_pred_train, padding])\n", " \n", " # Calculate TEST Metrics\n", " test_mae = mean_absolute_error(y_test, y_pred_test)\n", " test_rmse = rmse(y_test, y_pred_test)\n", " test_wmape = wmape(y_test, y_pred_test)\n", " test_mape = mape(y_test, y_pred_test)\n", " test_r2 = r2_score(y_test, y_pred_test)\n", " \n", " # Calculate TRAIN Metrics\n", " train_mae = mean_absolute_error(y_train_actual, y_pred_train)\n", " train_rmse = rmse(y_train_actual, y_pred_train)\n", " train_wmape = wmape(y_train_actual, y_pred_train)\n", " train_mape = mape(y_train_actual, y_pred_train)\n", " train_r2 = r2_score(y_train_actual, y_pred_train)\n", " \n", " model_predictions[name] = y_pred_test\n", " model_test_scores[name] = {\n", " \"MAE\": test_mae,\n", " \"RMSE\": test_rmse,\n", " \"WMAPE\": test_wmape,\n", " \"MAPE\": test_mape,\n", " \"R2\": test_r2\n", " }\n", " model_train_scores[name] = { # Store train scores\n", " \"MAE\": train_mae,\n", " \"RMSE\": train_rmse,\n", " \"WMAPE\": train_wmape,\n", " \"MAPE\": train_mape,\n", " \"R2\": train_r2\n", " }\n", " except Exception as e:\n", " print(f\"[WARNING] Error training/predicting {name}: {e}\")\n", " model_predictions[name] = np.zeros_like(y_test)\n", " for metric_name in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]:\n", " model_test_scores[name][metric_name] = np.nan\n", " model_train_scores[name][metric_name] = np.nan # Store NaN for train too\n", " \n", " # --- STORE METRICS FOR THIS FOLD ---\n", " for name in all_names:\n", " for metric_name in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]:\n", " # Append the scores calculated for this fold\n", " metrics[name][metric_name].append(model_test_scores[name][metric_name])\n", " fold_train_metrics[name][metric_name].append(model_train_scores[name][metric_name]) # Append train scores\n", " \n", " # Store first fold results for plotting\n", " if fold1_actual is None:\n", " fold1_actual = y_test\n", " fold1_preds = {\n", " \"Croston\": croston_pred,\n", " \"TSB\": tsb_pred,\n", " \"SBA-LightGBM\": model_predictions.get(\"SBA-LightGBM\", np.zeros_like(y_test)),\n", " \"Croston-LightGBM\": model_predictions.get(\"Croston-LightGBM\", np.zeros_like(y_test))\n", " }\n", " \n", " valid_folds = 0\n", " # find a model name and metric list to count non-nan entries (safe method)\n", " any_model = next(iter(metrics))\n", " valid_folds = int(np.nanmax([len([x for x in metrics[m]['MAE'] if not (x is None or (isinstance(x, float) and np.isnan(x)))]) for m in metrics]) if metrics else 0)\n", " print(f\"[INFO] Cross-validation completed. Estimated valid folds per model (approx): {valid_folds}\")\n", " \n", " results = {}\n", " train_results = {} # NEW: Store aggregated train results\n", " for name in all_names: # Iterate over all model names\n", " results[name] = {}\n", " train_results[name] = {} # NEW: Initialize train results dict\n", " for metric in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]: # Iterate over metric names\n", " test_arr = np.array(metrics[name][metric])\n", " train_arr = np.array(fold_train_metrics[name][metric]) # NEW: Get train array\n", " results[name][f\"{metric}_mean\"] = np.nanmean(test_arr)\n", " results[name][f\"{metric}_std\"] = np.nanstd(test_arr)\n", " train_results[name][f\"{metric}_mean\"] = np.nanmean(train_arr) # NEW: Calculate train mean\n", " train_results[name][f\"{metric}_std\"] = np.nanstd(train_arr) # NEW: Calculate train std\n", " \n", " # --- OVERFITTING DIAGNOSIS ---\n", " # --- AGGREGATE RESULTS INTO DATAFRAMES (safe conversion) ---\n", " results_df = pd.DataFrame(results).T if isinstance(results, dict) else pd.DataFrame(results)\n", " train_df = pd.DataFrame(train_results).T if isinstance(train_results, dict) else pd.DataFrame(train_results)\n", " # Ensure columns follow the \"_mean\" pattern\n", " required_cols = [\"MAE_mean\",\"MAE_std\",\"RMSE_mean\",\"RMSE_std\",\"WMAPE_mean\",\"WMAPE_std\",\"MAPE_mean\",\"MAPE_std\",\"R2_mean\",\"R2_std\"]\n", " # If some are missing, fill with NaN\n", " for c in required_cols:\n", " if c not in results_df.columns:\n", " results_df[c] = np.nan\n", " if c not in train_df.columns:\n", " train_df[c] = np.nan\n", " \n", " # Overfitting diagnostics: Test - Train\n", " overfitting_df = pd.DataFrame(index=results_df.index)\n", " overfitting_df[\"Test_MAE\"] = results_df[\"MAE_mean\"]\n", " overfitting_df[\"Train_MAE\"] = train_df[\"MAE_mean\"]\n", " overfitting_df[\"Diff_MAE\"] = overfitting_df[\"Test_MAE\"] - overfitting_df[\"Train_MAE\"]\n", " overfitting_df[\"Test_R2\"] = results_df[\"R2_mean\"]\n", " overfitting_df[\"Train_R2\"] = train_df[\"R2_mean\"]\n", " overfitting_df[\"Diff_R2\"] = overfitting_df[\"Test_R2\"] - overfitting_df[\"Train_R2\"]\n", " \n", " # Print\n", " print(\"--- OVERFITTING DIAGNOSIS ---\")\n", " print(overfitting_df.round(6))\n", " print(\"Interpretation: Positive Diff_MAE or negative Diff_R2 indicates possible overfitting (Test worse than Train).\")\n", " \n", " return results_df.to_dict(orient=\"index\"), train_df.to_dict(orient=\"index\"), overfitting_df, fold1_actual, fold1_preds\n", "\n", "# ----------------------------- SHAP Analysis (Point 4) -----------------------------\n", "def run_shap_analysis(model, X_test, feature_names, outdir):\n", " if not SHAP_AVAILABLE:\n", " print(\"[WARNING] SHAP not available, skipping SHAP analysis.\")\n", " return\n", " if model is None:\n", " print(\"[WARNING] Model object is None, skipping SHAP analysis.\")\n", " return\n", " print(f\"[INFO] Calculating SHAP values for model...\")\n", " try:\n", " explainer = shap.TreeExplainer(model)\n", " # Use a sample for speed if X_test is large\n", " sample_size = min(1000, X_test.shape[0])\n", " X_test_sample = X_test[:sample_size]\n", " shap_values = explainer.shap_values(X_test_sample)\n", " shap.summary_plot(shap_values, X_test_sample, feature_names=feature_names, show=False, plot_size=(10, 8))\n", " plt.title(\"SHAP Feature Importance (Top 10)\")\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"figures\", \"fig6_shap.png\"))\n", " plt.close()\n", " print(f\"[INFO] SHAP plot saved to {os.path.join(outdir, 'figures', 'fig6_shap.png')}\")\n", " except Exception as e:\n", " print(f\"[WARNING] Could not generate SHAP plot: {e}\")\n", "\n", "# ----------------------------- Main Pipeline (Test Run) -----------------------------\n", "def run_test_pipeline(df_filtered, outdir=\"./result_test\", shos_k=0.1, shos_w=0.7):\n", " print(\"[INFO] Starting test pipeline...\")\n", " start_time = time.time()\n", " safe_mkdir(outdir)\n", " safe_mkdir(os.path.join(outdir, \"figures\")) # Ensure figures dir exists\n", " print(\"[INFO] 1. Aggregating and padding data...\")\n", " df_ag = aggregate_monthly(df_filtered)\n", " df_padded = pad_monthly_series_and_filter_inactive(df_ag)\n", " print(\"[INFO] 2. Computing intermittency metrics...\")\n", " intermittency_df = compute_intermittency(df_padded)\n", " intermittency_df.to_csv(os.path.join(outdir, \"intermittency_stats.csv\"), index=False)\n", " print(\"[INFO] 3. Building feature matrix once...\")\n", " df_features, base_cols, full_cols = build_feature_matrix_optimized(df_padded, shos_k=shos_k) # Pass shos_k for SHOS calc\n", " print(\"[INFO] 4. Running main cross-validation with optimized features...\")\n", " cv_results, train_results, overfitting_df, fold1_actual, fold1_preds = run_cross_validation_optimized(\n", " df_features, base_cols, full_cols, shos_k=shos_k, shos_w=shos_w # Pass pre-computed features and k, w\n", " )\n", " print(\"[INFO] 5. Saving results...\")\n", " test_scores_df = pd.DataFrame(cv_results).T\n", " test_scores_df.to_csv(os.path.join(outdir, \"test_results.csv\"))\n", " train_scores_df = pd.DataFrame(train_results).T # NEW: Save train results\n", " train_scores_df.to_csv(os.path.join(outdir, \"train_results.csv\"))\n", " overfitting_df.to_csv(os.path.join(outdir, \"overfitting_diagnosis.csv\")) # Save overfitting results\n", " print(\" --- TEST RESULTS (Mean ± Std) --- \")\n", " print(test_scores_df.round(4))\n", " print(\" --- TRAIN RESULTS (Mean ± Std) --- \") # NEW: Print train results\n", " print(train_scores_df.round(4))\n", " print(\" --- OVERFITTING DIAGNOSIS (Test - Train) --- \")\n", " print(overfitting_df.round(4))\n", " print(\"[INFO] 6. Generating figures...\")\n", " out_fig = os.path.join(outdir, \"figures\")\n", " safe_mkdir(out_fig)\n", " # Filter models for plotting (exclude TSB/Croston if too many, include hybrid models)\n", " plot_models = [m for m in test_scores_df.index if m not in [\"TSB\", \"Croston\"]] # Or include them if desired\n", " test_mae_means = [test_scores_df.loc[m, \"MAE_mean\"] for m in plot_models]\n", " test_mae_stds = [test_scores_df.loc[m, \"MAE_std\"] for m in plot_models]\n", " test_rmse_means = [test_scores_df.loc[m, \"RMSE_mean\"] for m in plot_models]\n", " test_rmse_stds = [test_scores_df.loc[m, \"RMSE_std\"] for m in plot_models]\n", " test_wmape_means = [test_scores_df.loc[m, \"WMAPE_mean\"] for m in plot_models]\n", " test_wmape_stds = [test_scores_df.loc[m, \"WMAPE_std\"] for m in plot_models]\n", " test_mape_means = [test_scores_df.loc[m, \"MAPE_mean\"] for m in plot_models]\n", " test_mape_stds = [test_scores_df.loc[m, \"MAPE_std\"] for m in plot_models]\n", " test_r2_means = [test_scores_df.loc[m, \"R2_mean\"] for m in plot_models]\n", " test_r2_stds = [test_scores_df.loc[m, \"R2_std\"] for m in plot_models]\n", " # NEW: Get train metrics for plotting\n", " train_mae_means = [train_scores_df.loc[m, \"MAE_mean\"] for m in plot_models]\n", " train_mae_stds = [train_scores_df.loc[m, \"MAE_std\"] for m in plot_models]\n", " train_r2_means = [train_scores_df.loc[m, \"R2_mean\"] for m in plot_models]\n", " train_r2_stds = [train_scores_df.loc[m, \"R2_std\"] for m in plot_models]\n", " x = np.arange(len(plot_models))\n", " width = 0.35\n", " # Figure 1: MAE (Train vs Test)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, test_mae_means, width, yerr=test_mae_stds, label='Test MAE')\n", " ax.bar(x + width/2, train_mae_means, width, yerr=train_mae_stds, label='Train MAE') # NEW: Add train bars\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"MAE\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig1_mae_train_test.png\"))\n", " # Figure 2: R2 (Train vs Test)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, test_r2_means, width, yerr=test_r2_stds, label='Test R2')\n", " ax.bar(x + width/2, train_r2_means, width, yerr=train_r2_stds, label='Train R2') # NEW: Add train bars\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"R2 Score\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig2_r2_train_test.png\"))\n", " # Figure 3: WMAPE/MAPE (Test only, as before)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, test_wmape_means, width, yerr=test_wmape_stds, label='Test WMAPE')\n", " ax.bar(x + width/2, test_mape_means, width, yerr=test_mape_stds, label='Test MAPE')\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"Percentage Error\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig3_wmape_mape_test.png\"))\n", " # Figure 5: Overfitting Bar Chart (Difference)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, overfitting_df['Diff_MAE'], width, label='Test MAE - Train MAE', alpha=0.8)\n", " ax.axhline(0, color='black', linewidth=0.8) # Add a horizontal line at 0\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"Difference in Metric (Test - Train)\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig5_overfitting_diff_mae.png\"))\n", " # Figure 6: Overfitting R2 Chart (Difference)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, overfitting_df['Diff_R2'], width, label='Test R2 - Train R2', alpha=0.8, color='orange')\n", " ax.axhline(0, color='black', linewidth=0.8) # Add a horizontal line at 0\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"Difference in R2 (Test - Train)\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig6_overfitting_diff_r2.png\"))\n", " # Intermittency Histograms\n", " fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n", " axes[0].hist(intermittency_df[\"ADI\"], bins=50, color=\"skyblue\")\n", " axes[0].set_title(f\"ADI (median={intermittency_df['ADI'].median():.2f})\")\n", " axes[1].hist(intermittency_df[\"CV2\"].clip(0, 10), bins=50, color=\"salmon\")\n", " axes[1].set_title(f\"CV² (median={intermittency_df['CV2'].median():.2f})\")\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig7_intermittency.png\"))\n", " print(f\"[INFO] Pipeline completed in {time.time() - start_time:.2f} seconds. Results saved to: {outdir}\")\n", " # Print summary\n", " print(\"--- SUMMARY ---\")\n", " print(\"Test Results (Mean ± Std):\")\n", " print(test_scores_df[[\"MAE_mean\", \"RMSE_mean\", \"WMAPE_mean\", \"MAPE_mean\", \"R2_mean\"]].round(4))\n", " best_test_mae_model = test_scores_df['MAE_mean'].idxmin()\n", " best_test_r2_model = test_scores_df['R2_mean'].idxmax()\n", " print(f\"Best Test MAE: {test_scores_df.loc[best_test_mae_model, 'MAE_mean']:.4f} ({best_test_mae_model})\")\n", " print(f\"Best Test R2: {test_scores_df.loc[best_test_r2_model, 'R2_mean']:.4f} ({best_test_r2_model})\")\n", " print(\"Train Results (Mean ± Std):\") # NEW: Print train summary\n", " print(train_scores_df[[\"MAE_mean\", \"RMSE_mean\", \"WMAPE_mean\", \"MAPE_mean\", \"R2_mean\"]].round(4))\n", " print(\"Overfitting Diagnosis (Mean Difference: Test - Train):\") # Print overfitting summary\n", " print(overfitting_df[[\"Diff_MAE\", \"Diff_R2\"]].round(4))\n", " print(f\"Median ADI: {intermittency_df['ADI'].median():.2f}\")\n", " print(f\"Median CV²: {intermittency_df['CV2'].median():.2f}\")\n", " return cv_results, train_results # Return both test and train results\n", "\n", "# ----------------------------- Entry Point -----------------------------\n", "if __name__ == \"__main__\":\n", " # Example usage:\n", " # df_filtered = pd.read_csv(\"your_data.csv\")\n", " run_test_pipeline(df_filtered, outdir=\"./result_test1\", shos_k=0.1, shos_w=0.7) # Provide your best k, w\n", " pass" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: lightgbm in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (4.6.0)\n", "Requirement already satisfied: numpy>=1.17.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from lightgbm) (2.2.6)\n", "Requirement already satisfied: scipy in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from lightgbm) (1.15.3)\n", "Note: you may need to restart the kernel to use updated packages.\n", "✅ Keras version 3.12.0 is active and working!\n", "[INFO] TensorFlow is available.\n", "[INFO] SHAP is available.\n", "[INFO] Starting stress test pipeline...\n", "[INFO] 1. Aggregating and padding data...\n", "[INFO] Aggregating data to monthly level...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING: You are using pip version 21.2.3; however, version 25.3 is available.\n", "You should consider upgrading via the 'c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\python.exe -m pip install --upgrade pip' command.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[INFO] Aggregated to 171111 monthly records.\n", "[INFO] Padding series to monthly grid and filtering inactive pairs...\n", "[INFO] Date range: 2023-08 to 2025-08. Total months: 25\n", "[INFO] Filtered from 171111 to 170827 records after removing inactive pairs.\n", "[INFO] Padding complete. Final shape: (1404700, 9)\n", "[INFO] 2. Computing intermittency metrics...\n", "[INFO] Computing intermittency metrics (ADI, CV²)...\n", "[INFO] Computed metrics for 56188 active series. Median ADI: 25.00, Median CV²: 24.00\n", "[INFO] 3. Building feature matrix once...\n", "[INFO] Building feature matrix ONCE for all folds...\n", "[INFO] Adding time series features (max_lag=6)...\n", "[INFO] Added 12 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 56188 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.1)...\n", "[INFO] Added SHOS features for 56188 unique series.\n", "[INFO] Building low-rank embeddings (n_components=12)...\n", "[INFO] Created 24 embedding features for 59 dealers and 7702 parts.\n", "[INFO] Final feature matrix has 50 columns and shape (1404700, 59).\n", "[INFO] 4. Calculating sparsity for each SKU...\n", "[INFO] Calculating sparsity for each SKU...\n", "[INFO] 5. Splitting SKUs into sparsity bins...\n", "[INFO] Splitting SKUs into sparsity bins...\n", " Bin 0.9+: 39929 SKUs\n", " Bin 0.7-0.8: 2139 SKUs\n", " Bin 0.8-0.9: 8856 SKUs\n", "[INFO] 6. Running stress test for each bin...\n", "[INFO] Running stress test for bin: 0.9+\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "WARNING:tensorflow:5 out of the last 5 calls to .one_step_on_data_distributed at 0x00000270126A8430> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n", "WARNING:tensorflow:6 out of the last 6 calls to .one_step_on_data_distributed at 0x0000026F9E972E60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n", "[INFO] Adding time series features (max_lag=12)...\n", "[INFO] Added 24 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 1 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.5)...\n", "[INFO] Added SHOS features for 1 unique series.\n" ] } ], "source": [ "%pip install lightgbm\n", "import os\n", "import math\n", "import pickle\n", "import warnings\n", "import time\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.decomposition import TruncatedSVD\n", "from sklearn.calibration import CalibratedClassifierCV\n", "from sklearn.linear_model import ElasticNet, Ridge, LogisticRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.model_selection import TimeSeriesSplit\n", "from sklearn.metrics import(\n", " mean_squared_error, mean_absolute_error, r2_score\n", ")\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from scipy.fft import fft, fftfreq\n", "from scipy.signal import correlate\n", "from scipy.stats import pearsonr\n", "from sklearn.metrics import mean_absolute_error\n", "from sklearn.preprocessing import StandardScaler\n", "import lightgbm as lgb\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.preprocessing import MinMaxScaler\n", "import tensorflow as tf\n", "import os\n", "import math\n", "import pickle\n", "import warnings\n", "import time\n", "from typing import Optional, Dict, Any, Tuple, List\n", "warnings.filterwarnings(\"ignore\")\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.decomposition import TruncatedSVD\n", "from sklearn.calibration import CalibratedClassifierCV\n", "from sklearn.linear_model import ElasticNet, Ridge, LogisticRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.model_selection import TimeSeriesSplit\n", "from sklearn.metrics import (\n", " mean_squared_error, mean_absolute_error, r2_score\n", ")\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "# Deep Learning Libraries\n", "try:\n", " import tensorflow as tf\n", " # Instead of: from tensorflow.keras.models import Sequential\n", "# Use this:\n", " import keras\n", " from keras.models import Sequential\n", " from keras.layers import LSTM, Dense, Dropout, Input, Conv1D, MaxPooling1D, Flatten, GlobalAveragePooling1D\n", " from keras.layers import MultiHeadAttention, LayerNormalization, Add\n", " from keras.optimizers import Adam\n", " from keras.callbacks import EarlyStopping\n", "\n", " print(f\"✅ Keras version {keras.__version__} is active and working!\")\n", "\n", " DL_AVAILABLE = True\n", " print(\"[INFO] TensorFlow is available.\")\n", "except ImportError:\n", " DL_AVAILABLE = False\n", " print(\"[INFO] TensorFlow is NOT available.\")\n", "\n", "# Optional SHAP library\n", "try:\n", " import shap\n", " SHAP_AVAILABLE = True\n", " print(\"[INFO] SHAP is available.\")\n", "except ImportError:\n", " SHAP_AVAILABLE = False\n", " print(\"[INFO] SHAP is NOT available.\")\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "# ----------------------------- Utilities -----------------------------\n", "def safe_mkdir(path: str):\n", " os.makedirs(path, exist_ok=True)\n", "\n", "def rmse(y_true, y_pred):\n", " return math.sqrt(mean_squared_error(y_true, y_pred))\n", "\n", "def wmape(y_true, y_pred):\n", " denom = np.sum(np.abs(y_true))\n", " return np.sum(np.abs(y_true - y_pred)) / denom * 100.0 if denom != 0 else np.nan\n", "\n", "def mape(y_true, y_pred):\n", " mask = y_true != 0\n", " return np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask])) * 100.0 if mask.sum() > 0 else np.nan\n", "\n", "# ----------------------------- Padding with Inactive Filtering (Moved to Top) -----------------------------\n", "def pad_monthly_series_and_filter_inactive(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Padding series to monthly grid and filtering inactive pairs...\")\n", " df = df.copy()\n", " df[\"MonthStart\"] = pd.to_datetime(df[\"MonthStart\"])\n", " min_date = df[\"MonthStart\"].min()\n", " max_date = df[\"MonthStart\"].max()\n", " all_months = pd.date_range(start=min_date, end=max_date, freq=\"MS\")\n", " print(f\"[INFO] Date range: {min_date.strftime('%Y-%m')} to {max_date.strftime('%Y-%m')}. Total months: {len(all_months)}\")\n", " df_active = df.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0)\n", " print(f\"[INFO] Filtered from {df.shape[0]} to {df_active.shape[0]} records after removing inactive pairs.\")\n", " if df_active.empty:\n", " print(\"[WARNING] No active series found after filtering.\")\n", " return df_active\n", " full_index = pd.MultiIndex.from_product(\n", " [df_active[\"DealerCode\"].unique(), df_active[\"DummyID\"].unique(), all_months],\n", " names=[\"DealerCode\", \"DummyID\", \"MonthStart\"]\n", " )\n", " df_indexed = df_active.set_index([\"DealerCode\", \"DummyID\", \"MonthStart\"])\n", " df_padded = df_indexed.reindex(full_index, fill_value=0).reset_index()\n", " df_padded[\"OTC_Flag\"] = (df_padded[\"OTC_Qty\"] > 0).astype(int)\n", " for col in [\"PartQty\", \"OTC_Qty\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]:\n", " df_padded[col] = pd.to_numeric(df_padded[col], errors=\"coerce\").fillna(0)\n", " df_final = df_padded.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0).reset_index(drop=True)\n", " print(f\"[INFO] Padding complete. Final shape: {df_final.shape}\")\n", " return df_final\n", "\n", "# ----------------------------- Data helpers (Same as before) -----------------------------\n", "def aggregate_monthly(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Aggregating data to monthly level...\")\n", " df = df.copy()\n", " colmap = {}\n", " for c in df.columns:\n", " lc = c.lower().strip()\n", " if lc in (\"dealer code\", \"dealercode\", \"dealer_code\", \"dealercode\"):\n", " colmap[c] = \"DealerCode\"\n", " if lc in (\"dummy id\", \"dummyid\", \"partno\", \"part_no\", \"dummy_id\"):\n", " colmap[c] = \"DummyID\"\n", " if lc in (\"sale date\", \"saledate\", \"sale_date\"):\n", " colmap[c] = \"SaleDate\"\n", " if lc in (\"part qty.\", \"part qty\", \"partqty\", \"qty\", \"quantity\"):\n", " colmap[c] = \"PartQty\"\n", " if lc in (\"sale type\", \"saletype\", \"sale_type\"):\n", " colmap[c] = \"SaleType\"\n", " if lc in (\"dealer price\", \"dealerprice\", \"price\"):\n", " colmap[c] = \"DealerPrice\"\n", " if lc in (\"prc_unit_weight\", \"weight\", \"prc unit weight\"):\n", " colmap[c] = \"PRC_UNIT_WEIGHT\"\n", " if lc in (\"prc_unit_cube\", \"cube\", \"prc unit cube\"):\n", " colmap[c] = \"PRC_UNIT_CUBE\"\n", " if lc in (\"otc_qty\",\"otc qty\",\"otcqty\"):\n", " colmap[c] = \"OTC_Qty\"\n", " df = df.rename(columns=colmap)\n", " if \"MonthStart\" in df.columns and \"PartQty\" in df.columns:\n", " out = df.copy()\n", " out[\"MonthStart\"] = pd.to_datetime(out[\"MonthStart\"])\n", " if \"OTC_Qty\" not in out.columns:\n", " if \"SaleType\" in out.columns:\n", " out[\"OTC_Qty\"] = out[\"PartQty\"] * 0\n", " mask = out[\"SaleType\"].astype(str).str.upper() == \"OTC\"\n", " out.loc[mask, \"OTC_Qty\"] = out.loc[mask, \"PartQty\"]\n", " else:\n", " out[\"OTC_Qty\"] = 0\n", " out[\"OTC_Flag\"] = (out[\"OTC_Qty\"] > 0).astype(int)\n", " out[\"DealerPrice\"] = out.get(\"DealerPrice\", 0.0)\n", " out[\"PRC_UNIT_WEIGHT\"] = out.get(\"PRC_UNIT_WEIGHT\", 0.0)\n", " out[\"PRC_UNIT_CUBE\"] = out.get(\"PRC_UNIT_CUBE\", 0.0)\n", " return out[[\"DealerCode\", \"DummyID\", \"MonthStart\", \"PartQty\", \"OTC_Qty\", \"OTC_Flag\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]]\n", " if \"SaleDate\" not in df.columns:\n", " raise ValueError(\"Input must have 'SaleDate' or 'MonthStart' + 'PartQty'.\")\n", " df[\"SaleDate\"] = pd.to_datetime(df[\"SaleDate\"])\n", " df[\"MonthStart\"] = df[\"SaleDate\"].dt.to_period(\"M\").dt.to_timestamp()\n", " agg = df.groupby([\"DealerCode\", \"DummyID\", \"MonthStart\"], as_index=False).agg(\n", " PartQty=(\"PartQty\", \"sum\"),\n", " OTC_Qty=(\"PartQty\", lambda x: x[df.loc[x.index, \"SaleType\"].astype(str).str.upper() == \"OTC\"].sum() if \"SaleType\" in df.columns else 0),\n", " DealerPrice=(\"DealerPrice\", \"mean\"),\n", " PRC_UNIT_WEIGHT=(\"PRC_UNIT_WEIGHT\", \"mean\"),\n", " PRC_UNIT_CUBE=(\"PRC_UNIT_CUBE\", \"mean\"),\n", " )\n", " agg[\"OTC_Qty\"] = agg[\"OTC_Qty\"].fillna(0).astype(int)\n", " agg[\"OTC_Flag\"] = (agg[\"OTC_Qty\"] > 0).astype(int)\n", " agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]] = agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]].fillna(0.0)\n", " print(f\"[INFO] Aggregated to {len(agg)} monthly records.\")\n", " return agg\n", "\n", "def add_time_series_features(df: pd.DataFrame, max_lag: int = 12) -> pd.DataFrame:\n", " print(f\"[INFO] Adding time series features (max_lag={max_lag})...\")\n", " df = df.sort_values([\"DealerCode\", \"DummyID\", \"MonthStart\"]).copy()\n", " rows = []\n", " grp = df.groupby([\"DealerCode\", \"DummyID\"])\n", " for (d, p), g in grp:\n", " g = g.sort_values(\"MonthStart\").copy()\n", " for lag in range(1, max_lag+1):\n", " g[f\"qty_lag_{lag}\"] = g[\"PartQty\"].shift(lag).fillna(0)\n", " g[f\"otc_lag_{lag}\"] = g[\"OTC_Flag\"].shift(lag).fillna(0)\n", " g[\"qty_sum_1_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"qty_sum_1_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"nonzero_count_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).apply(lambda x: np.sum(x>0)).shift(1).fillna(0)\n", " last_pos = -1\n", " t_since = []\n", " for idx, val in enumerate(g[\"PartQty\"].values):\n", " if val > 0:\n", " last_pos = idx\n", " t_since.append(0)\n", " else:\n", " t_since.append(idx - last_pos if last_pos != -1 else g.shape[0]+1)\n", " g[\"time_since_last_sale\"] = pd.Series(t_since).astype(float)\n", " g[\"qty_ma_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"qty_ma_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"otc_share_slope_3\"] = g[\"OTC_Flag\"].rolling(3, min_periods=1).apply(lambda x: np.polyfit(np.arange(len(x)), x, 1)[0] if len(x)>1 else 0).shift(1).fillna(0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added {len([c for c in out.columns if c.startswith(('qty_lag_', 'otc_lag_'))])} lag features.\")\n", " return out\n", "\n", "def croston_forecast(series: np.ndarray, alpha: float = 0.3) -> np.ndarray:\n", " n = len(series)\n", " if n == 0:\n", " return np.zeros(0)\n", " demand = np.asarray(series, dtype=float)\n", " forecast = np.zeros(n)\n", " nz_idx = np.where(demand > 0)[0]\n", " if len(nz_idx) == 0:\n", " return forecast\n", " last = nz_idx[0]\n", " z = demand[last]\n", " p = 1.0\n", " for t in range(last+1, n):\n", " if demand[t] > 0:\n", " interval = t - last\n", " z = alpha * demand[t] + (1-alpha) * z\n", " p = alpha * interval + (1-alpha) * p\n", " last = t\n", " forecast[t] = z / max(1e-9, p)\n", " forecast[:last+1] = forecast[last+1] if last+1 < n else forecast[last]\n", " return forecast\n", "\n", "def add_croston_features(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Adding Croston features...\")\n", " rows = []\n", " for (d,p), g in df.groupby([\"DealerCode\",\"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " g[\"croston_forecast\"] = croston_forecast(g[\"PartQty\"].values, alpha=0.3)\n", " g[\"croston_ratio\"] = np.where(g[\"qty_sum_1_3\"]!=0, g[\"croston_forecast\"]/g[\"qty_sum_1_3\"], 0.0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added Croston features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- SHOS with Train-Only Sparsity (Same as before) -----------------------------\n", "def compute_shos_for_series(y: np.ndarray, alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5,\n", " n_train: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n", " n = len(y)\n", " if n == 0:\n", " return np.zeros(0), np.zeros(0), np.zeros(0)\n", " if n_train is not None and n_train > 0:\n", " y_sparsity = y[:n_train]\n", " else:\n", " y_sparsity = y\n", " nonzero_count = np.sum(y_sparsity > 0)\n", " sparsity = 1.0 - (nonzero_count / max(1, len(y_sparsity)))\n", " if adaptivity:\n", " alpha = alpha_base * (1.0 - sparsity_k * sparsity)\n", " beta = beta_base * (1.0 - sparsity_k * sparsity)\n", " last_pos_all = np.where(y > 0)[0]\n", " if len(last_pos_all) > 0 and (n - 1 - last_pos_all[-1]) <= 3:\n", " alpha = min(0.9, alpha * 1.2)\n", " beta = min(0.9, beta * 1.2)\n", " else:\n", " alpha, beta = alpha_base, beta_base\n", " # --- IMPROVED INITIALIZATION (Point 6) ---\n", " if n >= 3:\n", " init_window = 3\n", " y_init_window = y[:init_window]\n", " q0 = float(np.sum(y_init_window > 0)) / max(1, init_window)\n", " positive_y_init = y_init_window[y_init_window > 0]\n", " if len(positive_y_init) > 0:\n", " z0 = float(np.mean(positive_y_init))\n", " else:\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " else:\n", " q0 = float(np.sum(y > 0)) / max(1, n)\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " # --- END IMPROVED INITIALIZATION ---\n", " q_prev, z_prev = q0, z0\n", " q, z, f = np.zeros(n), np.zeros(n), np.zeros(n)\n", " for t in range(n):\n", " yt = y[t]\n", " if yt > 0:\n", " z_curr = alpha * yt + (1 - alpha) * z_prev\n", " q_curr = beta * 1.0 + (1 - beta) * q_prev\n", " else:\n", " z_curr = z_prev\n", " q_curr = beta * 0.0 + (1 - beta) * q_prev\n", " q[t], z[t], f[t] = q_curr, z_curr, q_curr * z_curr\n", " q_prev, z_prev = q_curr, z_curr\n", " return q, z, f\n", "\n", "def add_shos_features_with_ntrain(df: pd.DataFrame, n_train_per_series: Dict[Tuple, int],\n", " alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5) -> pd.DataFrame:\n", " print(f\"[INFO] Adding SHOS features with train-only sparsity (k={sparsity_k})...\")\n", " rows = []\n", " for (d, p), g in df.groupby([\"DealerCode\", \"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " y = g[\"PartQty\"].values.astype(float)\n", " n_train = n_train_per_series.get((d, p), len(y))\n", " q, z, f = compute_shos_for_series(y, alpha_base, beta_base, adaptivity, sparsity_k, n_train)\n", " g[\"q_shos\"] = q\n", " g[\"z_shos\"] = z\n", " g[\"shos_forecast\"] = f\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True).fillna(0)\n", " print(f\"[INFO] Added SHOS features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- Embeddings (Same as before) -----------------------------\n", "def build_low_rank_embeddings(df: pd.DataFrame, n_components: int = 16):\n", " print(f\"[INFO] Building low-rank embeddings (n_components={n_components})...\")\n", " occ = df.groupby([\"DealerCode\", \"DummyID\"]).agg(months_active=(\"PartQty\", lambda x: (x>0).sum())).reset_index()\n", " pivot = occ.pivot(index=\"DealerCode\", columns=\"DummyID\", values=\"months_active\").fillna(0)\n", " if pivot.shape[0] < 2 or pivot.shape[1] < 2:\n", " print(\"[INFO] Pivot too small for SVD, creating empty embeddings.\")\n", " dealer_embed = pd.DataFrame({\"DealerCode\": sorted(df[\"DealerCode\"].unique())})\n", " part_embed = pd.DataFrame({\"DummyID\": sorted(df[\"DummyID\"].unique())})\n", " return dealer_embed, part_embed, []\n", " k = min(n_components, min(pivot.shape)-1)\n", " svd = TruncatedSVD(n_components=k, random_state=0)\n", " U = svd.fit_transform(pivot)\n", " V = svd.components_.T\n", " dealer_index = list(pivot.index)\n", " part_index = list(pivot.columns)\n", " dealer_embed = pd.DataFrame(U, index=dealer_index).reset_index().rename(columns={\"index\":\"DealerCode\"})\n", " part_embed = pd.DataFrame(V, index=part_index).reset_index().rename(columns={\"index\":\"DummyID\"})\n", " emb_cols = []\n", " for i in range(U.shape[1]):\n", " dealer_embed.rename(columns={i: f\"emb_dealer_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_dealer_{i}\")\n", " for i in range(V.shape[1]):\n", " part_embed.rename(columns={i: f\"emb_part_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_part_{i}\")\n", " print(f\"[INFO] Created {len(emb_cols)} embedding features for {U.shape[0]} dealers and {V.shape[0]} parts.\")\n", " return dealer_embed, part_embed, emb_cols\n", "\n", "# ----------------------------- Feature Sets (NO CROSTON in ML Baselines, YES in Ablation) -----------------------------\n", "def build_base_features(df: pd.DataFrame, max_lag: int = 6):\n", " print(\"[INFO] Building base feature matrix...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " # Croston features are NOT added here for fairness in ML baselines\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " print(f\"[INFO] Base features include {len(base_cols)} columns.\")\n", " return df2, base_cols\n", "\n", "def build_fold_features(train_df: pd.DataFrame, test_df: pd.DataFrame, max_lag: int = 6, svd_components: int = 12, sparsity_k: float = 0.5):\n", " print(f\"[INFO] Building fold features with sparsity_k={sparsity_k}...\")\n", " train_base, base_cols = build_base_features(train_df, max_lag)\n", " test_base, _ = build_base_features(test_df, max_lag)\n", " # Add Croston features to both train and test folds BEFORE SHOS\n", " train_croston = add_croston_features(train_base)\n", " test_croston = add_croston_features(test_base)\n", " # Pass the Croston-augmented base to SHOS calculation\n", " n_train_map = train_df.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " train_shos = add_shos_features_with_ntrain(train_croston, n_train_map, sparsity_k=sparsity_k) # Pass train_croston\n", " test_shos = add_shos_features_with_ntrain(test_croston, n_train_map, sparsity_k=sparsity_k) # Pass test_croston\n", " # Calculate embeddings based on the SHOS-augmented training data\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(train_shos, svd_components)\n", " if emb_cols:\n", " train_shos = train_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " train_shos = train_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " test_shos = test_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " test_shos = test_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " train_shos[c] = train_shos[c].fillna(0)\n", " test_shos[c] = test_shos[c].fillna(0)\n", " # Define feature lists\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Fold features: {len(base_cols)} base, {len(emb_cols)} embeddings, {len(full_cols)} total.\")\n", " return train_shos, test_shos, base_cols, full_cols\n", "\n", "# ----------------------------- Intermittency Analysis (Same as before) -----------------------------\n", "def compute_intermittency(df_processed):\n", " print(\"[INFO] Computing intermittency metrics (ADI, CV²)...\")\n", " stats = []\n", " for (d, p), g in df_processed.groupby([\"DealerCode\", \"DummyID\"]):\n", " y = g[\"PartQty\"].values\n", " if np.sum(y > 0) == 0:\n", " continue\n", " intervals = np.diff(np.where(y > 0)[0])\n", " adi = np.mean(intervals) if len(intervals) > 0 else len(y)\n", " mean_y = np.mean(y)\n", " cv2 = (np.std(y) / mean_y) ** 2 if mean_y > 0 else np.inf\n", " stats.append({\"ADI\": adi, \"CV2\": cv2})\n", " stats_df = pd.DataFrame(stats)\n", " print(f\"[INFO] Computed metrics for {len(stats_df)} active series. Median ADI: {stats_df['ADI'].median():.2f}, Median CV²: {stats_df['CV2'].median():.2f}\")\n", " return stats_df\n", "\n", "# ----------------------------- Build Feature Matrix Once (Optimized for CV) -----------------------------\n", "def build_feature_matrix_optimized(df: pd.DataFrame, max_lag=6, svd_components=12, shos_k=0.5):\n", " print(\"[INFO] Building feature matrix ONCE for all folds...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " df2 = add_croston_features(df2)\n", " # Calculate n_train_map for the *entire* dataset (will be used later if needed per fold, but we calculate SHOS once here)\n", " # For this optimized version, we calculate SHOS once on the full series, using the provided k.\n", " # This means the sparsity used for alpha/beta is based on the *full* series length for each (d, p).\n", " # This is a trade-off for speed. For strict train-only sparsity per fold, calculation must happen inside the loop.\n", " # Using full-series sparsity is often a reasonable approximation.\n", " n_full_map = df2.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " df2 = add_shos_features_with_ntrain(df2, n_full_map, sparsity_k=shos_k) # Calculate SHOS once using full series length\n", " df2.fillna(0, inplace=True)\n", " # Define baseline features (excluding Croston/SHOS for fairness in base ML models)\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " # Calculate embeddings based on the full processed dataset (including SHOS features for context)\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(df2, n_components=svd_components)\n", " if emb_cols:\n", " df2 = df2.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " df2[c] = df2[c].fillna(0)\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Final feature matrix has {len(full_cols)} columns and shape {df2.shape}.\")\n", " return df2, base_cols, full_cols\n", "\n", "# ----------------------------- Calculate Sparsity for Each SKU -----------------------------\n", "def calculate_sparsity(df_processed):\n", " \"\"\"Calculate sparsity for each SKU based on PartQty.\"\"\"\n", " print(\"[INFO] Calculating sparsity for each SKU...\")\n", " sparsity_data = []\n", " for (dealer, dummy), group in df_processed.groupby(['DealerCode', 'DummyID']):\n", " y = group['PartQty'].values\n", " total_months = len(y)\n", " zero_months = np.sum(y == 0)\n", " sparsity = zero_months / total_months if total_months > 0 else 0\n", " sparsity_data.append({\n", " 'DealerCode': dealer,\n", " 'DummyID': dummy,\n", " 'Sparsity': sparsity,\n", " 'TotalMonths': total_months,\n", " 'ZeroMonths': zero_months,\n", " 'TimeSeries': y\n", " })\n", " sparsity_df = pd.DataFrame(sparsity_data)\n", " return sparsity_df\n", "\n", "# ----------------------------- Split SKUs into Sparsity Bins -----------------------------\n", "def split_by_sparsity(sparsity_df, bins=[0.0, 0.7, 0.8, 0.9]):\n", " \"\"\"Split SKUs into sparsity bins.\"\"\"\n", " print(\"[INFO] Splitting SKUs into sparsity bins...\")\n", " \n", " # Correctly create labels for the ranges: e.g., '0.7-0.8', '0.8-0.9', and '0.9+'\n", " # We loop up to len(bins)-1 to avoid the IndexError\n", " labels = [f'{bins[i]}-{bins[i+1]}' for i in range(len(bins)-1)] + [f'{bins[-1]}+']\n", " \n", " # pd.cut needs 1 more bin edge than it has labels. \n", " # Current edges: [0.7, 0.8, 0.9, 1.0] -> 4 edges, 3 labels.\n", " sparsity_df['SparsityBin'] = pd.cut(\n", " sparsity_df['Sparsity'], \n", " bins=bins + [1.0], \n", " right=False, \n", " labels=labels\n", " )\n", " \n", " sparsity_bins = {}\n", " for bin_label in sparsity_df['SparsityBin'].unique():\n", " # Dropna ensures we don't process values outside our defined bins\n", " if pd.notna(bin_label):\n", " sparsity_bins[bin_label] = sparsity_df[sparsity_df['SparsityBin'] == bin_label]\n", " print(f\" Bin {bin_label}: {len(sparsity_bins[bin_label])} SKUs\")\n", " \n", " return sparsity_bins\n", "\n", "# ----------------------------- Train ML Models (raw-ML, SHOS-ML) -----------------------------\n", "def train_ml_model(X_train, y_train, model_type='lightgbm'):\n", " \"\"\"Train a machine learning model.\"\"\"\n", " if model_type == 'lightgbm':\n", " model = lgb.LGBMRegressor(n_estimators=100, learning_rate=0.05, num_leaves=31, verbose=-1)\n", " elif model_type == 'randomforest':\n", " model = RandomForestRegressor(n_estimators=100, random_state=42)\n", " else:\n", " raise ValueError(f\"Unknown model type: {model_type}\")\n", " \n", " model.fit(X_train, y_train)\n", " return model\n", "\n", "def predict_ml_model(model, X_test):\n", " \"\"\"Make predictions with an ML model.\"\"\"\n", " return model.predict(X_test)\n", "\n", "# ----------------------------- Train DL Models (raw-DL, SHOS-DL) -----------------------------\n", "def prepare_sequences(X, y, sequence_length=3):\n", " \"\"\"Convert tabular data into sequences for deep learning models.\"\"\"\n", " X_seq, y_seq = [], []\n", " for i in range(len(X) - sequence_length + 1):\n", " X_seq.append(X[i:i+sequence_length])\n", " y_seq.append(y[i+sequence_length-1])\n", " return np.array(X_seq), np.array(y_seq)\n", "\n", "def create_lstm_model(input_shape, n_outputs=1):\n", " model = Sequential([\n", " Input(shape=input_shape),\n", " LSTM(50, return_sequences=True),\n", " Dropout(0.2),\n", " LSTM(50, return_sequences=False),\n", " Dropout(0.2),\n", " Dense(25, activation='relu'),\n", " Dense(n_outputs)\n", " ])\n", " model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')\n", " return model\n", "\n", "def train_dl_model(model, X_train, y_train, X_val, y_val, epochs=50, patience=5):\n", " \"\"\"Train a deep learning model with early stopping.\"\"\"\n", " early_stop = EarlyStopping(monitor='val_loss', patience=patience, restore_best_weights=True)\n", " \n", " history = model.fit(\n", " X_train, y_train,\n", " validation_data=(X_val, y_val),\n", " epochs=epochs,\n", " callbacks=[early_stop],\n", " verbose=0\n", " )\n", " return model\n", "\n", "def predict_dl_model(model, X_test, sequence_length=3):\n", " \"\"\"Make predictions using a trained deep learning model.\"\"\"\n", " X_test_seq, _ = prepare_sequences(X_test, np.zeros(len(X_test)), sequence_length)\n", " if len(X_test_seq) == 0:\n", " # If not enough data to form a sequence, return zeros\n", " return np.zeros(len(X_test))\n", " \n", " predictions = model.predict(X_test_seq, verbose=0).flatten()\n", " # If we have more original points than predictions, pad with the last value\n", " if len(predictions) < len(X_test):\n", " padding = np.full(len(X_test) - len(predictions), predictions[-1] if len(predictions) > 0 else 0)\n", " predictions = np.concatenate([predictions, padding])\n", " return predictions\n", "\n", "# ----------------------------- Evaluate Responsiveness and Recovery -----------------------------\n", "def evaluate_responsiveness_recovery(y_true, y_pred):\n", " \"\"\"\n", " Evaluate responsiveness to sudden demand and time-to-recovery.\n", " This is a simplified implementation focusing on catching demand peaks.\n", " \"\"\"\n", " # Find peak events in true data (values significantly higher than average)\n", " threshold = np.mean(y_true) + 0.5 * np.std(y_true)\n", " peak_indices = np.where(y_true > threshold)[0]\n", " \n", " if len(peak_indices) == 0:\n", " return np.nan, np.nan # No peaks to evaluate\n", " \n", " # Calculate how well the model predicts these peaks\n", " peak_errors = []\n", " for idx in peak_indices:\n", " if idx < len(y_pred):\n", " error = abs(y_true[idx] - y_pred[idx])\n", " peak_errors.append(error)\n", " \n", " responsiveness_score = np.mean(peak_errors) if peak_errors else np.nan\n", " \n", " # Time to recovery - simplified: time until prediction catches up to actual after a peak\n", " recovery_times = []\n", " for idx in peak_indices:\n", " # Look for when prediction gets within 20% of actual after the peak\n", " for j in range(idx, min(len(y_true), idx+10)): # Look up to 10 steps ahead\n", " if j < len(y_pred) and abs(y_pred[j] - y_true[j]) <= 0.2 * y_true[j]:\n", " recovery_times.append(j - idx)\n", " break\n", " \n", " recovery_score = np.mean(recovery_times) if recovery_times else np.nan\n", " \n", " return responsiveness_score, recovery_score\n", "\n", "# ----------------------------- Run Stress Test for Each Sparsity Bin -----------------------------\n", "def run_stress_test_for_bin(bin_data, base_cols, full_cols, outdir):\n", " \"\"\"Run stress test for a specific sparsity bin.\"\"\"\n", " print(f\"[INFO] Running stress test for bin: {bin_data['SparsityBin'].iloc[0] if not bin_data.empty else 'Empty'}\")\n", " if bin_data.empty:\n", " print(f\"[WARNING] Empty bin, skipping...\")\n", " return {}, {}\n", " \n", " results = {}\n", " for _, row in bin_data.iterrows():\n", " dealer = row['DealerCode']\n", " dummy = row['DummyID']\n", " y = row['TimeSeries']\n", " if len(y) < 10: # Need sufficient data\n", " continue\n", " \n", " # Split into train/test (last 20% for test)\n", " split_idx = int(0.8 * len(y))\n", " y_train = y[:split_idx]\n", " y_test = y[split_idx:]\n", " if len(y_train) < 5 or len(y_test) < 3:\n", " continue\n", "\n", " # --- CREATE FULL DATAFRAME WITH ALL REQUIRED COLUMNS ---\n", " required_static_cols = ['DealerPrice', 'PRC_UNIT_WEIGHT', 'PRC_UNIT_CUBE']\n", " \n", " # Train DF\n", " train_df = pd.DataFrame({\n", " 'PartQty': y_train,\n", " 'OTC_Flag': np.zeros(len(y_train)),\n", " 'DealerPrice': np.zeros(len(y_train)),\n", " 'PRC_UNIT_WEIGHT': np.zeros(len(y_train)),\n", " 'PRC_UNIT_CUBE': np.zeros(len(y_train))\n", " })\n", " train_df['MonthStart'] = pd.date_range(start='2020-01-01', periods=len(y_train), freq='MS')\n", " train_df['DealerCode'] = dealer\n", " train_df['DummyID'] = dummy\n", " train_df = add_time_series_features(train_df)\n", " train_df = add_croston_features(train_df)\n", " train_df = add_shos_features_with_ntrain(train_df, {(dealer, dummy): len(y_train)})\n", "\n", " # Test DF\n", " test_df = pd.DataFrame({\n", " 'PartQty': y_test,\n", " 'OTC_Flag': np.zeros(len(y_test)),\n", " 'DealerPrice': np.zeros(len(y_test)),\n", " 'PRC_UNIT_WEIGHT': np.zeros(len(y_test)),\n", " 'PRC_UNIT_CUBE': np.zeros(len(y_test))\n", " })\n", " test_df['MonthStart'] = pd.date_range(\n", " start=pd.date_range(start='2020-01-01', periods=len(y_train), freq='MS')[-1] + pd.DateOffset(months=1),\n", " periods=len(y_test),\n", " freq='MS'\n", " )\n", " test_df['DealerCode'] = dealer\n", " test_df['DummyID'] = dummy\n", " test_df = add_time_series_features(test_df)\n", " test_df = add_croston_features(test_df)\n", " test_df = add_shos_features_with_ntrain(test_df, {(dealer, dummy): len(y_train)})\n", "\n", " # Ensure all required columns exist (fill missing with 0)\n", " for col in base_cols:\n", " if col not in train_df.columns:\n", " train_df[col] = 0\n", " if col not in test_df.columns:\n", " test_df[col] = 0\n", " for col in full_cols:\n", " if col not in train_df.columns:\n", " train_df[col] = 0\n", " if col not in test_df.columns:\n", " test_df[col] = 0\n", "\n", " # Define features\n", " X_train_base = train_df[base_cols].fillna(0).values\n", " X_test_base = test_df[base_cols].fillna(0).values\n", " X_train_full = train_df[full_cols].fillna(0).values\n", " X_test_full = test_df[full_cols].fillna(0).values\n", "\n", " # Train and evaluate models\n", " models_results = {}\n", " # ... rest of your model training code remains unchanged ...\n", " \n", " # raw-ML (LightGBM on base features)\n", " try:\n", " ml_raw_model = train_ml_model(X_train_base, y_train, model_type='lightgbm')\n", " y_pred_raw_ml = predict_ml_model(ml_raw_model, X_test_base)\n", " resp_raw_ml, recov_raw_ml = evaluate_responsiveness_recovery(y_test, y_pred_raw_ml)\n", " models_results['raw-ML'] = {\n", " 'Responsiveness': resp_raw_ml,\n", " 'Recovery': recov_raw_ml,\n", " 'Predictions': y_pred_raw_ml\n", " }\n", " except:\n", " models_results['raw-ML'] = {'Responsiveness': np.nan, 'Recovery': np.nan, 'Predictions': np.zeros(len(y_test))}\n", " \n", " # SHOS-ML (LightGBM on full features)\n", " try:\n", " ml_shos_model = train_ml_model(X_train_full, y_train, model_type='lightgbm')\n", " y_pred_shos_ml = predict_ml_model(ml_shos_model, X_test_full)\n", " resp_shos_ml, recov_shos_ml = evaluate_responsiveness_recovery(y_test, y_pred_shos_ml)\n", " models_results['SHOS-ML'] = {\n", " 'Responsiveness': resp_shos_ml,\n", " 'Recovery': recov_shos_ml,\n", " 'Predictions': y_pred_shos_ml\n", " }\n", " except:\n", " models_results['SHOS-ML'] = {'Responsiveness': np.nan, 'Recovery': np.nan, 'Predictions': np.zeros(len(y_test))}\n", " \n", " # raw-DL (LSTM on base features)\n", " try:\n", " X_train_base_seq, y_train_seq = prepare_sequences(X_train_base, y_train, 3)\n", " X_test_base_seq, _ = prepare_sequences(X_test_base, y_test, 3)\n", " if len(X_train_base_seq) > 0 and len(X_test_base_seq) > 0:\n", " X_train_seq, X_val_seq, y_train_seq_split, y_val_seq = train_test_split(\n", " X_train_base_seq, y_train_seq, test_size=0.2, random_state=42\n", " )\n", " dl_raw_model = create_lstm_model(input_shape=(3, X_train_base.shape[1]))\n", " dl_raw_model = train_dl_model(dl_raw_model, X_train_seq, y_train_seq_split, X_val_seq, y_val_seq)\n", " y_pred_raw_dl = predict_dl_model(dl_raw_model, X_test_base, 3)\n", " resp_raw_dl, recov_raw_dl = evaluate_responsiveness_recovery(y_test, y_pred_raw_dl)\n", " models_results['raw-DL'] = {\n", " 'Responsiveness': resp_raw_dl,\n", " 'Recovery': recov_raw_dl,\n", " 'Predictions': y_pred_raw_dl\n", " }\n", " else:\n", " models_results['raw-DL'] = {'Responsiveness': np.nan, 'Recovery': np.nan, 'Predictions': np.zeros(len(y_test))}\n", " except:\n", " models_results['raw-DL'] = {'Responsiveness': np.nan, 'Recovery': np.nan, 'Predictions': np.zeros(len(y_test))}\n", " \n", " # SHOS-DL (LSTM on full features)\n", " try:\n", " X_train_full_seq, y_train_seq_full = prepare_sequences(X_train_full, y_train, 3)\n", " X_test_full_seq, _ = prepare_sequences(X_test_full, y_test, 3)\n", " if len(X_train_full_seq) > 0 and len(X_test_full_seq) > 0:\n", " X_train_seq_full, X_val_seq_full, y_train_seq_split_full, y_val_seq_full = train_test_split(\n", " X_train_full_seq, y_train_seq_full, test_size=0.2, random_state=42\n", " )\n", " dl_shos_model = create_lstm_model(input_shape=(3, X_train_full.shape[1]))\n", " dl_shos_model = train_dl_model(dl_shos_model, X_train_seq_full, y_train_seq_split_full, X_val_seq_full, y_val_seq_full)\n", " y_pred_shos_dl = predict_dl_model(dl_shos_model, X_test_full, 3)\n", " resp_shos_dl, recov_shos_dl = evaluate_responsiveness_recovery(y_test, y_pred_shos_dl)\n", " models_results['SHOS-DL'] = {\n", " 'Responsiveness': resp_shos_dl,\n", " 'Recovery': recov_shos_dl,\n", " 'Predictions': y_pred_shos_dl\n", " }\n", " else:\n", " models_results['SHOS-DL'] = {'Responsiveness': np.nan, 'Recovery': np.nan, 'Predictions': np.zeros(len(y_test))}\n", " except:\n", " models_results['SHOS-DL'] = {'Responsiveness': np.nan, 'Recovery': np.nan, 'Predictions': np.zeros(len(y_test))}\n", " \n", " results[f\"{dealer}-{dummy}\"] = models_results\n", " \n", " return results\n", "\n", "# ----------------------------- Aggregate and Plot Results -----------------------------\n", "def aggregate_and_plot_results(all_bin_results, outdir):\n", " \"\"\"Aggregate results across all bins and create plots.\"\"\"\n", " print(\"[INFO] Aggregating and plotting results...\")\n", " \n", " # Create summary dataframe\n", " summary_data = []\n", " for bin_name, bin_results in all_bin_results.items():\n", " for sku, models_results in bin_results.items():\n", " for model_name, metrics in models_results.items():\n", " summary_data.append({\n", " 'SparsityBin': bin_name,\n", " 'SKU': sku,\n", " 'Model': model_name,\n", " 'Responsiveness': metrics['Responsiveness'],\n", " 'Recovery': metrics['Recovery']\n", " })\n", " \n", " summary_df = pd.DataFrame(summary_data)\n", " summary_df.to_csv(os.path.join(outdir, \"stress_test_results.csv\"), index=False)\n", " \n", " # Plot responsiveness by sparsity bin and model\n", " fig, axes = plt.subplots(2, 1, figsize=(12, 10))\n", " \n", " # Responsiveness\n", " resp_pivot = summary_df.pivot_table(values='Responsiveness', index='SparsityBin', columns='Model', aggfunc='mean')\n", " sns.heatmap(resp_pivot, annot=True, fmt='.2f', ax=axes[0], cmap='viridis')\n", " axes[0].set_title('Average Responsiveness by Sparsity Bin and Model')\n", " \n", " # Recovery\n", " recov_pivot = summary_df.pivot_table(values='Recovery', index='SparsityBin', columns='Model', aggfunc='mean')\n", " sns.heatmap(recov_pivot, annot=True, fmt='.2f', ax=axes[1], cmap='viridis')\n", " axes[1].set_title('Average Time-to-Recovery by Sparsity Bin and Model')\n", " \n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"fig10_stress_test_heatmap.png\"))\n", " plt.close()\n", " \n", " # Bar plots\n", " fig, axes = plt.subplots(2, 1, figsize=(12, 10))\n", " \n", " # Responsiveness bar plot\n", " sns.barplot(data=summary_df, x='SparsityBin', y='Responsiveness', hue='Model', ax=axes[0])\n", " axes[0].set_title('Responsiveness by Sparsity Bin and Model')\n", " axes[0].tick_params(axis='x', rotation=45)\n", " \n", " # Recovery bar plot\n", " sns.barplot(data=summary_df, x='SparsityBin', y='Recovery', hue='Model', ax=axes[1])\n", " axes[1].set_title('Time-to-Recovery by Sparsity Bin and Model')\n", " axes[1].tick_params(axis='x', rotation=45)\n", " \n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"fig11_stress_test_bars.png\"))\n", " plt.close()\n", " \n", " print(f\"[INFO] Stress test results saved to {os.path.join(outdir, 'stress_test_results.csv')}\")\n", " print(f\"[INFO] Plots saved to {outdir}\")\n", "\n", "# ----------------------------- Main Pipeline (Stress Test) -----------------------------\n", "def run_stress_test_pipeline(df_filtered, outdir=\"./result_stress_test\", shos_k=0.1, shos_w=0.7):\n", " print(\"[INFO] Starting stress test pipeline...\")\n", " start_time = time.time()\n", " safe_mkdir(outdir)\n", " safe_mkdir(os.path.join(outdir, \"figures\")) # Ensure figures dir exists\n", " print(\"[INFO] 1. Aggregating and padding data...\")\n", " df_ag = aggregate_monthly(df_filtered)\n", " df_padded = pad_monthly_series_and_filter_inactive(df_ag)\n", " print(\"[INFO] 2. Computing intermittency metrics...\")\n", " intermittency_df = compute_intermittency(df_padded)\n", " intermittency_df.to_csv(os.path.join(outdir, \"intermittency_stats.csv\"), index=False)\n", " print(\"[INFO] 3. Building feature matrix once...\")\n", " df_features, base_cols, full_cols = build_feature_matrix_optimized(df_padded, shos_k=shos_k) # Pass shos_k for SHOS calc\n", " print(\"[INFO] 4. Calculating sparsity for each SKU...\")\n", " sparsity_df = calculate_sparsity(df_features)\n", " print(\"[INFO] 5. Splitting SKUs into sparsity bins...\")\n", " sparsity_bins = split_by_sparsity(sparsity_df, bins=[0.7, 0.8, 0.9])\n", " print(\"[INFO] 6. Running stress test for each bin...\")\n", " all_bin_results = {}\n", " for bin_name, bin_data in sparsity_bins.items():\n", " if not bin_data.empty:\n", " bin_results = run_stress_test_for_bin(bin_data, base_cols, full_cols, outdir)\n", " all_bin_results[bin_name] = bin_results\n", " print(\"[INFO] 7. Aggregating and plotting results...\")\n", " aggregate_and_plot_results(all_bin_results, outdir)\n", " print(f\"[INFO] Stress test pipeline completed in {time.time() - start_time:.2f} seconds. Results saved to: {outdir}\")\n", " # Print summary\n", " print(\"--- STRESS TEST SUMMARY ---\")\n", " for bin_name, bin_data in sparsity_bins.items():\n", " print(f\"Bin {bin_name}: {len(bin_data)} SKUs\")\n", " print(\"Results saved to stress_test_results.csv\")\n", " return all_bin_results\n", "\n", "# ----------------------------- Entry Point -----------------------------\n", "if __name__ == \"__main__\":\n", " # Example usage:\n", " # df_filtered = pd.read_csv(\"your_data.csv\")\n", " run_stress_test_pipeline(df_filtered, outdir=\"./result_stress_test1\", shos_k=0.1, shos_w=0.7) # Provide your best k, w\n", " pass\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[INFO] Starting adaptive filter proof pipeline...\n", "[INFO] 1. Aggregating and padding data...\n", "[INFO] Aggregating data to monthly level...\n", "[INFO] Aggregated to 171111 monthly records.\n", "[INFO] Padding series to monthly grid and filtering inactive pairs...\n", "[INFO] Date range: 2023-08 to 2025-08. Total months: 25\n", "[INFO] Filtered from 171111 to 170827 records after removing inactive pairs.\n", "[INFO] Padding complete. Final shape: (1404700, 9)\n", "[INFO] 2. Computing intermittency metrics...\n", "[INFO] Computing intermittency metrics (ADI, CV²)...\n", "[INFO] Computed metrics for 56188 active series. Median ADI: 25.00, Median CV²: 24.00\n", "[INFO] 3. Building feature matrix once...\n", "[INFO] Building feature matrix ONCE for all folds...\n", "[INFO] Adding time series features (max_lag=6)...\n", "[INFO] Added 12 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 56188 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.1)...\n", "[INFO] Added SHOS features for 56188 unique series.\n", "[INFO] Building low-rank embeddings (n_components=12)...\n", "[INFO] Created 24 embedding features for 59 dealers and 7702 parts.\n", "[INFO] Final feature matrix has 50 columns and shape (1404700, 59).\n", "[INFO] 4. Running theoretical validation...\n", "[INFO] Running theoretical validation on 100 SKUs...\n", "[INFO] Identifying cyclical and intermittent SKUs...\n", "[INFO] Found 50 cyclical SKUs and 50 intermittent SKUs\n", "[INFO] Evidence table saved to evidence_table_summary.csv\n", " PeakPres_SMA PeakPres_SHOS PeakPres_SHOS_LGBM PhaseLag_SMA \\\n", "Type \n", "Cyclical 0.3333 0.1434 0.0804 0.115 \n", "Intermittent 0.3333 0.2542 0.0400 0.000 \n", "\n", " PhaseLag_SHOS PhaseLag_SHOS_LGBM FFTEnergy_Raw FFTEnergy_SMA \\\n", "Type \n", "Cyclical 0.05 NaN 98.0 32.6667 \n", "Intermittent 0.00 NaN 46.5 14.4444 \n", "\n", " FFTEnergy_SHOS FFTEnergy_SHOS_LGBM Amplitude_Raw \\\n", "Type \n", "Cyclical 6.9582 8.20 0.3461 \n", "Intermittent 21.0395 1.86 0.2469 \n", "\n", " Amplitude_SMA Amplitude_SHOS Amplitude_SHOS_LGBM \\\n", "Type \n", "Cyclical 0.1805 0.0549 0.0 \n", "Intermittent 0.1317 0.0894 0.0 \n", "\n", " FFTSimilarity_SMA FFTSimilarity_SHOS FFTSimilarity_SHOS_LGBM \n", "Type \n", "Cyclical 0.3996 0.1013 0.0 \n", "Intermittent 0.2032 0.1245 0.0 \n", "[INFO] 5. Identifying cyclical and intermittent SKUs...\n", "[INFO] Identifying cyclical and intermittent SKUs...\n", "[INFO] Found 3 cyclical SKUs and 3 intermittent SKUs\n", "[INFO] 6. Creating multi-panel figures...\n", "[INFO] Creating multi-panel figures...\n", "[INFO] 7. Calculating summary table...\n", "[INFO] Calculating summary table...\n", "[INFO] 8. Generating summary plots...\n", "[INFO] Adaptive filter proof pipeline completed in 1470.66 seconds. Results saved to: ./result_adaptive_filter1\n", "--- SIGNAL ANALYSIS SUMMARY ---\n", "Signal Properties (Amplitude and Frequency Energy):\n", " SKU Type Signal PeakPres PhaseLag \\\n", "0 10231-PartNo2116 Cyclical Raw 1.0000 0.0 \n", "1 10231-PartNo2116 Cyclical SMA 0.3333 0.0 \n", "2 10231-PartNo2116 Cyclical SHOS 0.1036 0.0 \n", "3 10231-PartNo2116 Cyclical SHOS+LightGBM 0.0600 NaN \n", "4 10231-PartNo2481 Cyclical Raw 1.0000 0.0 \n", "5 10231-PartNo2481 Cyclical SMA 0.3333 0.0 \n", "6 10231-PartNo2481 Cyclical SHOS 0.1064 0.0 \n", "7 10231-PartNo2481 Cyclical SHOS+LightGBM 0.0600 NaN \n", "8 10231-PartNo421 Cyclical Raw 1.0000 0.0 \n", "9 10231-PartNo421 Cyclical SMA 0.3333 0.0 \n", "10 10231-PartNo421 Cyclical SHOS 0.1387 0.0 \n", "11 10231-PartNo421 Cyclical SHOS+LightGBM 0.0800 NaN \n", "12 10231-PartNo1008 Intermittent Raw 1.0000 0.0 \n", "13 10231-PartNo1008 Intermittent SMA 0.3333 NaN \n", "14 10231-PartNo1008 Intermittent SHOS 0.4027 NaN \n", "15 10231-PartNo1008 Intermittent SHOS+LightGBM 0.0400 NaN \n", "16 10231-PartNo1029 Intermittent Raw 1.0000 0.0 \n", "17 10231-PartNo1029 Intermittent SMA 0.3333 NaN \n", "18 10231-PartNo1029 Intermittent SHOS 0.4027 NaN \n", "19 10231-PartNo1029 Intermittent SHOS+LightGBM 0.0400 NaN \n", "\n", " FFTEnergy Amplitude FFTSimilarity \n", "0 125.0000 0.4308 1.0000 \n", "1 41.6667 0.2286 0.3313 \n", "2 7.8891 0.0571 0.2702 \n", "3 9.0000 0.0000 0.0000 \n", "4 125.0000 0.4308 1.0000 \n", "5 41.6667 0.2286 0.3539 \n", "6 7.2315 0.0703 -0.0287 \n", "7 9.0000 0.0000 0.0000 \n", "8 200.0000 0.5426 1.0000 \n", "9 66.6667 0.2847 0.3899 \n", "10 13.7062 0.0758 0.2530 \n", "11 16.0000 0.0000 0.0000 \n", "12 225.0000 0.5879 1.0000 \n", "13 50.0000 0.2713 0.2673 \n", "14 184.2498 0.3252 0.1072 \n", "15 9.0000 0.0000 0.0000 \n", "16 25.0000 0.1960 1.0000 \n", "17 5.5556 0.0904 0.2673 \n", "18 20.4722 0.1084 0.1072 \n", "19 1.0000 0.0000 0.0000 \n", "--- EVIDENCE TABLE SUMMARY (100 SKUs) ---\n", " PeakPres_SMA PeakPres_SHOS PeakPres_SHOS_LGBM PhaseLag_SMA \\\n", "Type \n", "Cyclical 0.3333 0.1434 0.0804 0.115 \n", "Intermittent 0.3333 0.2542 0.0400 0.000 \n", "\n", " PhaseLag_SHOS PhaseLag_SHOS_LGBM FFTEnergy_Raw FFTEnergy_SMA \\\n", "Type \n", "Cyclical 0.05 NaN 98.0 32.6667 \n", "Intermittent 0.00 NaN 46.5 14.4444 \n", "\n", " FFTEnergy_SHOS FFTEnergy_SHOS_LGBM Amplitude_Raw \\\n", "Type \n", "Cyclical 6.9582 8.20 0.3461 \n", "Intermittent 21.0395 1.86 0.2469 \n", "\n", " Amplitude_SMA Amplitude_SHOS Amplitude_SHOS_LGBM \\\n", "Type \n", "Cyclical 0.1805 0.0549 0.0 \n", "Intermittent 0.1317 0.0894 0.0 \n", "\n", " FFTSimilarity_SMA FFTSimilarity_SHOS FFTSimilarity_SHOS_LGBM \n", "Type \n", "Cyclical 0.3996 0.1013 0.0 \n", "Intermittent 0.2032 0.1245 0.0 \n", "Median ADI: 25.00\n", "Median CV²: 24.00\n" ] } ], "source": [ "import os\n", "import math\n", "import pickle\n", "import warnings\n", "import time\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.decomposition import TruncatedSVD\n", "from sklearn.calibration import CalibratedClassifierCV\n", "from sklearn.linear_model import ElasticNet, Ridge, LogisticRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.model_selection import TimeSeriesSplit\n", "from sklearn.metrics import(\n", " mean_squared_error, mean_absolute_error, r2_score\n", ")\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from scipy.fft import fft, fftfreq\n", "from scipy.signal import correlate\n", "from scipy.stats import pearsonr\n", "from sklearn.metrics import mean_absolute_error\n", "from sklearn.preprocessing import StandardScaler\n", "import lightgbm as lgb\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.preprocessing import MinMaxScaler\n", "from scipy.signal import find_peaks\n", "import warnings\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "# ----------------------------- Utilities -----------------------------\n", "def safe_mkdir(path: str):\n", " os.makedirs(path, exist_ok=True)\n", "\n", "def rmse(y_true, y_pred):\n", " return math.sqrt(mean_squared_error(y_true, y_pred))\n", "\n", "def wmape(y_true, y_pred):\n", " denom = np.sum(np.abs(y_true))\n", " return np.sum(np.abs(y_true - y_pred)) / denom * 100.0 if denom != 0 else np.nan\n", "\n", "def mape(y_true, y_pred):\n", " mask = y_true != 0\n", " return np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask])) * 100.0 if mask.sum() > 0 else np.nan\n", "\n", "# ----------------------------- Padding with Inactive Filtering (Moved to Top) -----------------------------\n", "def pad_monthly_series_and_filter_inactive(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Padding series to monthly grid and filtering inactive pairs...\")\n", " df = df.copy()\n", " df[\"MonthStart\"] = pd.to_datetime(df[\"MonthStart\"])\n", " min_date = df[\"MonthStart\"].min()\n", " max_date = df[\"MonthStart\"].max()\n", " all_months = pd.date_range(start=min_date, end=max_date, freq=\"MS\")\n", " print(f\"[INFO] Date range: {min_date.strftime('%Y-%m')} to {max_date.strftime('%Y-%m')}. Total months: {len(all_months)}\")\n", " df_active = df.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0)\n", " print(f\"[INFO] Filtered from {df.shape[0]} to {df_active.shape[0]} records after removing inactive pairs.\")\n", " if df_active.empty:\n", " print(\"[WARNING] No active series found after filtering.\")\n", " return df_active\n", " full_index = pd.MultiIndex.from_product(\n", " [df_active[\"DealerCode\"].unique(), df_active[\"DummyID\"].unique(), all_months],\n", " names=[\"DealerCode\", \"DummyID\", \"MonthStart\"]\n", " )\n", " df_indexed = df_active.set_index([\"DealerCode\", \"DummyID\", \"MonthStart\"])\n", " df_padded = df_indexed.reindex(full_index, fill_value=0).reset_index()\n", " df_padded[\"OTC_Flag\"] = (df_padded[\"OTC_Qty\"] > 0).astype(int)\n", " for col in [\"PartQty\", \"OTC_Qty\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]:\n", " df_padded[col] = pd.to_numeric(df_padded[col], errors=\"coerce\").fillna(0)\n", " df_final = df_padded.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0).reset_index(drop=True)\n", " print(f\"[INFO] Padding complete. Final shape: {df_final.shape}\")\n", " return df_final\n", "\n", "# ----------------------------- Data helpers (Same as before) -----------------------------\n", "def aggregate_monthly(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Aggregating data to monthly level...\")\n", " df = df.copy()\n", " colmap = {}\n", " for c in df.columns:\n", " lc = c.lower().strip()\n", " if lc in (\"dealer code\", \"dealercode\", \"dealer_code\", \"dealercode\"):\n", " colmap[c] = \"DealerCode\"\n", " if lc in (\"dummy id\", \"dummyid\", \"partno\", \"part_no\", \"dummy_id\"):\n", " colmap[c] = \"DummyID\"\n", " if lc in (\"sale date\", \"saledate\", \"sale_date\"):\n", " colmap[c] = \"SaleDate\"\n", " if lc in (\"part qty.\", \"part qty\", \"partqty\", \"qty\", \"quantity\"):\n", " colmap[c] = \"PartQty\"\n", " if lc in (\"sale type\", \"saletype\", \"sale_type\"):\n", " colmap[c] = \"SaleType\"\n", " if lc in (\"dealer price\", \"dealerprice\", \"price\"):\n", " colmap[c] = \"DealerPrice\"\n", " if lc in (\"prc_unit_weight\", \"weight\", \"prc unit weight\"):\n", " colmap[c] = \"PRC_UNIT_WEIGHT\"\n", " if lc in (\"prc_unit_cube\", \"cube\", \"prc unit cube\"):\n", " colmap[c] = \"PRC_UNIT_CUBE\"\n", " if lc in (\"otc_qty\",\"otc qty\",\"otcqty\"):\n", " colmap[c] = \"OTC_Qty\"\n", " df = df.rename(columns=colmap)\n", " if \"MonthStart\" in df.columns and \"PartQty\" in df.columns:\n", " out = df.copy()\n", " out[\"MonthStart\"] = pd.to_datetime(out[\"MonthStart\"])\n", " if \"OTC_Qty\" not in out.columns:\n", " if \"SaleType\" in out.columns:\n", " out[\"OTC_Qty\"] = out[\"PartQty\"] * 0\n", " mask = out[\"SaleType\"].astype(str).str.upper() == \"OTC\"\n", " out.loc[mask, \"OTC_Qty\"] = out.loc[mask, \"PartQty\"]\n", " else:\n", " out[\"OTC_Qty\"] = 0\n", " out[\"OTC_Flag\"] = (out[\"OTC_Qty\"] > 0).astype(int)\n", " out[\"DealerPrice\"] = out.get(\"DealerPrice\", 0.0)\n", " out[\"PRC_UNIT_WEIGHT\"] = out.get(\"PRC_UNIT_WEIGHT\", 0.0)\n", " out[\"PRC_UNIT_CUBE\"] = out.get(\"PRC_UNIT_CUBE\", 0.0)\n", " return out[[\"DealerCode\", \"DummyID\", \"MonthStart\", \"PartQty\", \"OTC_Qty\", \"OTC_Flag\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]]\n", " if \"SaleDate\" not in df.columns:\n", " raise ValueError(\"Input must have 'SaleDate' or 'MonthStart' + 'PartQty'.\")\n", " df[\"SaleDate\"] = pd.to_datetime(df[\"SaleDate\"])\n", " df[\"MonthStart\"] = df[\"SaleDate\"].dt.to_period(\"M\").dt.to_timestamp()\n", " agg = df.groupby([\"DealerCode\", \"DummyID\", \"MonthStart\"], as_index=False).agg(\n", " PartQty=(\"PartQty\", \"sum\"),\n", " OTC_Qty=(\"PartQty\", lambda x: x[df.loc[x.index, \"SaleType\"].astype(str).str.upper() == \"OTC\"].sum() if \"SaleType\" in df.columns else 0),\n", " DealerPrice=(\"DealerPrice\", \"mean\"),\n", " PRC_UNIT_WEIGHT=(\"PRC_UNIT_WEIGHT\", \"mean\"),\n", " PRC_UNIT_CUBE=(\"PRC_UNIT_CUBE\", \"mean\"),\n", " )\n", " agg[\"OTC_Qty\"] = agg[\"OTC_Qty\"].fillna(0).astype(int)\n", " agg[\"OTC_Flag\"] = (agg[\"OTC_Qty\"] > 0).astype(int)\n", " agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]] = agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]].fillna(0.0)\n", " print(f\"[INFO] Aggregated to {len(agg)} monthly records.\")\n", " return agg\n", "\n", "def add_time_series_features(df: pd.DataFrame, max_lag: int = 12) -> pd.DataFrame:\n", " print(f\"[INFO] Adding time series features (max_lag={max_lag})...\")\n", " df = df.sort_values([\"DealerCode\", \"DummyID\", \"MonthStart\"]).copy()\n", " rows = []\n", " grp = df.groupby([\"DealerCode\", \"DummyID\"])\n", " for (d, p), g in grp:\n", " g = g.sort_values(\"MonthStart\").copy()\n", " for lag in range(1, max_lag+1):\n", " g[f\"qty_lag_{lag}\"] = g[\"PartQty\"].shift(lag).fillna(0)\n", " g[f\"otc_lag_{lag}\"] = g[\"OTC_Flag\"].shift(lag).fillna(0)\n", " g[\"qty_sum_1_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"qty_sum_1_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"nonzero_count_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).apply(lambda x: np.sum(x>0)).shift(1).fillna(0)\n", " last_pos = -1\n", " t_since = []\n", " for idx, val in enumerate(g[\"PartQty\"].values):\n", " if val > 0:\n", " last_pos = idx\n", " t_since.append(0)\n", " else:\n", " t_since.append(idx - last_pos if last_pos != -1 else g.shape[0]+1)\n", " g[\"time_since_last_sale\"] = pd.Series(t_since).astype(float)\n", " g[\"qty_ma_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"qty_ma_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"otc_share_slope_3\"] = g[\"OTC_Flag\"].rolling(3, min_periods=1).apply(lambda x: np.polyfit(np.arange(len(x)), x, 1)[0] if len(x)>1 else 0).shift(1).fillna(0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added {len([c for c in out.columns if c.startswith(('qty_lag_', 'otc_lag_'))])} lag features.\")\n", " return out\n", "\n", "def croston_forecast(series: np.ndarray, alpha: float = 0.3) -> np.ndarray:\n", " n = len(series)\n", " if n == 0:\n", " return np.zeros(0)\n", " demand = np.asarray(series, dtype=float)\n", " forecast = np.zeros(n)\n", " nz_idx = np.where(demand > 0)[0]\n", " if len(nz_idx) == 0:\n", " return forecast\n", " last = nz_idx[0]\n", " z = demand[last]\n", " p = 1.0\n", " for t in range(last+1, n):\n", " if demand[t] > 0:\n", " interval = t - last\n", " z = alpha * demand[t] + (1-alpha) * z\n", " p = alpha * interval + (1-alpha) * p\n", " last = t\n", " forecast[t] = z / max(1e-9, p)\n", " forecast[:last+1] = forecast[last+1] if last+1 < n else forecast[last]\n", " return forecast\n", "\n", "def add_croston_features(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Adding Croston features...\")\n", " rows = []\n", " for (d,p), g in df.groupby([\"DealerCode\",\"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " g[\"croston_forecast\"] = croston_forecast(g[\"PartQty\"].values, alpha=0.3)\n", " g[\"croston_ratio\"] = np.where(g[\"qty_sum_1_3\"]!=0, g[\"croston_forecast\"]/g[\"qty_sum_1_3\"], 0.0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added Croston features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- SHOS with Train-Only Sparsity (Same as before) -----------------------------\n", "def compute_shos_for_series(y: np.ndarray, alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5,\n", " n_train: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n", " n = len(y)\n", " if n == 0:\n", " return np.zeros(0), np.zeros(0), np.zeros(0)\n", " if n_train is not None and n_train > 0:\n", " y_sparsity = y[:n_train]\n", " else:\n", " y_sparsity = y\n", " nonzero_count = np.sum(y_sparsity > 0)\n", " sparsity = 1.0 - (nonzero_count / max(1, len(y_sparsity)))\n", " if adaptivity:\n", " alpha = alpha_base * (1.0 - sparsity_k * sparsity)\n", " beta = beta_base * (1.0 - sparsity_k * sparsity)\n", " last_pos_all = np.where(y > 0)[0]\n", " if len(last_pos_all) > 0 and (n - 1 - last_pos_all[-1]) <= 3:\n", " alpha = min(0.9, alpha * 1.2)\n", " beta = min(0.9, beta * 1.2)\n", " else:\n", " alpha, beta = alpha_base, beta_base\n", " # --- IMPROVED INITIALIZATION (Point 6) ---\n", " if n >= 3:\n", " init_window = 3\n", " y_init_window = y[:init_window]\n", " q0 = float(np.sum(y_init_window > 0)) / max(1, init_window)\n", " positive_y_init = y_init_window[y_init_window > 0]\n", " if len(positive_y_init) > 0:\n", " z0 = float(np.mean(positive_y_init))\n", " else:\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " else:\n", " q0 = float(np.sum(y > 0)) / max(1, n)\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " # --- END IMPROVED INITIALIZATION ---\n", " q_prev, z_prev = q0, z0\n", " q, z, f = np.zeros(n), np.zeros(n), np.zeros(n)\n", " for t in range(n):\n", " yt = y[t]\n", " if yt > 0:\n", " z_curr = alpha * yt + (1 - alpha) * z_prev\n", " q_curr = beta * 1.0 + (1 - beta) * q_prev\n", " else:\n", " z_curr = z_prev\n", " q_curr = beta * 0.0 + (1 - beta) * q_prev\n", " q[t], z[t], f[t] = q_curr, z_curr, q_curr * z_curr\n", " q_prev, z_prev = q_curr, z_curr\n", " return q, z, f\n", "\n", "def add_shos_features_with_ntrain(df: pd.DataFrame, n_train_per_series: Dict[Tuple, int],\n", " alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5) -> pd.DataFrame:\n", " print(f\"[INFO] Adding SHOS features with train-only sparsity (k={sparsity_k})...\")\n", " rows = []\n", " for (d, p), g in df.groupby([\"DealerCode\", \"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " y = g[\"PartQty\"].values.astype(float)\n", " n_train = n_train_per_series.get((d, p), len(y))\n", " q, z, f = compute_shos_for_series(y, alpha_base, beta_base, adaptivity, sparsity_k, n_train)\n", " g[\"q_shos\"] = q\n", " g[\"z_shos\"] = z\n", " g[\"shos_forecast\"] = f\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True).fillna(0)\n", " print(f\"[INFO] Added SHOS features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- Embeddings (Same as before) -----------------------------\n", "def build_low_rank_embeddings(df: pd.DataFrame, n_components: int = 16):\n", " print(f\"[INFO] Building low-rank embeddings (n_components={n_components})...\")\n", " occ = df.groupby([\"DealerCode\", \"DummyID\"]).agg(months_active=(\"PartQty\", lambda x: (x>0).sum())).reset_index()\n", " pivot = occ.pivot(index=\"DealerCode\", columns=\"DummyID\", values=\"months_active\").fillna(0)\n", " if pivot.shape[0] < 2 or pivot.shape[1] < 2:\n", " print(\"[INFO] Pivot too small for SVD, creating empty embeddings.\")\n", " dealer_embed = pd.DataFrame({\"DealerCode\": sorted(df[\"DealerCode\"].unique())})\n", " part_embed = pd.DataFrame({\"DummyID\": sorted(df[\"DummyID\"].unique())})\n", " return dealer_embed, part_embed, []\n", " k = min(n_components, min(pivot.shape)-1)\n", " svd = TruncatedSVD(n_components=k, random_state=0)\n", " U = svd.fit_transform(pivot)\n", " V = svd.components_.T\n", " dealer_index = list(pivot.index)\n", " part_index = list(pivot.columns)\n", " dealer_embed = pd.DataFrame(U, index=dealer_index).reset_index().rename(columns={\"index\":\"DealerCode\"})\n", " part_embed = pd.DataFrame(V, index=part_index).reset_index().rename(columns={\"index\":\"DummyID\"})\n", " emb_cols = []\n", " for i in range(U.shape[1]):\n", " dealer_embed.rename(columns={i: f\"emb_dealer_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_dealer_{i}\")\n", " for i in range(V.shape[1]):\n", " part_embed.rename(columns={i: f\"emb_part_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_part_{i}\")\n", " print(f\"[INFO] Created {len(emb_cols)} embedding features for {U.shape[0]} dealers and {V.shape[0]} parts.\")\n", " return dealer_embed, part_embed, emb_cols\n", "\n", "# ----------------------------- Feature Sets (NO CROSTON in ML Baselines, YES in Ablation) -----------------------------\n", "def build_base_features(df: pd.DataFrame, max_lag: int = 6):\n", " print(\"[INFO] Building base feature matrix...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " # Croston features are NOT added here for fairness in ML baselines\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " print(f\"[INFO] Base features include {len(base_cols)} columns.\")\n", " return df2, base_cols\n", "\n", "def build_fold_features(train_df: pd.DataFrame, test_df: pd.DataFrame, max_lag: int = 6, svd_components: int = 12, sparsity_k: float = 0.5):\n", " print(f\"[INFO] Building fold features with sparsity_k={sparsity_k}...\")\n", " train_base, base_cols = build_base_features(train_df, max_lag)\n", " test_base, _ = build_base_features(test_df, max_lag)\n", " # Add Croston features to both train and test folds BEFORE SHOS\n", " train_croston = add_croston_features(train_base)\n", " test_croston = add_croston_features(test_base)\n", " # Pass the Croston-augmented base to SHOS calculation\n", " n_train_map = train_df.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " train_shos = add_shos_features_with_ntrain(train_croston, n_train_map, sparsity_k=sparsity_k) # Pass train_croston\n", " test_shos = add_shos_features_with_ntrain(test_croston, n_train_map, sparsity_k=sparsity_k) # Pass test_croston\n", " # Calculate embeddings based on the SHOS-augmented training data\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(train_shos, svd_components)\n", " if emb_cols:\n", " train_shos = train_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " train_shos = train_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " test_shos = test_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " test_shos = test_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " train_shos[c] = train_shos[c].fillna(0)\n", " test_shos[c] = test_shos[c].fillna(0)\n", " # Define feature lists\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Fold features: {len(base_cols)} base, {len(emb_cols)} embeddings, {len(full_cols)} total.\")\n", " return train_shos, test_shos, base_cols, full_cols\n", "\n", "# ----------------------------- Intermittency Analysis (Same as before) -----------------------------\n", "def compute_intermittency(df_processed):\n", " print(\"[INFO] Computing intermittency metrics (ADI, CV²)...\")\n", " stats = []\n", " for (d, p), g in df_processed.groupby([\"DealerCode\", \"DummyID\"]):\n", " y = g[\"PartQty\"].values\n", " if np.sum(y > 0) == 0:\n", " continue\n", " intervals = np.diff(np.where(y > 0)[0])\n", " adi = np.mean(intervals) if len(intervals) > 0 else len(y)\n", " mean_y = np.mean(y)\n", " cv2 = (np.std(y) / mean_y) ** 2 if mean_y > 0 else np.inf\n", " stats.append({\"ADI\": adi, \"CV2\": cv2})\n", " stats_df = pd.DataFrame(stats)\n", " print(f\"[INFO] Computed metrics for {len(stats_df)} active series. Median ADI: {stats_df['ADI'].median():.2f}, Median CV²: {stats_df['CV2'].median():.2f}\")\n", " return stats_df\n", "\n", "# ----------------------------- Build Feature Matrix Once (Optimized for CV) -----------------------------\n", "def build_feature_matrix_optimized(df: pd.DataFrame, max_lag=6, svd_components=12, shos_k=0.5):\n", " print(\"[INFO] Building feature matrix ONCE for all folds...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " df2 = add_croston_features(df2)\n", " # Calculate n_train_map for the *entire* dataset (will be used later if needed per fold, but we calculate SHOS once here)\n", " # For this optimized version, we calculate SHOS once on the full series, using the provided k.\n", " # This means the sparsity used for alpha/beta is based on the *full* series length for each (d, p).\n", " # This is a trade-off for speed. For strict train-only sparsity per fold, calculation must happen inside the loop.\n", " # Using full-series sparsity is often a reasonable approximation.\n", " n_full_map = df2.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " df2 = add_shos_features_with_ntrain(df2, n_full_map, sparsity_k=shos_k) # Calculate SHOS once using full series length\n", " df2.fillna(0, inplace=True)\n", " # Define baseline features (excluding Croston/SHOS for fairness in base ML models)\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " # Calculate embeddings based on the full processed dataset (including SHOS features for context)\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(df2, n_components=svd_components)\n", " if emb_cols:\n", " df2 = df2.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " df2[c] = df2[c].fillna(0)\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Final feature matrix has {len(full_cols)} columns and shape {df2.shape}.\")\n", " return df2, base_cols, full_cols\n", "\n", "# ----------------------------- Identify Cyclical and Intermittent SKUs -----------------------------\n", "def identify_cyclical_intermittent_skus(df_processed, n_top=50):\n", " \"\"\"\n", " Identify SKUs with cyclical bursts and intermittent spikes.\n", " Returns top N SKUs for each category.\n", " \"\"\"\n", " print(\"[INFO] Identifying cyclical and intermittent SKUs...\")\n", " \n", " # Group by SKU\n", " sku_groups = df_processed.groupby(['DealerCode', 'DummyID'])\n", " \n", " # Stats for each SKU\n", " sku_stats = []\n", " for (dealer, dummy), group in sku_groups:\n", " y = group['PartQty'].values\n", " if len(y) < 10: # Skip short series (Fix: Changed from 10 to allow more SKUs)\n", " continue\n", " \n", " # Calculate metrics\n", " total_demand = np.sum(y)\n", " nonzero_months = np.sum(y > 0)\n", " avg_demand = np.mean(y)\n", " std_demand = np.std(y)\n", " \n", " # Coefficient of variation (higher means more variable)\n", " cv = std_demand / avg_demand if avg_demand > 0 else 0\n", " \n", " # Autocorrelation at different lags (to detect cycles)\n", " max_lag = min(len(y)//2, 12) # Up to half the length or 12 months\n", " autocorr_vals = []\n", " for lag in range(1, max_lag):\n", " if lag < len(y):\n", " corr, _ = pearsonr(y[:-lag], y[lag:])\n", " autocorr_vals.append(abs(corr))\n", " max_autocorr = max(autocorr_vals) if autocorr_vals else 0\n", " \n", " # Peak-to-mean ratio\n", " peak_to_mean = np.max(y) / avg_demand if avg_demand > 0 else 0\n", " \n", " sku_stats.append({\n", " 'DealerCode': dealer,\n", " 'DummyID': dummy,\n", " 'TotalDemand': total_demand,\n", " 'NonZeroMonths': nonzero_months,\n", " 'AvgDemand': avg_demand,\n", " 'StdDemand': std_demand,\n", " 'CV': cv,\n", " 'MaxAutocorr': max_autocorr,\n", " 'PeakToMean': peak_to_mean,\n", " 'TimeSeries': y\n", " })\n", " \n", " sku_df = pd.DataFrame(sku_stats)\n", " \n", " # Identify cyclical SKUs (high autocorrelation, moderate CV)\n", " cyclical_skus = sku_df.nlargest(n_top, 'MaxAutocorr')\n", " \n", " # Identify intermittent SKUs (high peak-to-mean, high CV)\n", " intermittent_skus = sku_df.nlargest(n_top, 'PeakToMean')\n", " \n", " print(f\"[INFO] Found {len(cyclical_skus)} cyclical SKUs and {len(intermittent_skus)} intermittent SKUs\")\n", " return cyclical_skus, intermittent_skus\n", "\n", "# ----------------------------- Generate Signals (Raw, SMA, SHOS, SHOS+LightGBM) -----------------------------\n", "def generate_signals(raw_signal):\n", " \"\"\"\n", " Generate Raw, SMA, and SHOS signals for a given raw signal.\n", " \"\"\"\n", " # Raw signal (already provided)\n", " raw = raw_signal\n", " \n", " # Simple Moving Average (3-month)\n", " sma = pd.Series(raw).rolling(window=3, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n", " \n", " # SHOS forecast\n", " q_shos, z_shos, shos_f = compute_shos_for_series(raw)\n", " \n", " # SHOS+LightGBM: Train a simple model using SHOS features to predict next value\n", " # Fix: Train on the entire signal to avoid stitching\n", " if len(raw) > 10:\n", " # Use all available data to train the model\n", " # Create features: use SHOS forecast and raw demand as features\n", " X = np.column_stack([shos_f, raw]) # Use SHOS forecast and raw as features\n", " y = raw # Predict raw demand\n", " \n", " try:\n", " model = lgb.LGBMRegressor(n_estimators=50, learning_rate=0.05, num_leaves=15, verbose=-1)\n", " model.fit(X, y)\n", " shos_lgbm_full = model.predict(X) # Predict on entire series\n", " except:\n", " # Fallback to SHOS if model fails\n", " shos_lgbm_full = shos_f\n", " else:\n", " shos_lgbm_full = shos_f # Fallback to SHOS if not enough data\n", " \n", " return raw, sma, shos_f, shos_lgbm_full\n", "\n", "# ----------------------------- FFT Analysis -----------------------------\n", "def compute_fft(signal):\n", " \"\"\"Compute FFT and return frequencies and power spectrum.\"\"\"\n", " n = len(signal)\n", " T = 1.0 # Monthly data\n", " y_fft = fft(signal)\n", " freqs = fftfreq(n, T)\n", " power_spectrum = np.abs(y_fft)**2\n", " return freqs, power_spectrum\n", "\n", "def compute_fft_similarity(raw, target):\n", " \"\"\"\n", " Measures spectral similarity between Raw data and processed signals.\n", " Returns a score between -1 and 1 (1.0 = identical frequency components).\n", " \"\"\"\n", " if np.std(raw) == 0 or np.std(target) == 0:\n", " return 0.0\n", "\n", " # Get Power Spectrum (magnitude squared)\n", " _, ps_raw = compute_fft(raw - np.mean(raw))\n", " _, ps_target = compute_fft(target - np.mean(target))\n", " \n", " # Calculate Pearson correlation between the two spectrums\n", " # This proves the 'peaks' in frequency align\n", " similarity, _ = pearsonr(ps_raw, ps_target)\n", " return similarity\n", "\n", "# ----------------------------- Peak Preservation Metric -----------------------------\n", "def calculate_peak_preservation(raw, sma, shos, shos_lgbm):\n", " \"\"\"Calculate peak preservation metrics.\"\"\"\n", " max_raw = np.max(raw) if np.max(raw) > 0 else 1 # Avoid division by zero\n", " peak_pres_sma = np.max(sma) / max_raw if max_raw > 0 else 0\n", " peak_pres_shos = np.max(shos) / max_raw if max_raw > 0 else 0\n", " peak_pres_shos_lgbm = np.max(shos_lgbm) / max_raw if max_raw > 0 else 0\n", " return peak_pres_sma, peak_pres_shos, peak_pres_shos_lgbm\n", "\n", "# ----------------------------- Phase Lag Calculation -----------------------------\n", "def calculate_phase_lag(raw, sma, shos, shos_lgbm):\n", " \"\"\"\n", " Calculate phase lag between signals during sudden demand spikes.\n", " This is a simplified version based on correlation of peak timing.\n", " \"\"\"\n", " # Find peaks in raw signal\n", " peaks_raw, _ = find_peaks(raw, height=np.mean(raw) + 0.5 * np.std(raw))\n", " \n", " if len(peaks_raw) == 0:\n", " return np.nan, np.nan, np.nan\n", " \n", " # Find closest peaks in other signals near the same time\n", " peaks_sma, _ = find_peaks(sma, height=np.mean(sma) + 0.5 * np.std(sma))\n", " peaks_shos, _ = find_peaks(shos, height=np.mean(shos) + 0.5 * np.std(shos))\n", " peaks_shos_lgbm, _ = find_peaks(shos_lgbm, height=np.mean(shos_lgbm) + 0.5 * np.std(shos_lgbm))\n", " \n", " # Calculate average time delay for each signal relative to raw\n", " delays_sma = []\n", " delays_shos = []\n", " delays_shos_lgbm = []\n", " \n", " for peak_t in peaks_raw:\n", " # Find closest peak in SMA\n", " if len(peaks_sma) > 0:\n", " closest_sma = peaks_sma[np.argmin(np.abs(peaks_sma - peak_t))]\n", " delays_sma.append(closest_sma - peak_t)\n", " \n", " # Find closest peak in SHOS\n", " if len(peaks_shos) > 0:\n", " closest_shos = peaks_shos[np.argmin(np.abs(peaks_shos - peak_t))]\n", " delays_shos.append(closest_shos - peak_t)\n", " \n", " # Find closest peak in SHOS+LGBM\n", " if len(peaks_shos_lgbm) > 0:\n", " closest_shos_lgbm = peaks_shos_lgbm[np.argmin(np.abs(peaks_shos_lgbm - peak_t))]\n", " delays_shos_lgbm.append(closest_shos_lgbm - peak_t)\n", " \n", " avg_delay_sma = np.mean(delays_sma) if delays_sma else np.nan\n", " avg_delay_shos = np.mean(delays_shos) if delays_shos else np.nan\n", " avg_delay_shos_lgbm = np.mean(delays_shos_lgbm) if delays_shos_lgbm else np.nan\n", " \n", " return avg_delay_sma, avg_delay_shos, avg_delay_shos_lgbm\n", "\n", "# ----------------------------- Autocorrelation Calculation -----------------------------\n", "def compute_acf(signal, max_lag=12):\n", " \"\"\"Compute autocorrelation function.\"\"\"\n", " acf_vals = []\n", " for lag in range(1, min(max_lag, len(signal))):\n", " if lag < len(signal):\n", " corr, _ = pearsonr(signal[:-lag], signal[lag:])\n", " acf_vals.append(corr)\n", " else:\n", " acf_vals.append(0)\n", " return np.array(acf_vals)\n", "\n", "# ----------------------------- Plot Multi-panel Figure -----------------------------\n", "def plot_multi_panel_figure(cyclical_skus, intermittent_skus, outdir):\n", " \"\"\"Create multi-panel figure for cyclical and intermittent SKUs.\"\"\"\n", " print(\"[INFO] Creating multi-panel figures...\")\n", " \n", " # Plot for cyclical SKUs\n", " if not cyclical_skus.empty:\n", " fig, axes = plt.subplots(3, 3, figsize=(18, 15))\n", " \n", " for idx, (_, row) in enumerate(cyclical_skus.head(3).iterrows()): # Only plot top 3\n", " y = row['TimeSeries']\n", " \n", " # Generate signals\n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " # Panel A: Time-series overlay (Raw vs SMA vs SHOS vs SHOS+LGBM)\n", " axes[0, idx].plot(range(len(raw)), raw, label='Raw', marker='o', markersize=4)\n", " axes[0, idx].plot(range(len(sma)), sma, label='SMA (3-month)', linestyle='--')\n", " axes[0, idx].plot(range(len(shos)), shos, label='SHOS', linestyle='-.')\n", " axes[0, idx].plot(range(len(shos_lgbm)), shos_lgbm, label='SHOS+LightGBM', linestyle=':')\n", " axes[0, idx].set_title(f'Cyclical SKU: {row[\"DealerCode\"]}-{row[\"DummyID\"]}')\n", " axes[0, idx].legend()\n", " axes[0, idx].grid(True, alpha=0.3)\n", " \n", " # Panel B: FFT Power Spectrum Overlay\n", " freqs_raw, ps_raw = compute_fft(raw)\n", " freqs_shos, ps_shos = compute_fft(shos)\n", " freqs_shos_lgbm, ps_shos_lgbm = compute_fft(shos_lgbm)\n", " \n", " # Only plot positive frequencies\n", " pos_mask_raw = freqs_raw > 0\n", " pos_mask_shos = freqs_shos > 0\n", " pos_mask_shos_lgbm = freqs_shos_lgbm > 0\n", " \n", " axes[1, idx].plot(freqs_raw[pos_mask_raw], ps_raw[pos_mask_raw], label='Raw FFT', alpha=0.7)\n", " axes[1, idx].plot(freqs_shos[pos_mask_shos], ps_shos[pos_mask_shos], label='SHOS FFT', alpha=0.7)\n", " axes[1, idx].plot(freqs_shos_lgbm[pos_mask_shos_lgbm], ps_shos_lgbm[pos_mask_shos_lgbm], label='SHOS+LGBM FFT', alpha=0.7)\n", " axes[1, idx].set_title('FFT Power Spectrum')\n", " axes[1, idx].set_xlabel('Frequency')\n", " axes[1, idx].set_ylabel('Power')\n", " axes[1, idx].legend()\n", " axes[1, idx].grid(True, alpha=0.3)\n", " \n", " # Panel C: Autocorrelation (ACF) comparison\n", " acf_raw = compute_acf(raw)\n", " acf_shos = compute_acf(shos)\n", " acf_shos_lgbm = compute_acf(shos_lgbm)\n", " \n", " lags = np.arange(1, len(acf_raw)+1)\n", " axes[2, idx].plot(lags, acf_raw, label='Raw ACF', marker='o')\n", " axes[2, idx].plot(lags, acf_shos, label='SHOS ACF', marker='s')\n", " axes[2, idx].plot(lags, acf_shos_lgbm, label='SHOS+LGBM ACF', marker='^')\n", " axes[2, idx].set_title('Autocorrelation Function')\n", " axes[2, idx].set_xlabel('Lag')\n", " axes[2, idx].set_ylabel('Correlation')\n", " axes[2, idx].legend()\n", " axes[2, idx].grid(True, alpha=0.3)\n", " \n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"fig8_cyclical_multi_panel.png\"))\n", " plt.close()\n", " \n", " # Plot for intermittent SKUs\n", " if not intermittent_skus.empty:\n", " fig, axes = plt.subplots(3, 3, figsize=(18, 15))\n", " \n", " for idx, (_, row) in enumerate(intermittent_skus.head(3).iterrows()): # Only plot top 3\n", " y = row['TimeSeries']\n", " \n", " # Generate signals\n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " # Panel A: Time-series overlay (Raw vs SMA vs SHOS vs SHOS+LGBM)\n", " axes[0, idx].plot(range(len(raw)), raw, label='Raw', marker='o', markersize=4)\n", " axes[0, idx].plot(range(len(sma)), sma, label='SMA (3-month)', linestyle='--')\n", " axes[0, idx].plot(range(len(shos)), shos, label='SHOS', linestyle='-.')\n", " axes[0, idx].plot(range(len(shos_lgbm)), shos_lgbm, label='SHOS+LightGBM', linestyle=':')\n", " axes[0, idx].set_title(f'Intermittent SKU: {row[\"DealerCode\"]}-{row[\"DummyID\"]}')\n", " axes[0, idx].legend()\n", " axes[0, idx].grid(True, alpha=0.3)\n", " \n", " # Panel B: FFT Power Spectrum Overlay\n", " freqs_raw, ps_raw = compute_fft(raw)\n", " freqs_shos, ps_shos = compute_fft(shos)\n", " freqs_shos_lgbm, ps_shos_lgbm = compute_fft(shos_lgbm)\n", " \n", " # Only plot positive frequencies\n", " pos_mask_raw = freqs_raw > 0\n", " pos_mask_shos = freqs_shos > 0\n", " pos_mask_shos_lgbm = freqs_shos_lgbm > 0\n", " \n", " axes[1, idx].plot(freqs_raw[pos_mask_raw], ps_raw[pos_mask_raw], label='Raw FFT', alpha=0.7)\n", " axes[1, idx].plot(freqs_shos[pos_mask_shos], ps_shos[pos_mask_shos], label='SHOS FFT', alpha=0.7)\n", " axes[1, idx].plot(freqs_shos_lgbm[pos_mask_shos_lgbm], ps_shos_lgbm[pos_mask_shos_lgbm], label='SHOS+LGBM FFT', alpha=0.7)\n", " axes[1, idx].set_title('FFT Power Spectrum')\n", " axes[1, idx].set_xlabel('Frequency')\n", " axes[1, idx].set_ylabel('Power')\n", " axes[1, idx].legend()\n", " axes[1, idx].grid(True, alpha=0.3)\n", " \n", " # Panel C: Autocorrelation (ACF) comparison\n", " acf_raw = compute_acf(raw)\n", " acf_shos = compute_acf(shos)\n", " acf_shos_lgbm = compute_acf(shos_lgbm)\n", " \n", " lags = np.arange(1, len(acf_raw)+1)\n", " axes[2, idx].plot(lags, acf_raw, label='Raw ACF', marker='o')\n", " axes[2, idx].plot(lags, acf_shos, label='SHOS ACF', marker='s')\n", " axes[2, idx].plot(lags, acf_shos_lgbm, label='SHOS+LGBM ACF', marker='^')\n", " axes[2, idx].set_title('Autocorrelation Function')\n", " axes[2, idx].set_xlabel('Lag')\n", " axes[2, idx].set_ylabel('Correlation')\n", " axes[2, idx].legend()\n", " axes[2, idx].grid(True, alpha=0.3)\n", " \n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"fig9_intermittent_multi_panel.png\"))\n", " plt.close()\n", "\n", "# ----------------------------- Calculate Summary Table -----------------------------\n", "def calculate_summary_table(cyclical_skus, intermittent_skus):\n", " \"\"\"Calculate summary table of signal properties.\"\"\"\n", " print(\"[INFO] Calculating summary table...\")\n", " \n", " results = []\n", " \n", " # Process cyclical SKUs\n", " for _, row in cyclical_skus.iterrows():\n", " y = row['TimeSeries']\n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " # Calculate peak preservation\n", " peak_pres_sma, peak_pres_shos, peak_pres_shos_lgbm = calculate_peak_preservation(raw, sma, shos, shos_lgbm)\n", " \n", " # Calculate phase lag\n", " phase_lag_sma, phase_lag_shos, phase_lag_shos_lgbm = calculate_phase_lag(raw, sma, shos, shos_lgbm)\n", " \n", " # Calculate FFT energy (sum of power spectrum)\n", " _, ps_raw = compute_fft(raw)\n", " _, ps_sma = compute_fft(sma)\n", " _, ps_shos = compute_fft(shos)\n", " _, ps_shos_lgbm = compute_fft(shos_lgbm)\n", " \n", " # Calculate FFT similarity\n", " fft_sim_sma = compute_fft_similarity(raw, sma)\n", " fft_sim_shos = compute_fft_similarity(raw, shos)\n", " fft_sim_shos_lgbm = compute_fft_similarity(raw, shos_lgbm)\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Cyclical',\n", " 'Signal': 'Raw',\n", " 'PeakPres': 1.0, # Reference\n", " 'PhaseLag': 0.0, # Reference\n", " 'FFTEnergy': np.sum(ps_raw),\n", " 'Amplitude': np.std(raw),\n", " 'FFTSimilarity': 1.0 # Reference\n", " })\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Cyclical',\n", " 'Signal': 'SMA',\n", " 'PeakPres': peak_pres_sma,\n", " 'PhaseLag': phase_lag_sma,\n", " 'FFTEnergy': np.sum(ps_sma),\n", " 'Amplitude': np.std(sma),\n", " 'FFTSimilarity': fft_sim_sma\n", " })\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Cyclical',\n", " 'Signal': 'SHOS',\n", " 'PeakPres': peak_pres_shos,\n", " 'PhaseLag': phase_lag_shos,\n", " 'FFTEnergy': np.sum(ps_shos),\n", " 'Amplitude': np.std(shos),\n", " 'FFTSimilarity': fft_sim_shos\n", " })\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Cyclical',\n", " 'Signal': 'SHOS+LightGBM',\n", " 'PeakPres': peak_pres_shos_lgbm,\n", " 'PhaseLag': phase_lag_shos_lgbm,\n", " 'FFTEnergy': np.sum(ps_shos_lgbm),\n", " 'Amplitude': np.std(shos_lgbm),\n", " 'FFTSimilarity': fft_sim_shos_lgbm\n", " })\n", " \n", " # Process intermittent SKUs\n", " for _, row in intermittent_skus.iterrows():\n", " y = row['TimeSeries']\n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " # Calculate peak preservation\n", " peak_pres_sma, peak_pres_shos, peak_pres_shos_lgbm = calculate_peak_preservation(raw, sma, shos, shos_lgbm)\n", " \n", " # Calculate phase lag\n", " phase_lag_sma, phase_lag_shos, phase_lag_shos_lgbm = calculate_phase_lag(raw, sma, shos, shos_lgbm)\n", " \n", " # Calculate FFT energy (sum of power spectrum)\n", " _, ps_raw = compute_fft(raw)\n", " _, ps_sma = compute_fft(sma)\n", " _, ps_shos = compute_fft(shos)\n", " _, ps_shos_lgbm = compute_fft(shos_lgbm)\n", " \n", " # Calculate FFT similarity\n", " fft_sim_sma = compute_fft_similarity(raw, sma)\n", " fft_sim_shos = compute_fft_similarity(raw, shos)\n", " fft_sim_shos_lgbm = compute_fft_similarity(raw, shos_lgbm)\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Intermittent',\n", " 'Signal': 'Raw',\n", " 'PeakPres': 1.0, # Reference\n", " 'PhaseLag': 0.0, # Reference\n", " 'FFTEnergy': np.sum(ps_raw),\n", " 'Amplitude': np.std(raw),\n", " 'FFTSimilarity': 1.0 # Reference\n", " })\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Intermittent',\n", " 'Signal': 'SMA',\n", " 'PeakPres': peak_pres_sma,\n", " 'PhaseLag': phase_lag_sma,\n", " 'FFTEnergy': np.sum(ps_sma),\n", " 'Amplitude': np.std(sma),\n", " 'FFTSimilarity': fft_sim_sma\n", " })\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Intermittent',\n", " 'Signal': 'SHOS',\n", " 'PeakPres': peak_pres_shos,\n", " 'PhaseLag': phase_lag_shos,\n", " 'FFTEnergy': np.sum(ps_shos),\n", " 'Amplitude': np.std(shos),\n", " 'FFTSimilarity': fft_sim_shos\n", " })\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Intermittent',\n", " 'Signal': 'SHOS+LightGBM',\n", " 'PeakPres': peak_pres_shos_lgbm,\n", " 'PhaseLag': phase_lag_shos_lgbm,\n", " 'FFTEnergy': np.sum(ps_shos_lgbm),\n", " 'Amplitude': np.std(shos_lgbm),\n", " 'FFTSimilarity': fft_sim_shos_lgbm\n", " })\n", " \n", " summary_df = pd.DataFrame(results)\n", " return summary_df\n", "\n", "# ----------------------------- Run Theoretical Validation (Fix C: More SKUs) -----------------------------\n", "def run_theoretical_validation(df_features, outdir):\n", " \"\"\"\n", " Run theoretical validation on 50 cyclical and 50 intermittent SKUs.\n", " \"\"\"\n", " print(\"[INFO] Running theoretical validation on 100 SKUs...\")\n", " \n", " # Identify 50 cyclical and 50 intermittent SKUs\n", " cyclical_skus, intermittent_skus = identify_cyclical_intermittent_skus(df_features, n_top=50)\n", " \n", " # Calculate metrics for all SKUs\n", " results = []\n", " \n", " # Process cyclical SKUs\n", " for _, row in cyclical_skus.iterrows():\n", " y = row['TimeSeries']\n", " if len(y) < 10: # Only process SKUs with sufficient data\n", " continue\n", " \n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " # Calculate metrics\n", " peak_pres_sma, peak_pres_shos, peak_pres_shos_lgbm = calculate_peak_preservation(raw, sma, shos, shos_lgbm)\n", " phase_lag_sma, phase_lag_shos, phase_lag_shos_lgbm = calculate_phase_lag(raw, sma, shos, shos_lgbm)\n", " \n", " # Calculate FFT energy\n", " _, ps_raw = compute_fft(raw)\n", " _, ps_sma = compute_fft(sma)\n", " _, ps_shos = compute_fft(shos)\n", " _, ps_shos_lgbm = compute_fft(shos_lgbm)\n", " \n", " # Calculate FFT similarity\n", " fft_sim_sma = compute_fft_similarity(raw, sma)\n", " fft_sim_shos = compute_fft_similarity(raw, shos)\n", " fft_sim_shos_lgbm = compute_fft_similarity(raw, shos_lgbm)\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Cyclical',\n", " 'PeakPres_SMA': peak_pres_sma,\n", " 'PeakPres_SHOS': peak_pres_shos,\n", " 'PeakPres_SHOS_LGBM': peak_pres_shos_lgbm,\n", " 'PhaseLag_SMA': phase_lag_sma,\n", " 'PhaseLag_SHOS': phase_lag_shos,\n", " 'PhaseLag_SHOS_LGBM': phase_lag_shos_lgbm,\n", " 'FFTEnergy_Raw': np.sum(ps_raw),\n", " 'FFTEnergy_SMA': np.sum(ps_sma),\n", " 'FFTEnergy_SHOS': np.sum(ps_shos),\n", " 'FFTEnergy_SHOS_LGBM': np.sum(ps_shos_lgbm),\n", " 'Amplitude_Raw': np.std(raw),\n", " 'Amplitude_SMA': np.std(sma),\n", " 'Amplitude_SHOS': np.std(shos),\n", " 'Amplitude_SHOS_LGBM': np.std(shos_lgbm),\n", " 'FFTSimilarity_SMA': fft_sim_sma,\n", " 'FFTSimilarity_SHOS': fft_sim_shos,\n", " 'FFTSimilarity_SHOS_LGBM': fft_sim_shos_lgbm\n", " })\n", " \n", " # Process intermittent SKUs\n", " for _, row in intermittent_skus.iterrows():\n", " y = row['TimeSeries']\n", " if len(y) < 10: # Only process SKUs with sufficient data\n", " continue\n", " \n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " # Calculate metrics\n", " peak_pres_sma, peak_pres_shos, peak_pres_shos_lgbm = calculate_peak_preservation(raw, sma, shos, shos_lgbm)\n", " phase_lag_sma, phase_lag_shos, phase_lag_shos_lgbm = calculate_phase_lag(raw, sma, shos, shos_lgbm)\n", " \n", " # Calculate FFT energy\n", " _, ps_raw = compute_fft(raw)\n", " _, ps_sma = compute_fft(sma)\n", " _, ps_shos = compute_fft(shos)\n", " _, ps_shos_lgbm = compute_fft(shos_lgbm)\n", " \n", " # Calculate FFT similarity\n", " fft_sim_sma = compute_fft_similarity(raw, sma)\n", " fft_sim_shos = compute_fft_similarity(raw, shos)\n", " fft_sim_shos_lgbm = compute_fft_similarity(raw, shos_lgbm)\n", " \n", " results.append({\n", " 'SKU': f\"{row['DealerCode']}-{row['DummyID']}\",\n", " 'Type': 'Intermittent',\n", " 'PeakPres_SMA': peak_pres_sma,\n", " 'PeakPres_SHOS': peak_pres_shos,\n", " 'PeakPres_SHOS_LGBM': peak_pres_shos_lgbm,\n", " 'PhaseLag_SMA': phase_lag_sma,\n", " 'PhaseLag_SHOS': phase_lag_shos,\n", " 'PhaseLag_SHOS_LGBM': phase_lag_shos_lgbm,\n", " 'FFTEnergy_Raw': np.sum(ps_raw),\n", " 'FFTEnergy_SMA': np.sum(ps_sma),\n", " 'FFTEnergy_SHOS': np.sum(ps_shos),\n", " 'FFTEnergy_SHOS_LGBM': np.sum(ps_shos_lgbm),\n", " 'Amplitude_Raw': np.std(raw),\n", " 'Amplitude_SMA': np.std(sma),\n", " 'Amplitude_SHOS': np.std(shos),\n", " 'Amplitude_SHOS_LGBM': np.std(shos_lgbm),\n", " 'FFTSimilarity_SMA': fft_sim_sma,\n", " 'FFTSimilarity_SHOS': fft_sim_shos,\n", " 'FFTSimilarity_SHOS_LGBM': fft_sim_shos_lgbm\n", " })\n", " \n", " # Create evidence table (mean of all SKUs)\n", " evidence_df = pd.DataFrame(results)\n", " evidence_summary = evidence_df.groupby('Type').agg({\n", " 'PeakPres_SMA': 'mean',\n", " 'PeakPres_SHOS': 'mean',\n", " 'PeakPres_SHOS_LGBM': 'mean',\n", " 'PhaseLag_SMA': 'mean',\n", " 'PhaseLag_SHOS': 'mean',\n", " 'PhaseLag_SHOS_LGBM': 'mean',\n", " 'FFTEnergy_Raw': 'mean',\n", " 'FFTEnergy_SMA': 'mean',\n", " 'FFTEnergy_SHOS': 'mean',\n", " 'FFTEnergy_SHOS_LGBM': 'mean',\n", " 'Amplitude_Raw': 'mean',\n", " 'Amplitude_SMA': 'mean',\n", " 'Amplitude_SHOS': 'mean',\n", " 'Amplitude_SHOS_LGBM': 'mean',\n", " 'FFTSimilarity_SMA': 'mean',\n", " 'FFTSimilarity_SHOS': 'mean',\n", " 'FFTSimilarity_SHOS_LGBM': 'mean'\n", " }).round(4)\n", " \n", " # Save evidence table\n", " evidence_df.to_csv(os.path.join(outdir, \"evidence_table_detailed.csv\"), index=False)\n", " evidence_summary.to_csv(os.path.join(outdir, \"evidence_table_summary.csv\"))\n", " \n", " print(\"[INFO] Evidence table saved to evidence_table_summary.csv\")\n", " print(evidence_summary)\n", " \n", " # Generate 3 golden example plots\n", " if not cyclical_skus.empty and len(cyclical_skus) >= 3:\n", " golden_cyclical = cyclical_skus.head(3)\n", " golden_intermittent = intermittent_skus.head(3) if not intermittent_skus.empty and len(intermittent_skus) >= 3 else intermittent_skus.head(len(intermittent_skus))\n", " \n", " # Plot golden examples\n", " fig, axes = plt.subplots(2, 3, figsize=(18, 10))\n", " \n", " # Cyclical examples\n", " for idx, (_, row) in enumerate(golden_cyclical.iterrows()):\n", " if idx >= 3:\n", " break\n", " y = row['TimeSeries']\n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " axes[0, idx].plot(range(len(raw)), raw, label='Raw', marker='o', markersize=4)\n", " axes[0, idx].plot(range(len(sma)), sma, label='SMA', linestyle='--')\n", " axes[0, idx].plot(range(len(shos)), shos, label='SHOS', linestyle='-.')\n", " axes[0, idx].plot(range(len(shos_lgbm)), shos_lgbm, label='SHOS+LGBM', linestyle=':')\n", " axes[0, idx].set_title(f'Cyclical: {row[\"DealerCode\"]}-{row[\"DummyID\"]}')\n", " axes[0, idx].legend()\n", " axes[0, idx].grid(True, alpha=0.3)\n", " \n", " # Intermittent examples\n", " for idx, (_, row) in enumerate(golden_intermittent.iterrows()):\n", " if idx >= 3:\n", " break\n", " y = row['TimeSeries']\n", " raw, sma, shos, shos_lgbm = generate_signals(y)\n", " \n", " axes[1, idx].plot(range(len(raw)), raw, label='Raw', marker='o', markersize=4)\n", " axes[1, idx].plot(range(len(sma)), sma, label='SMA', linestyle='--')\n", " axes[1, idx].plot(range(len(shos)), shos, label='SHOS', linestyle='-.')\n", " axes[1, idx].plot(range(len(shos_lgbm)), shos_lgbm, label='SHOS+LGBM', linestyle=':')\n", " axes[1, idx].set_title(f'Intermittent: {row[\"DealerCode\"]}-{row[\"DummyID\"]}')\n", " axes[1, idx].legend()\n", " axes[1, idx].grid(True, alpha=0.3)\n", " \n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"golden_examples.png\"))\n", " plt.close()\n", " \n", " return evidence_df, evidence_summary\n", "\n", "# ----------------------------- Main Pipeline (Adaptive Filter Proof) -----------------------------\n", "def run_adaptive_filter_proof_pipeline(df_filtered, outdir=\"./result_adaptive_filter\", shos_k=0.1, shos_w=0.7):\n", " print(\"[INFO] Starting adaptive filter proof pipeline...\")\n", " start_time = time.time()\n", " safe_mkdir(outdir)\n", " safe_mkdir(os.path.join(outdir, \"figures\")) # Ensure figures dir exists\n", " print(\"[INFO] 1. Aggregating and padding data...\")\n", " df_ag = aggregate_monthly(df_filtered)\n", " df_padded = pad_monthly_series_and_filter_inactive(df_ag)\n", " print(\"[INFO] 2. Computing intermittency metrics...\")\n", " intermittency_df = compute_intermittency(df_padded)\n", " intermittency_df.to_csv(os.path.join(outdir, \"intermittency_stats.csv\"), index=False)\n", " print(\"[INFO] 3. Building feature matrix once...\")\n", " df_features, base_cols, full_cols = build_feature_matrix_optimized(df_padded, shos_k=shos_k) # Pass shos_k for SHOS calc\n", " print(\"[INFO] 4. Running theoretical validation...\")\n", " evidence_df, evidence_summary = run_theoretical_validation(df_features, outdir)\n", " print(\"[INFO] 5. Identifying cyclical and intermittent SKUs...\")\n", " cyclical_skus, intermittent_skus = identify_cyclical_intermittent_skus(df_features, n_top=3) # For figures, keep 3\n", " print(\"[INFO] 6. Creating multi-panel figures...\")\n", " plot_multi_panel_figure(cyclical_skus, intermittent_skus, outdir)\n", " print(\"[INFO] 7. Calculating summary table...\")\n", " summary_df = calculate_summary_table(cyclical_skus, intermittent_skus)\n", " summary_df.to_csv(os.path.join(outdir, \"signal_analysis_summary.csv\"), index=False)\n", " print(\"[INFO] 8. Generating summary plots...\")\n", " out_fig = os.path.join(outdir, \"figures\")\n", " # Plot peak preservation comparison\n", " fig, axes = plt.subplots(1, 2, figsize=(14, 6))\n", " \n", " # Peak preservation by signal type\n", " for sig_type in ['SMA', 'SHOS', 'SHOS+LightGBM']:\n", " subset = summary_df[summary_df['Signal'] == sig_type]\n", " if not subset.empty:\n", " axes[0].scatter(subset.index, subset['PeakPres'], label=sig_type, alpha=0.7)\n", " axes[0].set_title('Peak Preservation by Signal Type')\n", " axes[0].set_xlabel('SKU Index')\n", " axes[0].set_ylabel('Peak Preservation Ratio (vs Raw)')\n", " axes[0].legend()\n", " axes[0].grid(True, alpha=0.3)\n", " \n", " # FFT Energy by signal type\n", " for sig_type in ['SMA', 'SHOS', 'SHOS+LightGBM']:\n", " subset = summary_df[summary_df['Signal'] == sig_type]\n", " if not subset.empty:\n", " axes[1].scatter(subset.index, subset['FFTEnergy'], label=sig_type, alpha=0.7)\n", " axes[1].set_title('FFT Energy by Signal Type')\n", " axes[1].set_xlabel('SKU Index')\n", " axes[1].set_ylabel('FFT Energy (Sum of Power Spectrum)')\n", " axes[1].legend()\n", " axes[1].grid(True, alpha=0.3)\n", " \n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig10_signal_comparison.png\"))\n", " plt.close()\n", " print(f\"[INFO] Adaptive filter proof pipeline completed in {time.time() - start_time:.2f} seconds. Results saved to: {outdir}\")\n", " # Print summary\n", " print(\"--- SIGNAL ANALYSIS SUMMARY ---\")\n", " print(\"Signal Properties (Amplitude and Frequency Energy):\")\n", " print(summary_df[['SKU', 'Type', 'Signal', 'PeakPres', 'PhaseLag', 'FFTEnergy', 'Amplitude', 'FFTSimilarity']].head(20).round(4))\n", " print(\"--- EVIDENCE TABLE SUMMARY (100 SKUs) ---\")\n", " print(evidence_summary)\n", " print(f\"Median ADI: {intermittency_df['ADI'].median():.2f}\")\n", " print(f\"Median CV²: {intermittency_df['CV2'].median():.2f}\")\n", " return summary_df, evidence_df, evidence_summary\n", "\n", "# ----------------------------- Entry Point -----------------------------\n", "if __name__ == \"__main__\":\n", " # Example usage:\n", " # df_filtered = pd.read_csv(\"your_data.csv\")\n", " run_adaptive_filter_proof_pipeline(df_filtered, outdir=\"./result_adaptive_filter1\", shos_k=0.1, shos_w=0.7) # Provide your best k, w\n", " pass" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- DEBUGGING REPORT ---\n", "1. Current Executable: c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\python.exe\n", "2. Is '1.venv' in Path? False\n", "3. Sklearn Location: c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\__init__.py\n", "4. TensorFlow Location: c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tensorflow\\__init__.py\n", "\n", "--- SYSTEM PATHS ---\n", "c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\python310.zip\n", "c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\DLLs\n", "c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\n", "c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\n", "\n", "C:\\Users\\pmarv\\AppData\\Roaming\\Python\\Python310\\site-packages\n", "c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\n" ] } ], "source": [ "import sys\n", "import os\n", "\n", "print(\"--- DEBUGGING REPORT ---\")\n", "# 1. Check which Python the Notebook is actually using\n", "print(f\"1. Current Executable: {sys.executable}\")\n", "\n", "# 2. Check if the path includes your virtual environment\n", "print(f\"2. Is '1.venv' in Path? {'1.venv' in sys.executable}\")\n", "\n", "# 3. Try to import and catch the specific error\n", "try:\n", " import sklearn\n", " print(f\"3. Sklearn Location: {sklearn.__file__}\")\n", "except ImportError as e:\n", " print(f\"3. ERROR: Sklearn not found in this kernel! {e}\")\n", "\n", "try:\n", " import tensorflow as tf\n", " print(f\"4. TensorFlow Location: {tf.__file__}\")\n", "except ImportError as e:\n", " print(f\"4. ERROR: TensorFlow not found in this kernel! {e}\")\n", "\n", "# 4. List all available paths (to see where Pylance is looking)\n", "print(\"\\n--- SYSTEM PATHS ---\")\n", "for path in sys.path:\n", " print(path)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- 🚀 SUCCESS REPORT ---\n", "TF Version: 2.20.0\n", "Keras Version: 3.12.0\n", "Keras Layers: Successfully resolved.\n" ] } ], "source": [ "try:\n", " import tensorflow as tf\n", " import keras\n", " from keras import layers\n", " \n", " # Test a simple model creation\n", " test_model = keras.Sequential([layers.Dense(10, input_shape=(5,))])\n", " \n", " print(\"--- 🚀 SUCCESS REPORT ---\")\n", " print(f\"TF Version: {tf.__version__}\")\n", " print(f\"Keras Version: {keras.__version__}\")\n", " print(\"Keras Layers: Successfully resolved.\")\n", "except Exception as e:\n", " print(\"--- ❌ STILL FAILING ---\")\n", " print(f\"Error: {e}\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: You are using pip version 21.2.3; however, version 25.3 is available.\n", "You should consider upgrading via the 'c:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\python.exe -m pip install --upgrade pip' command.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: tensorflow in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (2.20.0)Note: you may need to restart the kernel to use updated packages.\n", "✅ Keras version 3.12.0 is active and working!\n", "[INFO] TensorFlow is available.\n", "[INFO] SHAP is available.\n", "[INFO] Starting test pipeline...\n", "\n", "Requirement already satisfied: libclang>=13.0.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (18.1.1)\n", "Requirement already satisfied: setuptools in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (57.4.0)\n", "Requirement already satisfied: ml_dtypes<1.0.0,>=0.5.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (0.5.4)\n", "Requirement already satisfied: h5py>=3.11.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (3.15.1)\n", "Requirement already satisfied: numpy>=1.26.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (2.2.6)\n", "Requirement already satisfied: wrapt>=1.11.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (2.0.1)\n", "Requirement already satisfied: google_pasta>=0.1.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (0.2.0)\n", "Requirement already satisfied: grpcio<2.0,>=1.24.3 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (1.76.0)\n", "Requirement already satisfied: termcolor>=1.1.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (3.3.0)\n", "Requirement already satisfied: requests<3,>=2.21.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (2.32.5)\n", "Requirement already satisfied: typing_extensions>=3.6.6 in c:\\users\\pmarv\\appdata\\roaming\\python\\python310\\site-packages (from tensorflow) (4.15.0)\n", "Requirement already satisfied: keras>=3.10.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (3.12.0)\n", "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (0.7.0)\n", "Requirement already satisfied: packaging in c:\\users\\pmarv\\appdata\\roaming\\python\\python310\\site-packages (from tensorflow) (25.0)\n", "Requirement already satisfied: astunparse>=1.6.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (1.6.3)\n", "Requirement already satisfied: protobuf>=5.28.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (6.33.3)\n", "Requirement already satisfied: absl-py>=1.0.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (2.3.1)\n", "Requirement already satisfied: tensorboard~=2.20.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (2.20.0)\n", "Requirement already satisfied: flatbuffers>=24.3.25 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (25.12.19)\n", "Requirement already satisfied: six>=1.12.0 in c:\\users\\pmarv\\appdata\\roaming\\python\\python310\\site-packages (from tensorflow) (1.17.0)\n", "Requirement already satisfied: opt_einsum>=2.3.2 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorflow) (3.4.0)\n", "Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from astunparse>=1.6.0->tensorflow) (0.45.1)\n", "Requirement already satisfied: namex in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from keras>=3.10.0->tensorflow) (0.1.0)\n", "Requirement already satisfied: optree in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from keras>=3.10.0->tensorflow) (0.18.0)\n", "Requirement already satisfied: rich in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from keras>=3.10.0->tensorflow) (14.2.0)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow) (3.4.4)\n", "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow) (3.11)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow) (2.6.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow) (2026.1.4)\n", "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorboard~=2.20.0->tensorflow) (3.1.5)\n", "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorboard~=2.20.0->tensorflow) (0.7.2)\n", "Requirement already satisfied: pillow in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorboard~=2.20.0->tensorflow) (12.1.0)\n", "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tensorboard~=2.20.0->tensorflow) (3.10)\n", "Requirement already satisfied: markupsafe>=2.1.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard~=2.20.0->tensorflow) (3.0.3)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\pmarv\\appdata\\roaming\\python\\python310\\site-packages (from rich->keras>=3.10.0->tensorflow) (2.19.2)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from rich->keras>=3.10.0->tensorflow) (4.0.0)\n", "Requirement already satisfied: mdurl~=0.1 in c:\\users\\pmarv\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.10.0->tensorflow) (0.1.2)\n", "[INFO] 1. Aggregating and padding data...\n", "[INFO] Aggregating data to monthly level...\n", "[INFO] Aggregated to 171111 monthly records.\n", "[INFO] Padding series to monthly grid and filtering inactive pairs...\n", "[INFO] Date range: 2023-08 to 2025-08. Total months: 25\n", "[INFO] Filtered from 171111 to 170827 records after removing inactive pairs.\n", "[INFO] Padding complete. Final shape: (1404700, 9)\n", "[INFO] 2. Computing intermittency metrics...\n", "[INFO] Computing intermittency metrics (ADI, CV²)...\n", "[INFO] Computed metrics for 56188 active series. Median ADI: 25.00, Median CV²: 24.00\n", "[INFO] 3. Building feature matrix once...\n", "[INFO] Building feature matrix ONCE for all folds...\n", "[INFO] Adding time series features (max_lag=6)...\n", "[INFO] Added 12 lag features.\n", "[INFO] Adding Croston features...\n", "[INFO] Added Croston features for 56188 unique series.\n", "[INFO] Adding SHOS features with train-only sparsity (k=0.1)...\n", "[INFO] Added SHOS features for 56188 unique series.\n", "[INFO] Building low-rank embeddings (n_components=12)...\n", "[INFO] Created 24 embedding features for 59 dealers and 7702 parts.\n", "[INFO] Final feature matrix has 51 columns and shape (1404700, 59).\n", "[INFO] 4. Running main cross-validation with optimized features...\n", "[INFO] Starting optimized cross-validation with 12-month train, 3-month test windows...\n", "[INFO] Using SHOS k=0.1, w=0.7\n", "[INFO] Total possible folds: 11\n", "[INFO] Processing Fold 1: Train 2023-08 to 2024-07, Test 2024-08 to 2024-10\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[8], line 956\u001b[0m\n\u001b[0;32m 952\u001b[0m \u001b[38;5;66;03m# ----------------------------- Entry Point -----------------------------\u001b[39;00m\n\u001b[0;32m 953\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m 954\u001b[0m \u001b[38;5;66;03m# Example usage:\u001b[39;00m\n\u001b[0;32m 955\u001b[0m \u001b[38;5;66;03m# df_filtered = pd.read_csv(\"your_data.csv\")\u001b[39;00m\n\u001b[1;32m--> 956\u001b[0m \u001b[43mrun_test_pipeline\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf_filtered\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutdir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./DL\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshos_k\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshos_w\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.7\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Provide your best k, w\u001b[39;00m\n\u001b[0;32m 957\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n", "Cell \u001b[1;32mIn[8], line 845\u001b[0m, in \u001b[0;36mrun_test_pipeline\u001b[1;34m(df_filtered, outdir, shos_k, shos_w)\u001b[0m\n\u001b[0;32m 843\u001b[0m df_features, base_cols, full_cols \u001b[38;5;241m=\u001b[39m build_feature_matrix_optimized(df_padded, shos_k\u001b[38;5;241m=\u001b[39mshos_k) \u001b[38;5;66;03m# Pass shos_k for SHOS calc\u001b[39;00m\n\u001b[0;32m 844\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[INFO] 4. Running main cross-validation with optimized features...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 845\u001b[0m cv_results, train_results, overfitting_df, fold1_actual, fold1_preds \u001b[38;5;241m=\u001b[39m \u001b[43mrun_cross_validation_optimized\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 846\u001b[0m \u001b[43m \u001b[49m\u001b[43mdf_features\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbase_cols\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfull_cols\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshos_k\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshos_k\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshos_w\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshos_w\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Pass pre-computed features and k, w\u001b[39;49;00m\n\u001b[0;32m 847\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 848\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[INFO] 5. Saving results...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 849\u001b[0m test_scores_df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(cv_results)\u001b[38;5;241m.\u001b[39mT\n", "Cell \u001b[1;32mIn[8], line 582\u001b[0m, in \u001b[0;36mrun_cross_validation_optimized\u001b[1;34m(df_processed, base_cols, full_cols, train_window, test_window, shos_k, shos_w)\u001b[0m\n\u001b[0;32m 580\u001b[0m croston_preds_list \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m 581\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m (d, p), test_g \u001b[38;5;129;01min\u001b[39;00m test_df_fold\u001b[38;5;241m.\u001b[39mgroupby([\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDealerCode\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDummyID\u001b[39m\u001b[38;5;124m\"\u001b[39m]):\n\u001b[1;32m--> 582\u001b[0m train_g \u001b[38;5;241m=\u001b[39m train_df_fold[\u001b[43m(\u001b[49m\u001b[43mtrain_df_fold\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mDealerCode\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43md\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m&\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_df_fold\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mDummyID\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m]\n\u001b[0;32m 583\u001b[0m train_y \u001b[38;5;241m=\u001b[39m train_g[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPartQty\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalues \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m train_g\u001b[38;5;241m.\u001b[39mempty \u001b[38;5;28;01melse\u001b[39;00m np\u001b[38;5;241m.\u001b[39marray([])\n\u001b[0;32m 584\u001b[0m test_len \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(test_g)\n", "File \u001b[1;32mc:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\ops\\common.py:76\u001b[0m, in \u001b[0;36m_unpack_zerodim_and_defer..new_method\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m 72\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mNotImplemented\u001b[39m\n\u001b[0;32m 74\u001b[0m other \u001b[38;5;241m=\u001b[39m item_from_zerodim(other)\n\u001b[1;32m---> 76\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mother\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[1;32mc:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\arraylike.py:70\u001b[0m, in \u001b[0;36mOpsMixin.__and__\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[38;5;129m@unpack_zerodim_and_defer\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__and__\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 69\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__and__\u001b[39m(\u001b[38;5;28mself\u001b[39m, other):\n\u001b[1;32m---> 70\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_logical_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mother\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moperator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mand_\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[1;32mc:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\series.py:6149\u001b[0m, in \u001b[0;36mSeries._logical_method\u001b[1;34m(self, other, op)\u001b[0m\n\u001b[0;32m 6146\u001b[0m lvalues \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_values\n\u001b[0;32m 6147\u001b[0m rvalues \u001b[38;5;241m=\u001b[39m extract_array(other, extract_numpy\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, extract_range\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m-> 6149\u001b[0m res_values \u001b[38;5;241m=\u001b[39m \u001b[43mops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlogical_op\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 6150\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_construct_result(res_values, name\u001b[38;5;241m=\u001b[39mres_name)\n", "File \u001b[1;32mc:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\ops\\array_ops.py:448\u001b[0m, in \u001b[0;36mlogical_op\u001b[1;34m(left, right, op)\u001b[0m\n\u001b[0;32m 446\u001b[0m is_other_int_dtype \u001b[38;5;241m=\u001b[39m rvalues\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39mkind \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miu\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 447\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_other_int_dtype:\n\u001b[1;32m--> 448\u001b[0m rvalues \u001b[38;5;241m=\u001b[39m \u001b[43mfill_bool\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvalues\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 450\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 451\u001b[0m \u001b[38;5;66;03m# i.e. scalar\u001b[39;00m\n\u001b[0;32m 452\u001b[0m is_other_int_dtype \u001b[38;5;241m=\u001b[39m lib\u001b[38;5;241m.\u001b[39mis_integer(rvalues)\n", "File \u001b[1;32mc:\\Users\\pmarv\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\ops\\array_ops.py:420\u001b[0m, in \u001b[0;36mlogical_op..fill_bool\u001b[1;34m(x, left)\u001b[0m\n\u001b[0;32m 417\u001b[0m x[mask] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m 419\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m left \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m left\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39mkind \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 420\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[43mx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mastype\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mbool\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m x\n", "\u001b[1;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "%pip install tensorflow\n", "import os\n", "import math\n", "import pickle\n", "import warnings\n", "import time\n", "from typing import Optional, Dict, Any, Tuple, List\n", "warnings.filterwarnings(\"ignore\")\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.decomposition import TruncatedSVD\n", "from sklearn.calibration import CalibratedClassifierCV\n", "from sklearn.linear_model import ElasticNet, Ridge, LogisticRegression\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.model_selection import TimeSeriesSplit\n", "from sklearn.metrics import (\n", " mean_squared_error, mean_absolute_error, r2_score\n", ")\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "# Deep Learning Libraries\n", "try:\n", " import tensorflow as tf\n", " # Instead of: from tensorflow.keras.models import Sequential\n", "# Use this:\n", " import keras\n", " from keras.models import Sequential\n", " from keras.layers import LSTM, Dense, Dropout, Input, Conv1D, MaxPooling1D, Flatten, GlobalAveragePooling1D\n", " from keras.layers import MultiHeadAttention, LayerNormalization, Add\n", " from keras.optimizers import Adam\n", " from keras.callbacks import EarlyStopping\n", "\n", " print(f\"✅ Keras version {keras.__version__} is active and working!\")\n", "\n", " DL_AVAILABLE = True\n", " print(\"[INFO] TensorFlow is available.\")\n", "except ImportError:\n", " DL_AVAILABLE = False\n", " print(\"[INFO] TensorFlow is NOT available.\")\n", "\n", "# Optional SHAP library\n", "try:\n", " import shap\n", " SHAP_AVAILABLE = True\n", " print(\"[INFO] SHAP is available.\")\n", "except ImportError:\n", " SHAP_AVAILABLE = False\n", " print(\"[INFO] SHAP is NOT available.\")\n", "\n", "# ----------------------------- Utilities -----------------------------\n", "def safe_mkdir(path: str):\n", " os.makedirs(path, exist_ok=True)\n", "\n", "def rmse(y_true, y_pred):\n", " return math.sqrt(mean_squared_error(y_true, y_pred))\n", "\n", "def wmape(y_true, y_pred):\n", " denom = np.sum(np.abs(y_true))\n", " return np.sum(np.abs(y_true - y_pred)) / denom * 100.0 if denom != 0 else np.nan\n", "\n", "def mape(y_true, y_pred):\n", " mask = y_true != 0\n", " return np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask])) * 100.0 if mask.sum() > 0 else np.nan\n", "\n", "# ----------------------------- Padding with Inactive Filtering (Moved to Top) -----------------------------\n", "def pad_monthly_series_and_filter_inactive(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Padding series to monthly grid and filtering inactive pairs...\")\n", " df = df.copy()\n", " df[\"MonthStart\"] = pd.to_datetime(df[\"MonthStart\"])\n", " min_date = df[\"MonthStart\"].min()\n", " max_date = df[\"MonthStart\"].max()\n", " all_months = pd.date_range(start=min_date, end=max_date, freq=\"MS\")\n", " print(f\"[INFO] Date range: {min_date.strftime('%Y-%m')} to {max_date.strftime('%Y-%m')}. Total months: {len(all_months)}\")\n", " df_active = df.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0)\n", " print(f\"[INFO] Filtered from {df.shape[0]} to {df_active.shape[0]} records after removing inactive pairs.\")\n", " if df_active.empty:\n", " print(\"[WARNING] No active series found after filtering.\")\n", " return df_active\n", " full_index = pd.MultiIndex.from_product(\n", " [df_active[\"DealerCode\"].unique(), df_active[\"DummyID\"].unique(), all_months],\n", " names=[\"DealerCode\", \"DummyID\", \"MonthStart\"]\n", " )\n", " df_indexed = df_active.set_index([\"DealerCode\", \"DummyID\", \"MonthStart\"])\n", " df_padded = df_indexed.reindex(full_index, fill_value=0).reset_index()\n", " df_padded[\"OTC_Flag\"] = (df_padded[\"OTC_Qty\"] > 0).astype(int)\n", " for col in [\"PartQty\", \"OTC_Qty\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]:\n", " df_padded[col] = pd.to_numeric(df_padded[col], errors=\"coerce\").fillna(0)\n", " df_final = df_padded.groupby([\"DealerCode\", \"DummyID\"]).filter(lambda g: g[\"PartQty\"].sum() > 0).reset_index(drop=True)\n", " print(f\"[INFO] Padding complete. Final shape: {df_final.shape}\")\n", " return df_final\n", "\n", "# ----------------------------- Data helpers (Same as before) -----------------------------\n", "def aggregate_monthly(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Aggregating data to monthly level...\")\n", " df = df.copy()\n", " colmap = {}\n", " for c in df.columns:\n", " lc = c.lower().strip()\n", " if lc in (\"dealer code\", \"dealercode\", \"dealer_code\", \"dealercode\"):\n", " colmap[c] = \"DealerCode\"\n", " if lc in (\"dummy id\", \"dummyid\", \"partno\", \"part_no\", \"dummy_id\"):\n", " colmap[c] = \"DummyID\"\n", " if lc in (\"sale date\", \"saledate\", \"sale_date\"):\n", " colmap[c] = \"SaleDate\"\n", " if lc in (\"part qty.\", \"part qty\", \"partqty\", \"qty\", \"quantity\"):\n", " colmap[c] = \"PartQty\"\n", " if lc in (\"sale type\", \"saletype\", \"sale_type\"):\n", " colmap[c] = \"SaleType\"\n", " if lc in (\"dealer price\", \"dealerprice\", \"price\"):\n", " colmap[c] = \"DealerPrice\"\n", " if lc in (\"prc_unit_weight\", \"weight\", \"prc unit weight\"):\n", " colmap[c] = \"PRC_UNIT_WEIGHT\"\n", " if lc in (\"prc_unit_cube\", \"cube\", \"prc unit cube\"):\n", " colmap[c] = \"PRC_UNIT_CUBE\"\n", " if lc in (\"otc_qty\",\"otc qty\",\"otcqty\"):\n", " colmap[c] = \"OTC_Qty\"\n", " df = df.rename(columns=colmap)\n", " if \"MonthStart\" in df.columns and \"PartQty\" in df.columns:\n", " out = df.copy()\n", " out[\"MonthStart\"] = pd.to_datetime(out[\"MonthStart\"])\n", " if \"OTC_Qty\" not in out.columns:\n", " if \"SaleType\" in out.columns:\n", " out[\"OTC_Qty\"] = out[\"PartQty\"] * 0\n", " mask = out[\"SaleType\"].astype(str).str.upper() == \"OTC\"\n", " out.loc[mask, \"OTC_Qty\"] = out.loc[mask, \"PartQty\"]\n", " else:\n", " out[\"OTC_Qty\"] = 0\n", " out[\"OTC_Flag\"] = (out[\"OTC_Qty\"] > 0).astype(int)\n", " out[\"DealerPrice\"] = out.get(\"DealerPrice\", 0.0)\n", " out[\"PRC_UNIT_WEIGHT\"] = out.get(\"PRC_UNIT_WEIGHT\", 0.0)\n", " out[\"PRC_UNIT_CUBE\"] = out.get(\"PRC_UNIT_CUBE\", 0.0)\n", " return out[[\"DealerCode\", \"DummyID\", \"MonthStart\", \"PartQty\", \"OTC_Qty\", \"OTC_Flag\", \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]]\n", " if \"SaleDate\" not in df.columns:\n", " raise ValueError(\"Input must have 'SaleDate' or 'MonthStart' + 'PartQty'.\")\n", " df[\"SaleDate\"] = pd.to_datetime(df[\"SaleDate\"])\n", " df[\"MonthStart\"] = df[\"SaleDate\"].dt.to_period(\"M\").dt.to_timestamp()\n", " agg = df.groupby([\"DealerCode\", \"DummyID\", \"MonthStart\"], as_index=False).agg(\n", " PartQty=(\"PartQty\", \"sum\"),\n", " OTC_Qty=(\"PartQty\", lambda x: x[df.loc[x.index, \"SaleType\"].astype(str).str.upper() == \"OTC\"].sum() if \"SaleType\" in df.columns else 0),\n", " DealerPrice=(\"DealerPrice\", \"mean\"),\n", " PRC_UNIT_WEIGHT=(\"PRC_UNIT_WEIGHT\", \"mean\"),\n", " PRC_UNIT_CUBE=(\"PRC_UNIT_CUBE\", \"mean\"),\n", " )\n", " agg[\"OTC_Qty\"] = agg[\"OTC_Qty\"].fillna(0).astype(int)\n", " agg[\"OTC_Flag\"] = (agg[\"OTC_Qty\"] > 0).astype(int)\n", " agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]] = agg[[\"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\"]].fillna(0.0)\n", " print(f\"[INFO] Aggregated to {len(agg)} monthly records.\")\n", " return agg\n", "\n", "def add_time_series_features(df: pd.DataFrame, max_lag: int = 12) -> pd.DataFrame:\n", " print(f\"[INFO] Adding time series features (max_lag={max_lag})...\")\n", " df = df.sort_values([\"DealerCode\", \"DummyID\", \"MonthStart\"]).copy()\n", " rows = []\n", " grp = df.groupby([\"DealerCode\", \"DummyID\"])\n", " for (d, p), g in grp:\n", " g = g.sort_values(\"MonthStart\").copy()\n", " for lag in range(1, max_lag+1):\n", " g[f\"qty_lag_{lag}\"] = g[\"PartQty\"].shift(lag).fillna(0)\n", " g[f\"otc_lag_{lag}\"] = g[\"OTC_Flag\"].shift(lag).fillna(0)\n", " g[\"qty_sum_1_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"qty_sum_1_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).sum().shift(1).fillna(0)\n", " g[\"nonzero_count_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).apply(lambda x: np.sum(x>0)).shift(1).fillna(0)\n", " last_pos = -1\n", " t_since = []\n", " for idx, val in enumerate(g[\"PartQty\"].values):\n", " if val > 0:\n", " last_pos = idx\n", " t_since.append(0)\n", " else:\n", " t_since.append(idx - last_pos if last_pos != -1 else g.shape[0]+1)\n", " g[\"time_since_last_sale\"] = pd.Series(t_since).astype(float)\n", " g[\"qty_ma_3\"] = g[\"PartQty\"].rolling(3, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"qty_ma_6\"] = g[\"PartQty\"].rolling(6, min_periods=1).mean().shift(1).fillna(0)\n", " g[\"otc_share_slope_3\"] = g[\"OTC_Flag\"].rolling(3, min_periods=1).apply(lambda x: np.polyfit(np.arange(len(x)), x, 1)[0] if len(x)>1 else 0).shift(1).fillna(0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added {len([c for c in out.columns if c.startswith(('qty_lag_', 'otc_lag_'))])} lag features.\")\n", " return out\n", "\n", "def croston_forecast(series: np.ndarray, alpha: float = 0.3) -> np.ndarray:\n", " n = len(series)\n", " if n == 0:\n", " return np.zeros(0)\n", " demand = np.asarray(series, dtype=float)\n", " forecast = np.zeros(n)\n", " nz_idx = np.where(demand > 0)[0]\n", " if len(nz_idx) == 0:\n", " return forecast\n", " last = nz_idx[0]\n", " z = demand[last]\n", " p = 1.0\n", " for t in range(last+1, n):\n", " if demand[t] > 0:\n", " interval = t - last\n", " z = alpha * demand[t] + (1-alpha) * z\n", " p = alpha * interval + (1-alpha) * p\n", " last = t\n", " forecast[t] = z / max(1e-9, p)\n", " forecast[:last+1] = forecast[last+1] if last+1 < n else forecast[last]\n", " return forecast\n", "\n", "def add_croston_features(df: pd.DataFrame) -> pd.DataFrame:\n", " print(\"[INFO] Adding Croston features...\")\n", " rows = []\n", " for (d,p), g in df.groupby([\"DealerCode\",\"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " g[\"croston_forecast\"] = croston_forecast(g[\"PartQty\"].values, alpha=0.3)\n", " g[\"croston_ratio\"] = np.where(g[\"qty_sum_1_3\"]!=0, g[\"croston_forecast\"]/g[\"qty_sum_1_3\"], 0.0)\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True)\n", " out.fillna(0, inplace=True)\n", " print(f\"[INFO] Added Croston features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- SHOS with Train-Only Sparsity (Same as before) -----------------------------\n", "def compute_shos_for_series(y: np.ndarray, alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5,\n", " n_train: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n", " n = len(y)\n", " if n == 0:\n", " return np.zeros(0), np.zeros(0), np.zeros(0)\n", " if n_train is not None and n_train > 0:\n", " y_sparsity = y[:n_train]\n", " else:\n", " y_sparsity = y\n", " nonzero_count = np.sum(y_sparsity > 0)\n", " sparsity = 1.0 - (nonzero_count / max(1, len(y_sparsity)))\n", " if adaptivity:\n", " alpha = alpha_base * (1.0 - sparsity_k * sparsity)\n", " beta = beta_base * (1.0 - sparsity_k * sparsity)\n", " last_pos_all = np.where(y > 0)[0]\n", " if len(last_pos_all) > 0 and (n - 1 - last_pos_all[-1]) <= 3:\n", " alpha = min(0.9, alpha * 1.2)\n", " beta = min(0.9, beta * 1.2)\n", " else:\n", " alpha, beta = alpha_base, beta_base\n", " # --- IMPROVED INITIALIZATION (Point 6) ---\n", " if n >= 3:\n", " init_window = 3\n", " y_init_window = y[:init_window]\n", " q0 = float(np.sum(y_init_window > 0)) / max(1, init_window)\n", " positive_y_init = y_init_window[y_init_window > 0]\n", " if len(positive_y_init) > 0:\n", " z0 = float(np.mean(positive_y_init))\n", " else:\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " else:\n", " q0 = float(np.sum(y > 0)) / max(1, n)\n", " positive_y_all = y[y > 0]\n", " if len(positive_y_all) > 0:\n", " z0 = float(np.mean(positive_y_all))\n", " else:\n", " z0 = 1.0\n", " # --- END IMPROVED INITIALIZATION ---\n", " q_prev, z_prev = q0, z0\n", " q, z, f = np.zeros(n), np.zeros(n), np.zeros(n)\n", " for t in range(n):\n", " yt = y[t]\n", " if yt > 0:\n", " z_curr = alpha * yt + (1 - alpha) * z_prev\n", " q_curr = beta * 1.0 + (1 - beta) * q_prev\n", " else:\n", " z_curr = z_prev\n", " q_curr = beta * 0.0 + (1 - beta) * q_prev\n", " q[t], z[t], f[t] = q_curr, z_curr, q_curr * z_curr\n", " q_prev, z_prev = q_curr, z_curr\n", " return q, z, f\n", "\n", "def add_shos_features_with_ntrain(df: pd.DataFrame, n_train_per_series: Dict[Tuple, int],\n", " alpha_base: float = 0.2, beta_base: float = 0.2,\n", " adaptivity: bool = True, sparsity_k: float = 0.5) -> pd.DataFrame:\n", " print(f\"[INFO] Adding SHOS features with train-only sparsity (k={sparsity_k})...\")\n", " rows = []\n", " for (d, p), g in df.groupby([\"DealerCode\", \"DummyID\"]):\n", " g = g.sort_values(\"MonthStart\").copy()\n", " y = g[\"PartQty\"].values.astype(float)\n", " n_train = n_train_per_series.get((d, p), len(y))\n", " q, z, f = compute_shos_for_series(y, alpha_base, beta_base, adaptivity, sparsity_k, n_train)\n", " g[\"q_shos\"] = q\n", " g[\"z_shos\"] = z\n", " g[\"shos_forecast\"] = f\n", " rows.append(g)\n", " out = pd.concat(rows, axis=0).reset_index(drop=True).fillna(0)\n", " print(f\"[INFO] Added SHOS features for {len(rows)} unique series.\")\n", " return out\n", "\n", "# ----------------------------- Embeddings (Same as before) -----------------------------\n", "def build_low_rank_embeddings(df: pd.DataFrame, n_components: int = 16):\n", " print(f\"[INFO] Building low-rank embeddings (n_components={n_components})...\")\n", " occ = df.groupby([\"DealerCode\", \"DummyID\"]).agg(months_active=(\"PartQty\", lambda x: (x>0).sum())).reset_index()\n", " pivot = occ.pivot(index=\"DealerCode\", columns=\"DummyID\", values=\"months_active\").fillna(0)\n", " if pivot.shape[0] < 2 or pivot.shape[1] < 2:\n", " print(\"[INFO] Pivot too small for SVD, creating empty embeddings.\")\n", " dealer_embed = pd.DataFrame({\"DealerCode\": sorted(df[\"DealerCode\"].unique())})\n", " part_embed = pd.DataFrame({\"DummyID\": sorted(df[\"DummyID\"].unique())})\n", " return dealer_embed, part_embed, []\n", " k = min(n_components, min(pivot.shape)-1)\n", " svd = TruncatedSVD(n_components=k, random_state=0)\n", " U = svd.fit_transform(pivot)\n", " V = svd.components_.T\n", " dealer_index = list(pivot.index)\n", " part_index = list(pivot.columns)\n", " dealer_embed = pd.DataFrame(U, index=dealer_index).reset_index().rename(columns={\"index\":\"DealerCode\"})\n", " part_embed = pd.DataFrame(V, index=part_index).reset_index().rename(columns={\"index\":\"DummyID\"})\n", " emb_cols = []\n", " for i in range(U.shape[1]):\n", " dealer_embed.rename(columns={i: f\"emb_dealer_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_dealer_{i}\")\n", " for i in range(V.shape[1]):\n", " part_embed.rename(columns={i: f\"emb_part_{i}\"}, inplace=True)\n", " emb_cols.append(f\"emb_part_{i}\")\n", " print(f\"[INFO] Created {len(emb_cols)} embedding features for {U.shape[0]} dealers and {V.shape[0]} parts.\")\n", " return dealer_embed, part_embed, emb_cols\n", "\n", "# ----------------------------- Feature Sets (NO CROSTON in ML Baselines, YES in Ablation) -----------------------------\n", "def build_base_features(df: pd.DataFrame, max_lag: int = 6):\n", " print(\"[INFO] Building base feature matrix...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " # Croston features are NOT added here for fairness in ML baselines\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " print(f\"[INFO] Base features include {len(base_cols)} columns.\")\n", " return df2, base_cols\n", "\n", "def build_fold_features(train_df: pd.DataFrame, test_df: pd.DataFrame, max_lag: int = 6, svd_components: int = 12, sparsity_k: float = 0.5):\n", " print(f\"[INFO] Building fold features with sparsity_k={sparsity_k}...\")\n", " train_base, base_cols = build_base_features(train_df, max_lag)\n", " test_base, _ = build_base_features(test_df, max_lag)\n", " # Add Croston features to both train and test folds BEFORE SHOS\n", " train_croston = add_croston_features(train_base)\n", " test_croston = add_croston_features(test_base)\n", " # Pass the Croston-augmented base to SHOS calculation\n", " n_train_map = train_df.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " train_shos = add_shos_features_with_ntrain(train_croston, n_train_map, sparsity_k=sparsity_k) # Pass train_croston\n", " test_shos = add_shos_features_with_ntrain(test_croston, n_train_map, sparsity_k=sparsity_k) # Pass test_croston\n", " # Calculate embeddings based on the SHOS-augmented training data\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(train_shos, svd_components)\n", " if emb_cols:\n", " train_shos = train_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " train_shos = train_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " test_shos = test_shos.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " test_shos = test_shos.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " train_shos[c] = train_shos[c].fillna(0)\n", " test_shos[c] = test_shos[c].fillna(0)\n", " # Define feature lists\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Fold features: {len(base_cols)} base, {len(emb_cols)} embeddings, {len(full_cols)} total.\")\n", " return train_shos, test_shos, base_cols, full_cols\n", "\n", "# ----------------------------- Intermittency Analysis (Same as before) -----------------------------\n", "def compute_intermittency(df_processed):\n", " print(\"[INFO] Computing intermittency metrics (ADI, CV²)...\")\n", " stats = []\n", " for (d, p), g in df_processed.groupby([\"DealerCode\", \"DummyID\"]):\n", " y = g[\"PartQty\"].values\n", " if np.sum(y > 0) == 0:\n", " continue\n", " intervals = np.diff(np.where(y > 0)[0])\n", " adi = np.mean(intervals) if len(intervals) > 0 else len(y)\n", " mean_y = np.mean(y)\n", " cv2 = (np.std(y) / mean_y) ** 2 if mean_y > 0 else np.inf\n", " stats.append({\"ADI\": adi, \"CV2\": cv2})\n", " stats_df = pd.DataFrame(stats)\n", " print(f\"[INFO] Computed metrics for {len(stats_df)} active series. Median ADI: {stats_df['ADI'].median():.2f}, Median CV²: {stats_df['CV2'].median():.2f}\")\n", " return stats_df\n", "\n", "# ----------------------------- Deep Learning Model Creation Functions -----------------------------\n", "def create_lstm_model(input_shape, n_outputs=1):\n", " model = Sequential([\n", " Input(shape=input_shape),\n", " LSTM(50, return_sequences=True),\n", " Dropout(0.2),\n", " LSTM(50, return_sequences=False),\n", " Dropout(0.2),\n", " Dense(25, activation='relu'),\n", " Dense(n_outputs)\n", " ])\n", " model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')\n", " return model\n", "\n", "def create_tcn_model(input_shape, n_outputs=1):\n", " model = Sequential([\n", " Input(shape=input_shape),\n", " # First Block\n", " Conv1D(filters=64, kernel_size=2, activation='relu', padding='causal'),\n", " Conv1D(filters=64, kernel_size=2, activation='relu', padding='causal'),\n", " # Only one pooling layer for short sequences (length 3 or 6)\n", " MaxPooling1D(pool_size=2) if input_shape[0] >= 2 else Flatten(),\n", " \n", " # Second Block - Removed second MaxPooling\n", " Conv1D(filters=128, kernel_size=2, activation='relu', padding='causal'),\n", " GlobalAveragePooling1D(),\n", " \n", " Dense(50, activation='relu'),\n", " Dropout(0.2),\n", " Dense(n_outputs)\n", " ])\n", " model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')\n", " return model\n", "\n", "def create_transformer_model(input_shape, n_outputs=1, n_heads=4, ff_dim=64):\n", " inputs = Input(shape=input_shape)\n", " \n", " # Temporal convolution to extract local patterns\n", " x = Conv1D(filters=64, kernel_size=3, activation='relu', padding='same')(inputs)\n", " \n", " # Multi-head attention layer\n", " attention_output = MultiHeadAttention(num_heads=n_heads, key_dim=x.shape[-1])(x, x)\n", " attention_output = Dropout(0.1)(attention_output)\n", " add_norm_output = Add()([x, attention_output])\n", " norm_output = LayerNormalization()(add_norm_output)\n", " \n", " # Feed-forward layer\n", " ff_output = Dense(ff_dim, activation='relu')(norm_output)\n", " ff_output = Dropout(0.1)(ff_output)\n", " ff_output = Dense(x.shape[-1])(ff_output)\n", " \n", " # Add & Norm\n", " x = Add()([norm_output, ff_output])\n", " x = LayerNormalization()(x)\n", " \n", " # Global pooling and dense layers\n", " x = GlobalAveragePooling1D()(x)\n", " x = Dense(50, activation='relu')(x)\n", " outputs = Dense(n_outputs)(x)\n", " \n", " model = tf.keras.Model(inputs=inputs, outputs=outputs)\n", " model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')\n", " return model\n", "\n", "def prepare_sequences(X, y, sequence_length=6):\n", " \"\"\"Convert tabular data into sequences for deep learning models.\"\"\"\n", " X_seq, y_seq = [], []\n", " for i in range(len(X) - sequence_length + 1):\n", " X_seq.append(X[i:i+sequence_length])\n", " y_seq.append(y[i+sequence_length-1])\n", " return np.array(X_seq), np.array(y_seq)\n", "\n", "# ----------------------------- Train Deep Learning Models -----------------------------\n", "def train_dl_model(model, X_train, y_train, X_val, y_val, epochs=50, patience=5):\n", " \"\"\"Train a deep learning model with early stopping.\"\"\"\n", " if not DL_AVAILABLE:\n", " print(\"[WARNING] TensorFlow not available, skipping deep learning model training.\")\n", " return None\n", " \n", " early_stop = EarlyStopping(monitor='val_loss', patience=patience, restore_best_weights=True)\n", " \n", " history = model.fit(\n", " X_train, y_train,\n", " validation_data=(X_val, y_val),\n", " batch_size=1024,\n", " epochs=epochs,\n", " callbacks=[early_stop],\n", " verbose=1\n", " )\n", " return model\n", "\n", "def predict_dl_model(model, X_test, sequence_length=6):\n", " \"\"\"Make predictions using a trained deep learning model.\"\"\"\n", " if not DL_AVAILABLE or model is None:\n", " print(\"[WARNING] TensorFlow not available or model is None, returning zeros.\")\n", " # Estimate number of predictions needed\n", " num_predictions = len(X_test) - sequence_length + 1\n", " if num_predictions <= 0:\n", " num_predictions = 1\n", " return np.zeros(num_predictions)\n", " \n", " X_test_seq, _ = prepare_sequences(X_test, np.zeros(len(X_test)), sequence_length)\n", " if len(X_test_seq) == 0:\n", " # If not enough data to form a sequence, return zeros\n", " return np.zeros(len(X_test))\n", " \n", " predictions = model.predict(X_test_seq, verbose=0).flatten()\n", " # If we have more original points than predictions, pad with the last value\n", " if len(predictions) < len(X_test):\n", " padding = np.full(len(X_test) - len(predictions), predictions[-1] if len(predictions) > 0 else 0)\n", " predictions = np.concatenate([predictions, padding])\n", " return predictions\n", "\n", "# ----------------------------- Build Feature Matrix Once (Optimized for CV) -----------------------------\n", "def build_feature_matrix_optimized(df: pd.DataFrame, max_lag=6, svd_components=12, shos_k=0.5):\n", " print(\"[INFO] Building feature matrix ONCE for all folds...\")\n", " df2 = add_time_series_features(df, max_lag=max_lag)\n", " df2 = add_croston_features(df2)\n", " # Calculate n_train_map for the *entire* dataset (will be used later if needed per fold, but we calculate SHOS once here)\n", " # For this optimized version, we calculate SHOS once on the full series, using the provided k.\n", " # This means the sparsity used for alpha/beta is based on the *full* series length for each (d, p).\n", " # This is a trade-off for speed. For strict train-only sparsity per fold, calculation must happen inside the loop.\n", " # Using full-series sparsity is often a reasonable approximation.\n", " n_full_map = df2.groupby([\"DealerCode\", \"DummyID\"]).size().to_dict()\n", " df2 = add_shos_features_with_ntrain(df2, n_full_map, sparsity_k=shos_k) # Calculate SHOS once using full series length\n", " df2.fillna(0, inplace=True)\n", " # Define baseline features (excluding Croston/SHOS for fairness in base ML models)\n", " lag_cols = [c for c in df2.columns if c.startswith((\"qty_lag_\", \"otc_lag_\"))]\n", " base_cols = [\n", " \"DealerPrice\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_WEIGHT\", \"PRC_UNIT_CUBE\",\n", " \"qty_sum_1_3\", \"qty_sum_1_6\", \"nonzero_count_6\", \"time_since_last_sale\",\n", " \"qty_ma_3\", \"qty_ma_6\", \"otc_share_slope_3\"\n", " ] + lag_cols\n", " dealer_stats = df2.groupby(\"DealerCode\").agg(dealer_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " part_stats = df2.groupby(\"DummyID\").agg(part_qty_mean=(\"PartQty\", \"mean\")).reset_index()\n", " df2 = df2.merge(dealer_stats, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_stats, on=\"DummyID\", how=\"left\")\n", " extra_cols = [\"dealer_qty_mean\", \"part_qty_mean\"]\n", " base_cols += extra_cols\n", " # Calculate embeddings based on the full processed dataset (including SHOS features for context)\n", " dealer_embed, part_embed, emb_cols = build_low_rank_embeddings(df2, n_components=svd_components)\n", " if emb_cols:\n", " df2 = df2.merge(dealer_embed, on=\"DealerCode\", how=\"left\")\n", " df2 = df2.merge(part_embed, on=\"DummyID\", how=\"left\")\n", " for c in emb_cols:\n", " df2[c] = df2[c].fillna(0)\n", " full_cols = base_cols + [\"q_shos\", \"z_shos\"] + emb_cols\n", " print(f\"[INFO] Final feature matrix has {len(full_cols)} columns and shape {df2.shape}.\")\n", " return df2, base_cols, full_cols\n", "\n", "# ----------------------------- Cross-validation with Optimized Features (Deep Learning Models) -----------------------------\n", "def run_cross_validation_optimized(df_processed: pd.DataFrame, base_cols: List[str], full_cols: List[str], train_window=12, test_window=3, shos_k=0.1, shos_w=0.7):\n", " print(f\"[INFO] Starting optimized cross-validation with {train_window}-month train, {test_window}-month test windows...\")\n", " print(f\"[INFO] Using SHOS k={shos_k}, w={shos_w}\")\n", " df_processed = df_processed.sort_values([\"MonthStart\", \"DealerCode\", \"DummyID\"]).reset_index(drop=True)\n", " all_months = sorted(df_processed[\"MonthStart\"].unique())\n", " # Define models to evaluate (only the 6 specified deep learning models)\n", " model_names = [\"LSTM\", \"TCN\", \"Transformer\", \"SHOS + LSTM\", \"SHOS + TCN\", \"SHOS + Transformer\"]\n", " ablation_names = [] # No ablation models for deep learning\n", " all_names = model_names + ablation_names + [\"Croston\", \"TSB\"]\n", " metrics = {name: {\"MAE\": [], \"RMSE\": [], \"WMAPE\": [], \"MAPE\": [], \"R2\": []} for name in all_names}\n", " fold_train_metrics = {name: {\"MAE\": [], \"RMSE\": [], \"WMAPE\": [], \"MAPE\": [], \"R2\": []} for name in all_names} # NEW: Store train metrics\n", " fold1_actual, fold1_preds = None, {}\n", " total_folds = len(all_months) - train_window - test_window + 1\n", " print(f\"[INFO] Total possible folds: {total_folds}\")\n", " for i in range(total_folds):\n", " train_months = all_months[i:i + train_window]\n", " test_months = all_months[i + train_window:i + train_window + test_window]\n", " # --- OPTIMIZATION: Use pre-computed features and slice them ---\n", " train_df_fold = df_processed[df_processed[\"MonthStart\"].isin(train_months)].copy()\n", " test_df_fold = df_processed[df_processed[\"MonthStart\"].isin(test_months)].copy()\n", " if train_df_fold.empty or test_df_fold.empty:\n", " print(f\"[INFO] Skipping fold {i+1} due to empty train/test set.\")\n", " continue\n", " print(f\"[INFO] Processing Fold {i+1}: Train {train_months[0].strftime('%Y-%m')} to {train_months[-1].strftime('%Y-%m')}, Test {test_months[0].strftime('%Y-%m')} to {test_months[-1].strftime('%Y-%m')}\")\n", " # Slice pre-computed features\n", " train_f = df_processed[df_processed[\"MonthStart\"].isin(train_months)].copy()\n", " test_f = df_processed[df_processed[\"MonthStart\"].isin(test_months)].copy()\n", " # Ensure features are aligned\n", " X_train_base = train_f[base_cols].fillna(0).values\n", " X_test_base = test_f[base_cols].fillna(0).values\n", " X_train_full = train_f[full_cols].fillna(0).values\n", " X_test_full = test_f[full_cols].fillna(0).values\n", " y_train_actual = train_f[\"PartQty\"].values\n", " y_test = test_f[\"PartQty\"].values\n", " # --- MODEL TRAINING AND PREDICTION ---\n", " model_predictions = {}\n", " model_test_scores = {}\n", " model_train_scores = {} # NEW: Store train scores for this fold\n", " \n", " # Baselines: Croston & TSB (predictions are based on training logic, so calculated per fold)\n", " def compute_croston_on_train_for_test(train_series, test_len):\n", " full_series = np.concatenate([train_series, np.zeros(test_len)])\n", " forecast = croston_forecast(full_series, alpha=0.3)\n", " return forecast[-test_len:]\n", " \n", " croston_preds_list = []\n", " for (d, p), test_g in test_df_fold.groupby([\"DealerCode\", \"DummyID\"]):\n", " train_g = train_df_fold[(train_df_fold[\"DealerCode\"]==d) & (train_df_fold[\"DummyID\"]==p)]\n", " train_y = train_g[\"PartQty\"].values if not train_g.empty else np.array([])\n", " test_len = len(test_g)\n", " if len(train_y) > 0:\n", " croston_for_test = compute_croston_on_train_for_test(train_y, test_len)\n", " else:\n", " croston_for_test = np.zeros(test_len)\n", " croston_preds_list.append(croston_for_test)\n", " croston_pred = np.concatenate(croston_preds_list)\n", " tsb_pred = test_f[\"shos_forecast\"].values # This is the pre-computed SHOS forecast for the test period\n", " model_predictions[\"Croston\"] = croston_pred\n", " model_predictions[\"TSB\"] = tsb_pred\n", " \n", " # Calculate TEST scores for baselines\n", " for name in [\"Croston\", \"TSB\"]:\n", " y_pred = model_predictions[name]\n", " try:\n", " test_mae = mean_absolute_error(y_test, y_pred)\n", " test_rmse = rmse(y_test, y_pred)\n", " test_wmape = wmape(y_test, y_pred)\n", " test_mape = mape(y_test, y_pred)\n", " test_r2 = r2_score(y_test, y_pred)\n", " model_test_scores[name] = {\n", " \"MAE\": test_mae,\n", " \"RMSE\": test_rmse,\n", " \"WMAPE\": test_wmape,\n", " \"MAPE\": test_mape,\n", " \"R2\": test_r2\n", " }\n", " # Baselines don't have a meaningful train prediction in this context for *these specific metrics*\n", " # So we assign NaN for train metrics for Croston/TSB.\n", " model_train_scores[name] = {k: np.nan for k in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]}\n", " except Exception as e:\n", " print(f\"[WARNING] Error calculating test metrics for {name}: {e}\")\n", " model_test_scores[name] = {k: np.nan for k in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]}\n", " model_train_scores[name] = {k: np.nan for k in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]}\n", " \n", " # --- DEEP LEARNING MODELS ---\n", " sequence_length = 6\n", " # Prepare sequences for deep learning models\n", " X_train_base_seq, y_train_seq = prepare_sequences(X_train_base, y_train_actual, sequence_length)\n", " X_test_base_seq, _ = prepare_sequences(X_test_base, y_test, sequence_length)\n", " X_train_full_seq, _ = prepare_sequences(X_train_full, y_train_actual, sequence_length)\n", " X_test_full_seq, _ = prepare_sequences(X_test_full, y_test, sequence_length)\n", " \n", " # Split train data for validation (last 20%)\n", " val_split_idx = int(0.8 * len(X_train_base_seq))\n", " X_train_seq_base = X_train_base_seq[:val_split_idx]\n", " X_val_seq_base = X_train_base_seq[val_split_idx:]\n", " y_train_seq_split = y_train_seq[:val_split_idx]\n", " y_val_seq = y_train_seq[val_split_idx:]\n", " \n", " val_split_idx_full = int(0.8 * len(X_train_full_seq))\n", " X_train_seq_full = X_train_full_seq[:val_split_idx_full]\n", " X_val_seq_full = X_train_full_seq[val_split_idx_full:]\n", " y_train_seq_split_full = y_train_actual[sequence_length-1:len(X_train_seq_full)+sequence_length-1]\n", " y_val_seq_full = y_train_actual[len(X_train_seq_full)+sequence_length-1:len(X_train_seq_full)+len(X_val_seq_full)+sequence_length-1]\n", " \n", " # Create and train each model\n", " for name in model_names:\n", " try:\n", " if name == \"LSTM\":\n", " model = create_lstm_model(input_shape=(sequence_length, X_train_base.shape[1]))\n", " model = train_dl_model(model, X_train_seq_base, y_train_seq_split, X_val_seq_base, y_val_seq)\n", " y_pred_test = predict_dl_model(model, X_test_base, sequence_length)\n", " # Predict on train\n", " y_pred_train = predict_dl_model(model, X_train_base, sequence_length)\n", " \n", " elif name == \"TCN\":\n", " model = create_tcn_model(input_shape=(sequence_length, X_train_base.shape[1]))\n", " model = train_dl_model(model, X_train_seq_base, y_train_seq_split, X_val_seq_base, y_val_seq)\n", " y_pred_test = predict_dl_model(model, X_test_base, sequence_length)\n", " # Predict on train\n", " y_pred_train = predict_dl_model(model, X_train_base, sequence_length)\n", " \n", " elif name == \"Transformer\":\n", " model = create_transformer_model(input_shape=(sequence_length, X_train_base.shape[1]))\n", " model = train_dl_model(model, X_train_seq_base, y_train_seq_split, X_val_seq_base, y_val_seq)\n", " y_pred_test = predict_dl_model(model, X_test_base, sequence_length)\n", " # Predict on train\n", " y_pred_train = predict_dl_model(model, X_train_base, sequence_length)\n", " \n", " elif name == \"SHOS + LSTM\":\n", " model = create_lstm_model(input_shape=(sequence_length, X_train_full.shape[1]))\n", " model = train_dl_model(model, X_train_seq_full, y_train_seq_split_full, X_val_seq_full, y_val_seq_full)\n", " y_pred_test = predict_dl_model(model, X_test_full, sequence_length)\n", " # Predict on train\n", " y_pred_train = predict_dl_model(model, X_train_full, sequence_length)\n", " \n", " elif name == \"SHOS + TCN\":\n", " model = create_tcn_model(input_shape=(sequence_length, X_train_full.shape[1]))\n", " model = train_dl_model(model, X_train_seq_full, y_train_seq_split_full, X_val_seq_full, y_val_seq_full)\n", " y_pred_test = predict_dl_model(model, X_test_full, sequence_length)\n", " # Predict on train\n", " y_pred_train = predict_dl_model(model, X_train_full, sequence_length)\n", " \n", " elif name == \"SHOS + Transformer\":\n", " model = create_transformer_model(input_shape=(sequence_length, X_train_full.shape[1]))\n", " model = train_dl_model(model, X_train_seq_full, y_train_seq_split_full, X_val_seq_full, y_val_seq_full)\n", " y_pred_test = predict_dl_model(model, X_test_full, sequence_length)\n", " # Predict on train\n", " y_pred_train = predict_dl_model(model, X_train_full, sequence_length)\n", " \n", " # Adjust predictions to match test length\n", " if len(y_pred_test) > len(y_test):\n", " y_pred_test = y_pred_test[:len(y_test)]\n", " elif len(y_pred_test) < len(y_test):\n", " # Pad with last value\n", " padding = np.full(len(y_test) - len(y_pred_test), y_pred_test[-1] if len(y_pred_test) > 0 else 0)\n", " y_pred_test = np.concatenate([y_pred_test, padding])\n", " \n", " if len(y_pred_train) > len(y_train_actual):\n", " y_pred_train = y_pred_train[:len(y_train_actual)]\n", " elif len(y_pred_train) < len(y_train_actual):\n", " # Pad with last value\n", " padding = np.full(len(y_train_actual) - len(y_pred_train), y_pred_train[-1] if len(y_pred_train) > 0 else 0)\n", " y_pred_train = np.concatenate([y_pred_train, padding])\n", " \n", " # Calculate TEST Metrics\n", " test_mae = mean_absolute_error(y_test, y_pred_test)\n", " test_rmse = rmse(y_test, y_pred_test)\n", " test_wmape = wmape(y_test, y_pred_test)\n", " test_mape = mape(y_test, y_pred_test)\n", " test_r2 = r2_score(y_test, y_pred_test)\n", " \n", " # Calculate TRAIN Metrics\n", " train_mae = mean_absolute_error(y_train_actual, y_pred_train)\n", " train_rmse = rmse(y_train_actual, y_pred_train)\n", " train_wmape = wmape(y_train_actual, y_pred_train)\n", " train_mape = mape(y_train_actual, y_pred_train)\n", " train_r2 = r2_score(y_train_actual, y_pred_train)\n", " \n", " model_predictions[name] = y_pred_test\n", " model_test_scores[name] = {\n", " \"MAE\": test_mae,\n", " \"RMSE\": test_rmse,\n", " \"WMAPE\": test_wmape,\n", " \"MAPE\": test_mape,\n", " \"R2\": test_r2\n", " }\n", " model_train_scores[name] = { # Store train scores\n", " \"MAE\": train_mae,\n", " \"RMSE\": train_rmse,\n", " \"WMAPE\": train_wmape,\n", " \"MAPE\": train_mape,\n", " \"R2\": train_r2\n", " }\n", " except Exception as e:\n", " print(f\"[WARNING] Error training/predicting {name}: {e}\")\n", " model_predictions[name] = np.zeros_like(y_test)\n", " for metric_name in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]:\n", " model_test_scores[name][metric_name] = np.nan\n", " model_train_scores[name][metric_name] = np.nan # Store NaN for train too\n", " \n", " # --- STORE METRICS FOR THIS FOLD ---\n", " for name in all_names:\n", " for metric_name in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]:\n", " # Append the scores calculated for this fold\n", " metrics[name][metric_name].append(model_test_scores[name][metric_name])\n", " fold_train_metrics[name][metric_name].append(model_train_scores[name][metric_name]) # Append train scores\n", " \n", " # Store first fold results for plotting\n", " if fold1_actual is None:\n", " fold1_actual = y_test\n", " fold1_preds = {\n", " \"Croston\": croston_pred,\n", " \"TSB\": tsb_pred,\n", " \"LSTM\": model_predictions.get(\"LSTM\", np.zeros_like(y_test)),\n", " \"SHOS + LSTM\": model_predictions.get(\"SHOS + LSTM\", np.zeros_like(y_test)),\n", " # Standard Deep Learning\n", " \"TCN\": model_predictions.get(\"TCN\", np.zeros_like(y_test)),\n", " \"Transformer\": model_predictions.get(\"Transformer\", np.zeros_like(y_test)),\n", " \n", " # SHOS-Enhanced Deep Learning\n", " \"SHOS + LSTM\": model_predictions.get(\"SHOS + LSTM\", np.zeros_like(y_test)),\n", " \"SHOS + TCN\": model_predictions.get(\"SHOS + TCN\", np.zeros_like(y_test)),\n", " \"SHOS + Transformer\": model_predictions.get(\"SHOS + Transformer\", np.zeros_like(y_test))\n", " }\n", "\n", " \n", " valid_folds = 0\n", " # find a model name and metric list to count non-nan entries (safe method)\n", " any_model = next(iter(metrics))\n", " valid_folds = int(np.nanmax([len([x for x in metrics[m]['MAE'] if not (x is None or (isinstance(x, float) and np.isnan(x)))]) for m in metrics]) if metrics else 0)\n", " print(f\"[INFO] Cross-validation completed. Estimated valid folds per model (approx): {valid_folds}\")\n", " results = {}\n", " train_results = {} # NEW: Store aggregated train results\n", " for name in all_names: # Iterate over all model names\n", " results[name] = {}\n", " train_results[name] = {} # NEW: Initialize train results dict\n", " for metric in [\"MAE\", \"RMSE\", \"WMAPE\", \"MAPE\", \"R2\"]: # Iterate over metric names\n", " test_arr = np.array(metrics[name][metric])\n", " train_arr = np.array(fold_train_metrics[name][metric]) # NEW: Get train array\n", " results[name][f\"{metric}_mean\"] = np.nanmean(test_arr)\n", " results[name][f\"{metric}_std\"] = np.nanstd(test_arr)\n", " train_results[name][f\"{metric}_mean\"] = np.nanmean(train_arr) # NEW: Calculate train mean\n", " train_results[name][f\"{metric}_std\"] = np.nanstd(train_arr) # NEW: Calculate train std\n", " \n", " # --- OVERFITTING DIAGNOSIS ---\n", " # --- AGGREGATE RESULTS INTO DATAFRAMES (safe conversion) ---\n", " results_df = pd.DataFrame(results).T if isinstance(results, dict) else pd.DataFrame(results)\n", " train_df = pd.DataFrame(train_results).T if isinstance(train_results, dict) else pd.DataFrame(train_results)\n", " # Ensure columns follow the \"_mean\" pattern\n", " required_cols = [\"MAE_mean\",\"MAE_std\",\"RMSE_mean\",\"RMSE_std\",\"WMAPE_mean\",\"WMAPE_std\",\"MAPE_mean\",\"MAPE_std\",\"R2_mean\",\"R2_std\"]\n", " # If some are missing, fill with NaN\n", " for c in required_cols:\n", " if c not in results_df.columns:\n", " results_df[c] = np.nan\n", " if c not in train_df.columns:\n", " train_df[c] = np.nan\n", " # Overfitting diagnostics: Test - Train\n", " overfitting_df = pd.DataFrame(index=results_df.index)\n", " overfitting_df[\"Test_MAE\"] = results_df[\"MAE_mean\"]\n", " overfitting_df[\"Train_MAE\"] = train_df[\"MAE_mean\"]\n", " overfitting_df[\"Diff_MAE\"] = overfitting_df[\"Test_MAE\"] - overfitting_df[\"Train_MAE\"]\n", " overfitting_df[\"Test_R2\"] = results_df[\"R2_mean\"]\n", " overfitting_df[\"Train_R2\"] = train_df[\"R2_mean\"]\n", " overfitting_df[\"Diff_R2\"] = overfitting_df[\"Test_R2\"] - overfitting_df[\"Train_R2\"]\n", " # Print\n", " print(\"--- OVERFITTING DIAGNOSIS ---\")\n", " print(overfitting_df.round(6))\n", " print(\"Interpretation: Positive Diff_MAE or negative Diff_R2 indicates possible overfitting (Test worse than Train).\")\n", " return results_df.to_dict(orient=\"index\"), train_df.to_dict(orient=\"index\"), overfitting_df, fold1_actual, fold1_preds\n", "\n", "# ----------------------------- SHAP Analysis (Point 4) -----------------------------\n", "def run_shap_analysis(model, X_test, feature_names, outdir):\n", " if not SHAP_AVAILABLE:\n", " print(\"[WARNING] SHAP not available, skipping SHAP analysis.\")\n", " return\n", " if model is None:\n", " print(\"[WARNING] Model object is None, skipping SHAP analysis.\")\n", " return\n", " print(f\"[INFO] Calculating SHAP values for model...\")\n", " try:\n", " explainer = shap.TreeExplainer(model)\n", " # Use a sample for speed if X_test is large\n", " sample_size = min(1000, X_test.shape[0])\n", " X_test_sample = X_test[:sample_size]\n", " shap_values = explainer.shap_values(X_test_sample)\n", " shap.summary_plot(shap_values, X_test_sample, feature_names=feature_names, show=False, plot_size=(10, 8))\n", " plt.title(\"SHAP Feature Importance (Top 10)\")\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(outdir, \"figures\", \"fig6_shap.png\"))\n", " plt.close()\n", " print(f\"[INFO] SHAP plot saved to {os.path.join(outdir, 'figures', 'fig6_shap.png')}\")\n", " except Exception as e:\n", " print(f\"[WARNING] Could not generate SHAP plot: {e}\")\n", "\n", "# ----------------------------- Main Pipeline (Test Run) -----------------------------\n", "def run_test_pipeline(df_filtered, outdir=\"./result_test\", shos_k=0.1, shos_w=0.7):\n", " print(\"[INFO] Starting test pipeline...\")\n", " start_time = time.time()\n", " safe_mkdir(outdir)\n", " safe_mkdir(os.path.join(outdir, \"figures\")) # Ensure figures dir exists\n", " print(\"[INFO] 1. Aggregating and padding data...\")\n", " df_ag = aggregate_monthly(df_filtered)\n", " df_padded = pad_monthly_series_and_filter_inactive(df_ag)\n", " print(\"[INFO] 2. Computing intermittency metrics...\")\n", " intermittency_df = compute_intermittency(df_padded)\n", " intermittency_df.to_csv(os.path.join(outdir, \"intermittency_stats.csv\"), index=False)\n", " print(\"[INFO] 3. Building feature matrix once...\")\n", " df_features, base_cols, full_cols = build_feature_matrix_optimized(df_padded, shos_k=shos_k) # Pass shos_k for SHOS calc\n", " print(\"[INFO] 4. Running main cross-validation with optimized features...\")\n", " cv_results, train_results, overfitting_df, fold1_actual, fold1_preds = run_cross_validation_optimized(\n", " df_features, base_cols, full_cols, shos_k=shos_k, shos_w=shos_w # Pass pre-computed features and k, w\n", " )\n", " print(\"[INFO] 5. Saving results...\")\n", " test_scores_df = pd.DataFrame(cv_results).T\n", " test_scores_df.to_csv(os.path.join(outdir, \"test_results.csv\"))\n", " train_scores_df = pd.DataFrame(train_results).T # NEW: Save train results\n", " train_scores_df.to_csv(os.path.join(outdir, \"train_results.csv\"))\n", " overfitting_df.to_csv(os.path.join(outdir, \"overfitting_diagnosis.csv\")) # Save overfitting results\n", " print(\" --- TEST RESULTS (Mean ± Std) --- \")\n", " print(test_scores_df.round(4))\n", " print(\" --- TRAIN RESULTS (Mean ± Std) --- \") # NEW: Print train results\n", " print(train_scores_df.round(4))\n", " print(\" --- OVERFITTING DIAGNOSIS (Test - Train) --- \")\n", " print(overfitting_df.round(4))\n", " print(\"[INFO] 6. Generating figures...\")\n", " out_fig = os.path.join(outdir, \"figures\")\n", " safe_mkdir(out_fig)\n", " # Filter models for plotting (exclude TSB/Croston if too many, include deep learning models)\n", " plot_models = [m for m in test_scores_df.index if m not in [\"TSB\", \"Croston\"]] # Or include them if desired\n", " test_mae_means = [test_scores_df.loc[m, \"MAE_mean\"] for m in plot_models]\n", " test_mae_stds = [test_scores_df.loc[m, \"MAE_std\"] for m in plot_models]\n", " test_rmse_means = [test_scores_df.loc[m, \"RMSE_mean\"] for m in plot_models]\n", " test_rmse_stds = [test_scores_df.loc[m, \"RMSE_std\"] for m in plot_models]\n", " test_wmape_means = [test_scores_df.loc[m, \"WMAPE_mean\"] for m in plot_models]\n", " test_wmape_stds = [test_scores_df.loc[m, \"WMAPE_std\"] for m in plot_models]\n", " test_mape_means = [test_scores_df.loc[m, \"MAPE_mean\"] for m in plot_models]\n", " test_mape_stds = [test_scores_df.loc[m, \"MAPE_std\"] for m in plot_models]\n", " test_r2_means = [test_scores_df.loc[m, \"R2_mean\"] for m in plot_models]\n", " test_r2_stds = [test_scores_df.loc[m, \"R2_std\"] for m in plot_models]\n", " # NEW: Get train metrics for plotting\n", " train_mae_means = [train_scores_df.loc[m, \"MAE_mean\"] for m in plot_models]\n", " train_mae_stds = [train_scores_df.loc[m, \"MAE_std\"] for m in plot_models]\n", " train_r2_means = [train_scores_df.loc[m, \"R2_mean\"] for m in plot_models]\n", " train_r2_stds = [train_scores_df.loc[m, \"R2_std\"] for m in plot_models]\n", " x = np.arange(len(plot_models))\n", " width = 0.35\n", " # Figure 1: MAE (Train vs Test)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, test_mae_means, width, yerr=test_mae_stds, label='Test MAE')\n", " ax.bar(x + width/2, train_mae_means, width, yerr=train_mae_stds, label='Train MAE') # NEW: Add train bars\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"MAE\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig1_mae_train_test.png\"))\n", " # Figure 2: R2 (Train vs Test)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, test_r2_means, width, yerr=test_r2_stds, label='Test R2')\n", " ax.bar(x + width/2, train_r2_means, width, yerr=train_r2_stds, label='Train R2') # NEW: Add train bars\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"R2 Score\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig2_r2_train_test.png\"))\n", " # Figure 3: WMAPE/MAPE (Test only, as before)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, test_wmape_means, width, yerr=test_wmape_stds, label='Test WMAPE')\n", " ax.bar(x + width/2, test_mape_means, width, yerr=test_mape_stds, label='Test MAPE')\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"Percentage Error\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig3_wmape_mape_test.png\"))\n", " # Figure 5: Overfitting Bar Chart (Difference)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, overfitting_df['Diff_MAE'], width, label='Test MAE - Train MAE', alpha=0.8)\n", " ax.axhline(0, color='black', linewidth=0.8) # Add a horizontal line at 0\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"Difference in Metric (Test - Train)\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig5_overfitting_diff_mae.png\"))\n", " # Figure 6: Overfitting R2 Chart (Difference)\n", " fig, ax = plt.subplots(figsize=(12, 6))\n", " ax.bar(x - width/2, overfitting_df['Diff_R2'], width, label='Test R2 - Train R2', alpha=0.8, color='orange')\n", " ax.axhline(0, color='black', linewidth=0.8) # Add a horizontal line at 0\n", " ax.set_xticks(x)\n", " ax.set_xticklabels(plot_models, rotation=45)\n", " ax.set_ylabel(\"Difference in R2 (Test - Train)\"); ax.legend()\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig6_overfitting_diff_r2.png\"))\n", " # Intermittency Histograms\n", " fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n", " axes[0].hist(intermittency_df[\"ADI\"], bins=50, color=\"skyblue\")\n", " axes[0].set_title(f\"ADI (median={intermittency_df['ADI'].median():.2f})\")\n", " axes[1].hist(intermittency_df[\"CV2\"].clip(0, 10), bins=50, color=\"salmon\")\n", " axes[1].set_title(f\"CV² (median={intermittency_df['CV2'].median():.2f})\")\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(out_fig, \"fig7_intermittency.png\"))\n", " print(f\"[INFO] Pipeline completed in {time.time() - start_time:.2f} seconds. Results saved to: {outdir}\")\n", " # Print summary\n", " print(\"--- SUMMARY ---\")\n", " print(\"Test Results (Mean ± Std):\")\n", " print(test_scores_df[[\"MAE_mean\", \"RMSE_mean\", \"WMAPE_mean\", \"MAPE_mean\", \"R2_mean\"]].round(4))\n", " best_test_mae_model = test_scores_df['MAE_mean'].idxmin()\n", " best_test_r2_model = test_scores_df['R2_mean'].idxmax()\n", " print(f\"Best Test MAE: {test_scores_df.loc[best_test_mae_model, 'MAE_mean']:.4f} ({best_test_mae_model})\")\n", " print(f\"Best Test R2: {test_scores_df.loc[best_test_r2_model, 'R2_mean']:.4f} ({best_test_r2_model})\")\n", " print(\"Train Results (Mean ± Std):\") # NEW: Print train summary\n", " print(train_scores_df[[\"MAE_mean\", \"RMSE_mean\", \"WMAPE_mean\", \"MAPE_mean\", \"R2_mean\"]].round(4))\n", " print(\"Overfitting Diagnosis (Mean Difference: Test - Train):\") # Print overfitting summary\n", " print(overfitting_df[[\"Diff_MAE\", \"Diff_R2\"]].round(4))\n", " print(f\"Median ADI: {intermittency_df['ADI'].median():.2f}\")\n", " print(f\"Median CV²: {intermittency_df['CV2'].median():.2f}\")\n", " return cv_results, train_results # Return both test and train results\n", "\n", "# ----------------------------- Entry Point -----------------------------\n", "if __name__ == \"__main__\":\n", " # Example usage:\n", " # df_filtered = pd.read_csv(\"your_data.csv\")\n", " run_test_pipeline(df_filtered, outdir=\"./DL\", shos_k=0.1, shos_w=0.7) # Provide your best k, w\n", " pass" ] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.0" } }, "nbformat": 4, "nbformat_minor": 0 }