![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg/"){kind=icon}
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "N8_hf9P_oObo",
+ "outputId": "9bbeaa90-fe6a-4185-a6c7-c0c9e4fc08bb"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Collecting prophet\n",
+ " Downloading prophet-1.0.1.tar.gz (65 kB)\n",
+ "\u001b[K |████████████████████████████████| 65 kB 2.4 MB/s \n",
+ "\u001b[?25hRequirement already satisfied: Cython>=0.22 in /usr/local/lib/python3.7/dist-packages (from prophet) (0.29.28)\n",
+ "Collecting cmdstanpy==0.9.68\n",
+ " Downloading cmdstanpy-0.9.68-py3-none-any.whl (49 kB)\n",
+ "\u001b[K |████████████████████████████████| 49 kB 4.7 MB/s \n",
+ "\u001b[?25hRequirement already satisfied: pystan~=2.19.1.1 in /usr/local/lib/python3.7/dist-packages (from prophet) (2.19.1.1)\n",
+ "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from prophet) (1.21.6)\n",
+ "Requirement already satisfied: pandas>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from prophet) (1.3.5)\n",
+ "Requirement already satisfied: matplotlib>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from prophet) (3.2.2)\n",
+ "Requirement already satisfied: LunarCalendar>=0.0.9 in /usr/local/lib/python3.7/dist-packages (from prophet) (0.0.9)\n",
+ "Requirement already satisfied: convertdate>=2.1.2 in /usr/local/lib/python3.7/dist-packages (from prophet) (2.4.0)\n",
+ "Requirement already satisfied: holidays>=0.10.2 in /usr/local/lib/python3.7/dist-packages (from prophet) (0.10.5.2)\n",
+ "Requirement already satisfied: setuptools-git>=1.2 in /usr/local/lib/python3.7/dist-packages (from prophet) (1.2)\n",
+ "Requirement already satisfied: python-dateutil>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from prophet) (2.8.2)\n",
+ "Requirement already satisfied: tqdm>=4.36.1 in /usr/local/lib/python3.7/dist-packages (from prophet) (4.64.0)\n",
+ "Collecting ujson\n",
+ " Downloading ujson-5.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (45 kB)\n",
+ "\u001b[K |████████████████████████████████| 45 kB 2.7 MB/s \n",
+ "\u001b[?25hRequirement already satisfied: pymeeus<=1,>=0.3.13 in /usr/local/lib/python3.7/dist-packages (from convertdate>=2.1.2->prophet) (0.5.11)\n",
+ "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from holidays>=0.10.2->prophet) (1.15.0)\n",
+ "Requirement already satisfied: korean-lunar-calendar in /usr/local/lib/python3.7/dist-packages (from holidays>=0.10.2->prophet) (0.2.1)\n",
+ "Requirement already satisfied: hijri-converter in /usr/local/lib/python3.7/dist-packages (from holidays>=0.10.2->prophet) (2.2.3)\n",
+ "Requirement already satisfied: ephem>=3.7.5.3 in /usr/local/lib/python3.7/dist-packages (from LunarCalendar>=0.0.9->prophet) (4.1.3)\n",
+ "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from LunarCalendar>=0.0.9->prophet) (2022.1)\n",
+ "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.0.0->prophet) (1.4.2)\n",
+ "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.0.0->prophet) (3.0.8)\n",
+ "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.0.0->prophet) (0.11.0)\n",
+ "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib>=2.0.0->prophet) (4.2.0)\n",
+ "Building wheels for collected packages: prophet\n",
+ " Building wheel for prophet (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+ " Created wheel for prophet: filename=prophet-1.0.1-py3-none-any.whl size=6640026 sha256=687c63b02691e6913335a63cb25cc3c9ec20eabca166ef378aa5fbc9948ef5d7\n",
+ " Stored in directory: /root/.cache/pip/wheels/4e/a0/1a/02c9ec9e3e9de6bdbb3d769d11992a6926889d71567d6b9b67\n",
+ "Successfully built prophet\n",
+ "Installing collected packages: ujson, cmdstanpy, prophet\n",
+ " Attempting uninstall: cmdstanpy\n",
+ " Found existing installation: cmdstanpy 0.9.5\n",
+ " Uninstalling cmdstanpy-0.9.5:\n",
+ " Successfully uninstalled cmdstanpy-0.9.5\n",
+ "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+ "fbprophet 0.7.1 requires cmdstanpy==0.9.5, but you have cmdstanpy 0.9.68 which is incompatible.\u001b[0m\n",
+ "Successfully installed cmdstanpy-0.9.68 prophet-1.0.1 ujson-5.2.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install prophet"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "GmXmMc2NoObr"
+ },
+ "outputs": [],
+ "source": [
+ "from math import factorial\n",
+ "import datetime\n",
+ "import random\n",
+ "from scipy import stats\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "from prophet import Prophet\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "import matplotlib.pyplot as plt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "id": "GXK2MZ4MoObs"
+ },
+ "outputs": [],
+ "source": [
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "id": "Ao_YeF64oObs"
+ },
+ "outputs": [],
+ "source": [
+ "sns.set_theme()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "YNipPdJ7oObs"
+ },
+ "source": [
+ "# Outliers detection using time series batch processing\n",
+ "## 1. Preparing data random time series"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "id": "NqonA_aroObu"
+ },
+ "outputs": [],
+ "source": [
+ "N_FOURIER = 10\n",
+ "N_ROWS = 25\n",
+ "N_YEARS = 3\n",
+ "\n",
+ "years = np.arange(0.0, 3.65 * N_YEARS, 0.01)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TbB7CxmPoObu"
+ },
+ "source": [
+ "### 1.1. Generating time series with base seasonality and trend"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "xL-am2KLoObv"
+ },
+ "outputs": [],
+ "source": [
+ "def generate_waves(idx: int, time: np.array = years) -> np.array:\n",
+ "\n",
+ " waves = [\n",
+ " np.cos(np.random.RandomState(idx).uniform(0.45, 0.55) *\n",
+ " np.pi * time[: len(time) // N_YEARS]) +\n",
+ " np.cos(26 * np.pi * time[: len(time) // N_YEARS])] * N_YEARS # weekly\n",
+ "\n",
+ " waves = np.concatenate(waves)\n",
+ "\n",
+ " waves += abs(min(waves))\n",
+ "\n",
+ " growth = np.random.uniform(0.05 * N_YEARS, 0.1 * N_YEARS)\n",
+ "\n",
+ " waves += np.asarray([growth * x for x in time])\n",
+ "\n",
+ " waves += np.random.uniform(size=len(time))\n",
+ "\n",
+ " # adding outliers\n",
+ " out_idx = np.random.choice(\n",
+ " np.arange(len(time)),\n",
+ " np.random.randint(len(time) * 0.15, len(time) * 0.25))\n",
+ "\n",
+ " outs = np.ones(len(time))\n",
+ "\n",
+ " outs[out_idx] = np.random.uniform(1.25, 1.95, len(out_idx))\n",
+ "\n",
+ " waves *= outs\n",
+ " waves *= np.random.uniform(0.1, 1.0) * 10 # rows of different orders\n",
+ "\n",
+ " return waves"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "Fk-qgOlkoObv"
+ },
+ "outputs": [],
+ "source": [
+ "rows = [generate_waves(row) for row in range(N_ROWS)]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 301
+ },
+ "id": "CpQE_L_2oObw",
+ "outputId": "86249357-5a66-457e-b837-fb088ff10d43"
+ },
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ "Text(0.5, 1.0, 'Generating data for 1095 days')"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 8
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ "| \n", + " | col_idx | \n", + "col_dat | \n", + "col_val | \n", + "col_doy | \n", + "
|---|---|---|---|---|
| 27365 | \n", + "id_24 | \n", + "2023-12-22 | \n", + "51.859046 | \n", + "356 | \n", + "
| 27366 | \n", + "id_24 | \n", + "2023-12-23 | \n", + "48.073541 | \n", + "357 | \n", + "
| 27367 | \n", + "id_24 | \n", + "2023-12-24 | \n", + "42.071191 | \n", + "358 | \n", + "
| 27368 | \n", + "id_24 | \n", + "2023-12-25 | \n", + "43.294315 | \n", + "359 | \n", + "
| 27369 | \n", + "id_24 | \n", + "2023-12-26 | \n", + "43.300023 | \n", + "360 | \n", + "
| 27370 | \n", + "id_24 | \n", + "2023-12-27 | \n", + "90.698637 | \n", + "361 | \n", + "
| 27371 | \n", + "id_24 | \n", + "2023-12-28 | \n", + "55.648824 | \n", + "362 | \n", + "
| 27372 | \n", + "id_24 | \n", + "2023-12-29 | \n", + "59.889812 | \n", + "363 | \n", + "
| 27373 | \n", + "id_24 | \n", + "2023-12-30 | \n", + "54.696407 | \n", + "364 | \n", + "
| 27374 | \n", + "id_24 | \n", + "2023-12-31 | \n", + "45.151037 | \n", + "365 | \n", + "