{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0",
   "metadata": {},
   "source": [
    "# AWS Radar Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1",
   "metadata": {},
   "source": [
    "Demonstrates listing and downloading files from AWS radar data buckets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from radarx.io.aws_data import list_available_files, download_file\n",
    "import pandas as pd\n",
    "from itables import init_notebook_mode, show"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3",
   "metadata": {},
   "source": [
    "## Set up download directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "DOWNLOAD_DIR = \"./downloads\"\n",
    "os.makedirs(DOWNLOAD_DIR, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_files(bucket_name, prefix):\n",
    "    \"\"\"Utility function to list available files in a bucket and print them.\"\"\"\n",
    "    print(f\"Searching for files in bucket: {bucket_name}, prefix: {prefix}\")\n",
    "    files = list_available_files(bucket_name, prefix)\n",
    "    if files:\n",
    "        print(f\"Available files: {files[:5]}\")  # Display first 5 files for brevity\n",
    "    else:\n",
    "        print(\"No files found.\")\n",
    "    return files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6",
   "metadata": {},
   "source": [
    "## NEXRAD Level II"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_name = \"noaa-nexrad-level2\"\n",
    "prefix = \"2016/10/06/KAMX/\"\n",
    "files = display_files(bucket_name, prefix)\n",
    "\n",
    "if files:\n",
    "    file_to_download = files[0]\n",
    "    print(f\"Downloading: {file_to_download}\")\n",
    "    local_path = download_file(bucket_name, file_to_download, DOWNLOAD_DIR)\n",
    "    print(f\"Downloaded: {local_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8",
   "metadata": {},
   "source": [
    "## NEXRAD Real-Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_name = \"unidata-nexrad-level2-chunks\"\n",
    "prefix = \"KAMX/457/\"\n",
    "files = display_files(bucket_name, prefix)\n",
    "\n",
    "if files:\n",
    "    file_to_download = files[0]\n",
    "    print(f\"Downloading: {file_to_download}\")\n",
    "    local_path = download_file(bucket_name, file_to_download, DOWNLOAD_DIR)\n",
    "    print(f\"Downloaded: {local_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10",
   "metadata": {},
   "source": [
    "## NEXRAD Level III"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_name = \"unidata-nexrad-level3\"\n",
    "prefix = \"AMX_N0Q/AMX_N0Q_2016_10_06_17_00\"\n",
    "files = display_files(bucket_name, prefix)\n",
    "\n",
    "if files:\n",
    "    file_to_download = files[0]\n",
    "    print(f\"Downloading: {file_to_download}\")\n",
    "    local_path = download_file(bucket_name, file_to_download, DOWNLOAD_DIR)\n",
    "    print(f\"Downloaded: {local_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12",
   "metadata": {},
   "source": [
    "## MRMS Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_name = \"noaa-mrms-pds\"\n",
    "prefix = \"CONUS/ReflectivityAtLowestAltitude_00.50/20220330/\"\n",
    "files = display_files(bucket_name, prefix)\n",
    "\n",
    "if files:\n",
    "    file_to_download = files[0]\n",
    "    print(f\"Downloading: {file_to_download}\")\n",
    "    local_path = download_file(bucket_name, file_to_download, DOWNLOAD_DIR)\n",
    "    print(f\"Downloaded: {local_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "init_notebook_mode(all_interactive=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the configuration table as a dictionary\n",
    "data = {\n",
    "    \"Function Name\": [\n",
    "        \"list_available_files\",\n",
    "        \"download_file\",\n",
    "        \"list_and_filter_files\",\n",
    "        \"get_nexrad_file\",\n",
    "        \"get_mrms_file\",\n",
    "        \"download_latest_file\",\n",
    "        \"get_file_metadata\",\n",
    "        \"get_available_prefixes\",\n",
    "        \"download_files_in_batch\",\n",
    "        \"list_all_files\",\n",
    "    ],\n",
    "    \"Purpose\": [\n",
    "        \"Lists available files in a specified S3 bucket based on the given prefix.\",\n",
    "        \"Downloads a file from the specified S3 bucket to a local directory.\",\n",
    "        \"Combines `list_available_files` and filters results based on additional date criteria.\",\n",
    "        \"High-level function to download NEXRAD Level II or Level III files based on location and time.\",\n",
    "        \"High-level function to download MRMS files for a specific product and time.\",\n",
    "        \"Downloads the most recent file for a specific product.\",\n",
    "        \"Retrieves metadata for a specific file in the S3 bucket.\",\n",
    "        \"Lists all prefixes (subfolders) available in the S3 bucket.\",\n",
    "        \"Downloads multiple files from a list of file keys.\",\n",
    "        \"Recursively lists all files in the bucket for a given prefix.\",\n",
    "    ],\n",
    "    \"Supported Products\": [\n",
    "        \"noaa-nexrad-level2, unidata-nexrad-level2-chunks, unidata-nexrad-level3, noaa-mrms-pds\",\n",
    "        \"Same as above\",\n",
    "        \"Same as above\",\n",
    "        \"noaa-nexrad-level2, unidata-nexrad-level3\",\n",
    "        \"noaa-mrms-pds\",\n",
    "        \"unidata-nexrad-level2-chunks, noaa-mrms-pds\",\n",
    "        \"All buckets\",\n",
    "        \"All buckets\",\n",
    "        \"All buckets\",\n",
    "        \"All buckets\",\n",
    "    ],\n",
    "    \"Key Parameters\": [\n",
    "        \"`bucket` (str): S3 bucket name  <br> `prefix` (str): Prefix to filter files\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `file_key` (str): File path <br> `save_dir` (str): Local directory for saving\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `prefix` (str): Prefix to filter files <br> `date` (str): Date filter in `YYYYMMDD` format\",\n",
    "        \"`location` (str): Radar site code <br> `date` (str): Date in `YYYYMMDD` format <br> `time` (str): Time in `HHMM` format\",\n",
    "        \"`product` (str): MRMS product name <br> `date` (str): Date in `YYYYMMDD` format <br> `time` (str): Time in `HHMM` format\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `prefix` (str): Prefix to filter files\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `file_key` (str): File path\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `prefix` (str, optional): Filter for specific subdirectories\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `file_keys` (list): List of file paths <br> `save_dir` (str): Local directory for saving\",\n",
    "        \"`bucket` (str): S3 bucket name <br> `prefix` (str, optional): Filter for specific paths\",\n",
    "    ],\n",
    "    \"Output\": [\n",
    "        \"List of file keys\",\n",
    "        \"Downloads file to the specified directory\",\n",
    "        \"Filtered list of file keys\",\n",
    "        \"Downloads file to `./downloads/nexrad_level2` or `nexrad_level3` directories\",\n",
    "        \"Downloads file to `./downloads/mrms` directory\",\n",
    "        \"Downloads file to `./downloads/latest` directory\",\n",
    "        \"Metadata (e.g., size, last modified date)\",\n",
    "        \"List of available prefixes\",\n",
    "        \"Downloads files to specified directory\",\n",
    "        \"Full list of file keys\",\n",
    "    ],\n",
    "    \"Notes\": [\n",
    "        \"Use this to search for files based on time/location prefixes.\",\n",
    "        \"Ensure the `save_dir` exists before calling this function.\",\n",
    "        \"Use this to narrow down file searches for specific dates.\",\n",
    "        \"Automates searching and downloading of specific radar files.\",\n",
    "        \"Use this for MRMS-specific file downloads, e.g., Reflectivity, QPE.\",\n",
    "        \"Useful for real-time monitoring applications.\",\n",
    "        \"Use this for debugging or to retrieve additional information about a file before downloading.\",\n",
    "        \"Use this to explore bucket contents or verify structure for new products.\",\n",
    "        \"Ensure a valid list of file keys is passed.\",\n",
    "        \"Useful for large-scale data exploration; might be slow for buckets with massive datasets.\",\n",
    "    ],\n",
    "}\n",
    "\n",
    "# Convert to a pandas DataFrame\n",
    "df = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "show(df)  # scroll right to see the full list"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}