{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example Data Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['OPENTSDB_PYTHON_METRICS_TEST_MODE'] = 'True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from banzai.calibrations import make_master_calibrations\n",
    "import requests\n",
    "from banzai_nres import settings\n",
    "from banzai import dbs\n",
    "from banzai.utils.stage_utils import run_pipeline_stages\n",
    "from banzai.logs import set_log_level, get_logger\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setup logging and some settings show we know where the DB should be.\n",
    "------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_log_level('DEBUG')\n",
    "logger = get_logger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['DB_ADDRESS'] = 'sqlite:///test_data/test.db'\n",
    "\n",
    "settings.processed_path= os.path.join(os.getcwd(), 'test_data')\n",
    "settings.fpack=True\n",
    "settings.db_address = os.environ['DB_ADDRESS']\n",
    "settings.reduction_level = 92"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up the context object.\n",
    "import banzai.main\n",
    "context = banzai.main.parse_args(settings, parse_system_args=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download some test data from the archive\n",
    "----------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make directories for the test dataset.\n",
    "raw_data_dir = 'test_data/lsc/nres01/20180313/raw'\n",
    "os.makedirs(raw_data_dir, exist_ok=True)\n",
    "bpm_dir = 'test_data/lsc/nres01/bpm/'\n",
    "os.makedirs(bpm_dir, exist_ok=True)\n",
    "phoenix_dir = 'test_data/phoenix_models'\n",
    "os.makedirs(phoenix_dir, exist_ok=True)\n",
    "raw_phoenix_dir = 'test_data/raw_phoenix_models'\n",
    "os.makedirs(raw_phoenix_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose which files we will download for our test dataset.\n",
    "bpm_filename = 'bpm-lsc-nres01-fl09-20180215.fits.fz'\n",
    "test_filenames = ['lscnrs01-fl09-20180313-0001-w00.fits.fz', 'lscnrs01-fl09-20180313-0002-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0003-w00.fits.fz', 'lscnrs01-fl09-20180313-0004-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0005-w00.fits.fz', 'lscnrs01-fl09-20180313-0006-a00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0007-a00.fits.fz', 'lscnrs01-fl09-20180313-0008-a00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0009-w00.fits.fz', 'lscnrs01-fl09-20180313-0010-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0011-w00.fits.fz', 'lscnrs01-fl09-20180313-0012-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0013-w00.fits.fz', 'lscnrs01-fl09-20180313-0014-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0015-w00.fits.fz', 'lscnrs01-fl09-20180313-0016-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0017-w00.fits.fz', 'lscnrs01-fl09-20180313-0018-w00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0019-a00.fits.fz', 'lscnrs01-fl09-20180313-0020-a00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0021-a00.fits.fz', 'lscnrs01-fl09-20180313-0022-a00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0023-a00.fits.fz', 'lscnrs01-fl09-20180313-0042-b00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0043-b00.fits.fz', 'lscnrs01-fl09-20180313-0044-b00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0045-b00.fits.fz', 'lscnrs01-fl09-20180313-0046-b00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0047-d00.fits.fz', 'lscnrs01-fl09-20180313-0048-d00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0049-d00.fits.fz', 'lscnrs01-fl09-20180313-0028-e00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0029-e00.fits.fz', 'lscnrs01-fl09-20180313-0030-e00.fits.fz',\n",
    "                  'lscnrs01-fl09-20180313-0031-e00.fits.fz']\n",
    "\n",
    "bpm_frame_id = '22529799'\n",
    "test_frame_ids = ['8148793', '8148822', '8148805', '8148824', '8148826', '8148876', '8148898',\n",
    "                  '8148932', '8148960', '8148978', '8149024', '8149104', '8149068', '8149090',\n",
    "                  '8149128', '8149173', '8149217', '8151252', '8149314', '8149388', '8149469',\n",
    "                  '8149547', '8149570', '8156341', '8156360', '8156366', '8156376', '8156385',\n",
    "                  '8156433', '8156500', '8156560', '8151997', '8152050', '8152110', '8152153']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# THIS DOWNLOADS A NEW DATA SET. DO NOT RUN IF YOU HAVE ALREADY DOWNLOADED THE DATA\n",
    "# Copy the data into the correct directory\n",
    "for test_filename, frame_id in zip(test_filenames, test_frame_ids):\n",
    "    archive_url = f'https://archive-api.lco.global/frames/{frame_id}'\n",
    "    frame_info = requests.get(archive_url).json()\n",
    "    with open(os.path.join(raw_data_dir, test_filename), 'wb') as f:\n",
    "        f.write(requests.get(frame_info['url']).content)\n",
    "        \n",
    "archive_url = f'https://archive-api.lco.global/frames/{bpm_frame_id}'\n",
    "frame_info = requests.get(archive_url).json()\n",
    "with open(os.path.join(bpm_dir, bpm_filename), 'wb') as f:\n",
    "    f.write(requests.get(frame_info['url']).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Download the phoenix models and get them into the correct format. \n",
    "#Likewise, do not run if you have already downloaded the data.\n",
    "os.system(f'wget ftp://phoenix.astro.physik.uni-goettingen.de/v2.0/HiResFITS/PHOENIX-ACES-AGSS-COND-2011/Z-0.0/lte05700-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits -P {raw_phoenix_dir}/')\n",
    "os.system(f'wget ftp://phoenix.astro.physik.uni-goettingen.de/v2.0/HiResFITS/WAVE_PHOENIX-ACES-AGSS-COND-2011.fits -P {raw_phoenix_dir}/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.system(f'banzai_nres_munge_phoenix --input-dir={raw_phoenix_dir} --output-dir={phoenix_dir}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize the db and context object for the rest of the pipeline\n",
    "-----------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# THIS MAKES A FRESH DATABASE. DO NOT RUN ME UNLESS YOU WANT TO DELETE YOUR test.db AND MAKE A FRESH ONE\n",
    "full_phoenix_path=os.path.join(os.getcwd(), phoenix_dir)\n",
    "os.system(f'banzai_nres_create_db --db-address={os.environ[\"DB_ADDRESS\"]}')\n",
    "os.system(f'banzai_nres_populate_phoenix_models --model-location=\"{full_phoenix_path}\" --db-address={os.environ[\"DB_ADDRESS\"]}')\n",
    "os.system(f'banzai_add_site --site elp --latitude 30.67986944 --longitude -104.015175 --elevation 2027 --timezone -6 --db-address={os.environ[\"DB_ADDRESS\"]}')\n",
    "os.system(f'banzai_add_site --site lsc --latitude -30.1673833333 --longitude -70.8047888889 --elevation 2198 --timezone -4 --db-address={os.environ[\"DB_ADDRESS\"]}')   \n",
    "os.system(f'banzai_add_instrument --site lsc --camera fl09 --name nres01 --instrument-type 1m0-NRES-SciCam --nx=4096 --ny=4096 --db-address={os.environ[\"DB_ADDRESS\"]}')\n",
    "os.system(f'banzai_add_instrument --site elp --camera fl17 --name nres02 --instrument-type 1m0-NRES-SciCam --nx=4096 --ny=4096 --db-address={os.environ[\"DB_ADDRESS\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for bpm_filename in glob('test_data/*/nres??/bpm/*bpm*'):\n",
    "    logger.info(f'adding bpm {bpm_filename} to the database')\n",
    "    os.system(f'banzai_nres_add_bpm --filename {bpm_filename} --db-address={os.environ[\"DB_ADDRESS\"]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the instrument record\n",
    "------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "instrument = dbs.get_instruments_at_site('lsc', settings.db_address)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Process and Stack Bias Files\n",
    "----------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_files = glob(os.path.join('test_data/*/nres??/*/raw/*b00*'))\n",
    "for bias_file in bias_files: \n",
    "    run_pipeline_stages([{'path': bias_file}], context) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mark_frames_as_good(filenames):\n",
    "    for filename in glob(f'test_data/*/nres??/*/processed/{filenames}'):\n",
    "        dbs.mark_frame(os.path.basename(filename), \"good\", db_address=os.environ['DB_ADDRESS'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mark_frames_as_good('*b92*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_master_calibrations(instrument, 'BIAS', '2017-01-01', '2019-01-01', context) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Process and Stack Dark Files\n",
    "----------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dark_files = glob('test_data/*/nres??/*/raw/*d00*')\n",
    "for dark_file in dark_files: \n",
    "    run_pipeline_stages([{'path': dark_file}], context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mark_frames_as_good('*d92*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_master_calibrations(instrument, 'DARK', '2017-01-01', '2019-01-01', context) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Process and stack lamp flats. This is where we also solve for the traces and profile.\n",
    "----------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flat_files = glob('test_data/*/nres??/*/raw/*w00*')\n",
    "for flat_file in flat_files: \n",
    "    run_pipeline_stages([{'path': flat_file}], context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mark_frames_as_good('*w92*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_master_calibrations(instrument, 'LAMPFLAT', '2017-01-01', '2019-01-01', context)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Process and stack arc lamps. Find the wavelength solution\n",
    "--------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arc_files = glob('test_data/*/nres??/*/raw/*a00*')\n",
    "for arc_file in arc_files: \n",
    "    run_pipeline_stages([{'path': arc_file}], context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mark_frames_as_good('*a92*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_master_calibrations(instrument, 'DOUBLE', '2017-01-01', '2019-01-01', context)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Process the science spectra\n",
    "------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "science_files = glob('test_data/*/nres??/*/raw/*e00*')\n",
    "for science_file in science_files: \n",
    "    run_pipeline_stages([{'path': science_file}], context)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}