Data_Cleaning_Fernando.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scio\n",
    "from data import Data\n",
    "import metadatabase as mdb\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scio_length(file_name):\n",
    "\n",
    "    # Read file as a string.\n",
    "    try:\n",
    "        data = scio._read_file_as_string(file_name)\n",
    "    except:\n",
    "        return 0\n",
    "\n",
    "    # Catches empty files.\n",
    "    if len(data) == 0:\n",
    "        return 0\n",
    "\n",
    "    # Get the file dimension.\n",
    "    i = 0;\n",
    "    dimension = np.abs(np.frombuffer(data[i:i+4], dtype='int32')[0])\n",
    "    i = i + 4\n",
    "\n",
    "    # Extract the number of frequenct channels.\n",
    "    channels = np.frombuffer(data[i:i+4*dimension], dtype='int32')\n",
    "\n",
    "    # Get the data type.\n",
    "    i = i + 4*dimension\n",
    "    data_type = np.frombuffer(data[i:i+4], dtype='int32')[0]\n",
    "    i = i + 4\n",
    "\n",
    "    # File size sanity check.\n",
    "    bytes_per_frame = scio.int2nbyte(data_type)*np.product(channels)\n",
    "    number_of_bytes = len(data) - i\n",
    "    remainder = np.remainder(number_of_bytes, bytes_per_frame)\n",
    "\n",
    "    return number_of_bytes/bytes_per_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories=['Antenna', 'Switch']\n",
    "instruments=['70MHz']\n",
    "channels=['EW']\n",
    "intervals=[(1554200000.0, 1554300000.0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = Data.via_metadatabase(categories=categories,\n",
    "                          instruments=instruments,\n",
    "                          channels=channels,\n",
    "                          intervals=intervals,\n",
    "                          quality=[1, 'NULL'],\n",
    "                          integrity=[1, 0, 'NULL'],\n",
    "                          completeness=[1, 0, 'NULL'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths = mdb.locate(categories=categories,\n",
    "                   instruments=instruments,\n",
    "                   channels=channels,\n",
    "                   intervals=intervals,\n",
    "                   quality=[1, 0, 'NULL'],\n",
    "                   integrity=[1, 0, 'NULL'],\n",
    "                   completeness=[1, 0, 'NULL'])\n",
    "\n",
    "pol_files = [file_path for (_, _, file_path, _, file_name, _) in paths if 'pol' in file_name]\n",
    "pol_lines = np.array([scio_length('/project/s/sievers/prizm' + pol_file) for pol_file in pol_files])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(1, figsize=(100, 100))\n",
    "plt.title('pol')\n",
    "plt.imshow(np.log10(d.get(data='pol', instrument=instruments[0], channel=channels[0], partition=None)), vmin=4.5, vmax=8.5, aspect=1)\n",
    "plt.hlines(np.cumsum(pol_lines), np.zeros_like(pol_lines), 4095*np.ones_like(pol_lines), color='black', alpha=0.1, linewidth=1)\n",
    "plt.hlines(np.cumsum(pol_lines)[[0]], 0, 4095, color='red', alpha=1.0, linewidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(1, figsize=(100,100))\n",
    "plt.plot(d.get(data='time_sys_start', instrument=instruments[0], channel=channels[0], partition=None), '-', markersize=1)\n",
    "plt.vlines(np.cumsum(pol_lines), d[instruments[0]][channels[0]]['time_sys_start'][0], np.max(d[instruments[0]][channels[0]]['time_sys_start']), color='black', linewidth=1, alpha=0.1)\n",
    "plt.ylim(d[instruments[0]][channels[0]]['time_sys_start'][0], np.max(d[instruments[0]][channels[0]]['time_sys_start']))\n",
    "\n",
    "\n",
    "plt.vlines(np.cumsum(pol_lines)[[0]], d[instruments[0]][channels[0]]['time_sys_start'][0], np.max(d[instruments[0]][channels[0]]['time_sys_start']), color='red', linewidth=1, zorder=100)\n",
    "\n",
    "for (flag, ctime) in d[instruments[0]]['Switch']['antenna']:\n",
    "    plt.hlines(ctime, 0, d[instruments[0]][channels[0]]['time_sys_start'].shape[0], color='blue', linewidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pol_files[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pol_files[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "d.partition(instruments=['70MHz'], channels=['EW'], buffer=(0,0))\n",
    "plt.figure(1, figsize=(100, 100))\n",
    "plt.title('pol')\n",
    "plt.imshow(np.log10(d.get(data='pol', instrument=instruments[0], channel=channels[0], partition='antenna')), vmin=4.5, vmax=8.5, aspect=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}