notebooks/CXR_Mimic_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import cv2\n",
    "import time\n",
    "import random \n",
    "import copy\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import zipfile\n",
    "\n",
    "# data paths\n",
    "# root dir: root path where necessary files are stored\n",
    "# data dir: where image files are stored (change the path if it is stored somewhere else)\n",
    "# res_dir: where resulting annotations and image matrix should be saved\n",
    "root_dir = ... # to be filled\n",
    "data_dir = root_dir + 'files.zip' # to be filled\n",
    "res_dir = ... # to be filled\n",
    "\n",
    "# patient info - get patients.csv file from MIMIC-IV\n",
    "patients = pd.read_csv(root_dir + 'patients.csv')\n",
    "patients = patients[['subject_id','gender']]\n",
    "\n",
    "# admission info for gender - get admissions.csv file from MIMIC-IV\n",
    "admissions = pd.read_csv(root_dir + 'admissions.csv')\n",
    "admissions = admissions[['subject_id','insurance','ethnicity','marital_status']]\n",
    "admissions = admissions.drop_duplicates(subset=['subject_id'], keep='last')\n",
    "admissions = admissions.sort_values(by=['subject_id'])\n",
    "\n",
    "# chexpert labels\n",
    "labels = pd.read_csv(root_dir + 'mimic-cxr-2.0.0-chexpert.csv')\n",
    "\n",
    "# meta data\n",
    "df = pd.read_csv(root_dir + 'mimic-cxr-2.0.0-metadata.csv')\n",
    "df = df[['dicom_id','subject_id','study_id','ViewPosition']]\n",
    "df.drop(df[(df['ViewPosition'] != 'PA') & (df['ViewPosition'] != 'AP') ].index, inplace=True)\n",
    "# note that studies s52250236, s53104432, s55170157, s55802622 and s59316002 are also dropped\n",
    "\n",
    "# merge with tables for patientand label info\n",
    "df = pd.merge(df, patients)\n",
    "df = pd.merge(df,admissions)\n",
    "df = pd.merge(df, labels)\n",
    "\n",
    "# add a path column\n",
    "df['path'] = 'files/p' + df['subject_id'].astype(str).str[:2] + '/p' + df['subject_id'].astype(str) + '/s' + df['study_id'].astype(str) + '/' + df['dicom_id'].astype(str) + '.jpg'\n",
    "\n",
    "# resize parameter\n",
    "transResize = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create img_mat\n",
    "img_mat = np.zeros((len(df),transResize,transResize))\n",
    "df2 = df.copy()\n",
    "    \n",
    "# initialize cnt\n",
    "cnt = 0\n",
    "with zipfile.ZipFile(data_dir, 'r') as z:\n",
    "        \n",
    "    # iterate through files\n",
    "    for filename in tqdm(df['path']):\n",
    "            \n",
    "        # read image\n",
    "        try: \n",
    "            img = PIL.Image.open(z.open(filename)).convert('RGB') \n",
    "        except:\n",
    "            # drop the row\n",
    "            print(filename)\n",
    "            df2.drop(df2[df2['path'] == filename].index, inplace=True)\n",
    "            continue\n",
    "            \n",
    "        # cut depending on the size\n",
    "        width, height = img.size\n",
    "        r_min = max(0,(height-width)/2)\n",
    "        r_max = min(height,(height+width)/2)\n",
    "        c_min = max(0,(width-height)/2)\n",
    "        c_max = min(width,(height+width)/2)\n",
    "        img = img.crop((c_min,r_min,c_max,r_max))\n",
    "            \n",
    "        # hist equalize and reshape\n",
    "        img = img.resize((transResize,transResize))\n",
    "        img = PIL.ImageOps.equalize(img)\n",
    "        img = img.convert('L')\n",
    "            \n",
    "        # assign\n",
    "        img_mat[cnt,:,:] = np.array(img)   \n",
    "            \n",
    "        # increment\n",
    "        cnt = cnt + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# save\n",
    "img_mat = img_mat[0:len(df2),:,:]\n",
    "np.save(res_dir + 'files_' + str(transResize) + '.npy', img_mat)\n",
    "\n",
    "# save dataframe as csv\n",
    "df2.to_csv(res_dir + 'meta_data.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}