2D Results-GitHub1.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CTan Morphometry Results 2D\n",
    "##### Author: Anna Valentine (annavalentine@mines.edu)\n",
    "#### Date: 07/19/21\n",
    "\n",
    "#### Purpose:  \n",
    "Takes in Ctan/batman files from microCT and concatonates the 2D results from the snowpit into one dataframe. This is meant for one snowpit at a time. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write down my standard path here\n",
    "path = '/Users/annav/1_WORK_CRREL/CTan Project 1/' \n",
    "files = path+'Data/*.txt'  \n",
    "UTF8_folder = path + 'D_UTF8/'   # set up a folder for UTF8 conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#general takes a path and spits out all of the file names \n",
    "def list_files_local(path):\n",
    "    \"\"\" Get file list form local folder. \"\"\"\n",
    "    from glob import glob\n",
    "    return glob(path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converts files from ANSCI to UTF8, which python can read\n",
    "def UTF8_convert(list_files):\n",
    "    BLOCKSIZE = 300000 # desired size in bytes, this is 300 kB, which is larger than biggest file \n",
    "\n",
    "    for file in list_files: \n",
    "        name_conv = UTF8_folder + file[46:62] +'_UTF8.txt'   #naming convention and moves to folder for UTF8-Files\n",
    "        with codecs.open(file, \"r\", \"mbcs\") as sourceFile:\n",
    "            with codecs.open(name_conv, \"w\", \"utf-8\") as targetFile:     # convert to UTF-8\n",
    "                while True:\n",
    "                    contents = sourceFile.read(BLOCKSIZE)\n",
    "                    if not contents:\n",
    "                        break\n",
    "                    targetFile.write(contents)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gives the sample depth (lower) from the file name\n",
    "def sample_height(file):\n",
    "    #Find scan depth,\n",
    "    sc = file.split(\"_\") #This read from file name\n",
    "    scan_depth_tot = sc[4]\n",
    "    sc2 = scan_depth_tot.split(\"-\")\n",
    "    num = float(sc2[0])\n",
    "    \n",
    "    return num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Find the term in the file (used for knowing where to start dataframe)\n",
    "def find_term(term, file):\n",
    "    row = 0\n",
    "    file_o = open(file)\n",
    "    for line in file_o:\n",
    "        row += 1\n",
    "        line.strip().split('/n')\n",
    "        if term in line:\n",
    "            return (row)\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loop through all of the files in the snowpit and concatonates in one dataframe\n",
    "def loop_files(files):\n",
    "    \n",
    "    #start and end terms\n",
    "    start = \"2D analysis\"\n",
    "    end = \"3D analysis\"\n",
    "    frames = []\n",
    "\n",
    "    for file in files:\n",
    "\n",
    "        #Find start row, read in csv for Morpho Result\n",
    "        end_row = (find_term(end, file)-4)\n",
    "        start_row = find_term(start, file)+9\n",
    "        nrow = end_row - start_row\n",
    "        df_int = pd.read_csv(file, skiprows= (start_row), nrows=(nrow))\n",
    "\n",
    "        #Find scan depth,\n",
    "        sc = file.split(\"_\") #This read from file name\n",
    "        scan_depth = sc[4]\n",
    "\n",
    "        #find average depth\n",
    "        cutoff = scan_depth[0:-2]\n",
    "        x = cutoff.split(\"-\")\n",
    "        hi = float(x[0])\n",
    "        lo = float(x[1])\n",
    "        avg_depth = (hi + lo)/2\n",
    "\n",
    "\n",
    "        ### Add in some rows for this prelim info\n",
    "        ### Description, Abbreviation, Value, Unit\n",
    "        df_int['Scan Depth'] = [scan_depth]*nrow  # Row for Scan Depth\n",
    "\n",
    "        df_int = df_int.drop([df_int.index[0]])\n",
    "        df_int = df_int.rename(columns={'Unnamed: 0':'File Name'})\n",
    "\n",
    "        #Add the column we want (\"Values\") to the datafram\n",
    "        frames.append(df_int)\n",
    "       \n",
    "\n",
    "    result = pd.concat(frames)\n",
    "        \n",
    "    return result \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Main calls all other functions, takes in a file path and yes/no if you want a .csv out \n",
    "def main(files, to_csv):\n",
    "    list_files = list_files_local(files)  \n",
    "    \n",
    "    #get the snowpit name of file\n",
    "    snowpit = str(list_files[0][46:50])\n",
    "    \n",
    "    \n",
    "    #Convert to UTF-8\n",
    "    UTF8_convert(list_files)\n",
    "    \n",
    "    #Sort-Files\n",
    "    UTF8_files = list_files_local(UTF8_folder+ '/*.txt') \n",
    "    UTF8_files = sorted(UTF8_files, key = sample_height) \n",
    "    \n",
    "    \n",
    "    #Find Start/End of the dataframe\n",
    "    start = \"2D analysis\"\n",
    "    end = \"3D analysis\"\n",
    "    \n",
    "    #Loop through the files\n",
    "    result = loop_files(UTF8_files)\n",
    "    \n",
    "    #If to .csv is wanted: \n",
    "    if to_csv:\n",
    "        # Export our dataframe to a .csv\n",
    "        result.to_csv(\"M_RESULTS_2D\"+snowpit+\".csv\", index =False)\n",
    "        \n",
    "    return result "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File Name</th>\n",
       "      <th>Pos.Z</th>\n",
       "      <th>Obj.N</th>\n",
       "      <th>T.Ar</th>\n",
       "      <th>Obj.Ar</th>\n",
       "      <th>Obj.Ar/T.Ar</th>\n",
       "      <th>T.Pm</th>\n",
       "      <th>Obj.Pm</th>\n",
       "      <th>Obj.Pm/Obj.Ar</th>\n",
       "      <th>Av.Obj.Ar</th>\n",
       "      <th>...</th>\n",
       "      <th>MMI(min)</th>\n",
       "      <th>T.Or(phi)</th>\n",
       "      <th>Ecc</th>\n",
       "      <th>St.Th(pl)</th>\n",
       "      <th>St.Sp(pl)</th>\n",
       "      <th>St.Li.Dn(pl)</th>\n",
       "      <th>FD</th>\n",
       "      <th>i.Pm</th>\n",
       "      <th>Unnamed: 39</th>\n",
       "      <th>Scan Depth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1s17_5-3cm_20um_rec_voi_1040.bmp</td>\n",
       "      <td>21.41504</td>\n",
       "      <td>149.0</td>\n",
       "      <td>93.99345</td>\n",
       "      <td>30.53807</td>\n",
       "      <td>32.48957</td>\n",
       "      <td>36.26756</td>\n",
       "      <td>339.58716</td>\n",
       "      <td>11.12013</td>\n",
       "      <td>0.20495</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>166.54467</td>\n",
       "      <td>0.36757</td>\n",
       "      <td>0.17985</td>\n",
       "      <td>0.37372</td>\n",
       "      <td>1.80644</td>\n",
       "      <td>1.54754</td>\n",
       "      <td>12.48324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-3cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1s17_5-3cm_20um_rec_voi_1039.bmp</td>\n",
       "      <td>21.39445</td>\n",
       "      <td>151.0</td>\n",
       "      <td>93.99345</td>\n",
       "      <td>30.49222</td>\n",
       "      <td>32.44079</td>\n",
       "      <td>36.26756</td>\n",
       "      <td>339.39845</td>\n",
       "      <td>11.13066</td>\n",
       "      <td>0.20194</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>166.41858</td>\n",
       "      <td>0.39134</td>\n",
       "      <td>0.17968</td>\n",
       "      <td>0.37420</td>\n",
       "      <td>1.80544</td>\n",
       "      <td>1.69069</td>\n",
       "      <td>12.83122</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-3cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1s17_5-3cm_20um_rec_voi_1038.bmp</td>\n",
       "      <td>21.37386</td>\n",
       "      <td>153.0</td>\n",
       "      <td>93.99345</td>\n",
       "      <td>30.61306</td>\n",
       "      <td>32.56936</td>\n",
       "      <td>36.26756</td>\n",
       "      <td>343.01012</td>\n",
       "      <td>11.20470</td>\n",
       "      <td>0.20009</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>165.30597</td>\n",
       "      <td>0.40363</td>\n",
       "      <td>0.17850</td>\n",
       "      <td>0.36955</td>\n",
       "      <td>1.82465</td>\n",
       "      <td>1.55996</td>\n",
       "      <td>13.28716</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-3cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1s17_5-3cm_20um_rec_voi_1037.bmp</td>\n",
       "      <td>21.35327</td>\n",
       "      <td>152.0</td>\n",
       "      <td>93.99345</td>\n",
       "      <td>30.81860</td>\n",
       "      <td>32.78803</td>\n",
       "      <td>36.26756</td>\n",
       "      <td>341.37986</td>\n",
       "      <td>11.07707</td>\n",
       "      <td>0.20275</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>164.40286</td>\n",
       "      <td>0.41216</td>\n",
       "      <td>0.18055</td>\n",
       "      <td>0.37011</td>\n",
       "      <td>1.81598</td>\n",
       "      <td>1.67885</td>\n",
       "      <td>13.16008</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-3cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1s17_5-3cm_20um_rec_voi_1036.bmp</td>\n",
       "      <td>21.33267</td>\n",
       "      <td>138.0</td>\n",
       "      <td>93.99345</td>\n",
       "      <td>31.11482</td>\n",
       "      <td>33.10318</td>\n",
       "      <td>36.26756</td>\n",
       "      <td>340.59603</td>\n",
       "      <td>10.94642</td>\n",
       "      <td>0.22547</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>163.18175</td>\n",
       "      <td>0.41304</td>\n",
       "      <td>0.18271</td>\n",
       "      <td>0.36923</td>\n",
       "      <td>1.81181</td>\n",
       "      <td>1.68291</td>\n",
       "      <td>12.80607</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-3cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>1s17_94-92cm_20um_rec_voi_0044.bmp</td>\n",
       "      <td>0.90602</td>\n",
       "      <td>262.0</td>\n",
       "      <td>136.94602</td>\n",
       "      <td>45.29154</td>\n",
       "      <td>33.07255</td>\n",
       "      <td>46.80068</td>\n",
       "      <td>562.56584</td>\n",
       "      <td>12.42099</td>\n",
       "      <td>0.17287</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>137.80450</td>\n",
       "      <td>0.19643</td>\n",
       "      <td>0.16102</td>\n",
       "      <td>0.32584</td>\n",
       "      <td>2.05397</td>\n",
       "      <td>1.76204</td>\n",
       "      <td>0.08237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94-92cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>1s17_94-92cm_20um_rec_voi_0043.bmp</td>\n",
       "      <td>0.88543</td>\n",
       "      <td>258.0</td>\n",
       "      <td>136.94602</td>\n",
       "      <td>45.42197</td>\n",
       "      <td>33.16779</td>\n",
       "      <td>46.80068</td>\n",
       "      <td>561.18839</td>\n",
       "      <td>12.35500</td>\n",
       "      <td>0.17605</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>136.29174</td>\n",
       "      <td>0.22195</td>\n",
       "      <td>0.16188</td>\n",
       "      <td>0.32618</td>\n",
       "      <td>2.04894</td>\n",
       "      <td>1.82027</td>\n",
       "      <td>0.12355</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94-92cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>1s17_94-92cm_20um_rec_voi_0042.bmp</td>\n",
       "      <td>0.86484</td>\n",
       "      <td>253.0</td>\n",
       "      <td>136.94602</td>\n",
       "      <td>45.48435</td>\n",
       "      <td>33.21334</td>\n",
       "      <td>46.80068</td>\n",
       "      <td>563.00421</td>\n",
       "      <td>12.37798</td>\n",
       "      <td>0.17978</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>137.22689</td>\n",
       "      <td>0.23615</td>\n",
       "      <td>0.16158</td>\n",
       "      <td>0.32491</td>\n",
       "      <td>2.05557</td>\n",
       "      <td>1.82110</td>\n",
       "      <td>0.18532</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94-92cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>900</th>\n",
       "      <td>1s17_94-92cm_20um_rec_voi_0041.bmp</td>\n",
       "      <td>0.84425</td>\n",
       "      <td>262.0</td>\n",
       "      <td>136.94602</td>\n",
       "      <td>45.32307</td>\n",
       "      <td>33.09557</td>\n",
       "      <td>46.80068</td>\n",
       "      <td>564.17989</td>\n",
       "      <td>12.44796</td>\n",
       "      <td>0.17299</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>136.84910</td>\n",
       "      <td>0.22729</td>\n",
       "      <td>0.16067</td>\n",
       "      <td>0.32480</td>\n",
       "      <td>2.05986</td>\n",
       "      <td>1.82226</td>\n",
       "      <td>0.22651</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94-92cm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>901</th>\n",
       "      <td>1s17_94-92cm_20um_rec_voi_0040.bmp</td>\n",
       "      <td>0.82366</td>\n",
       "      <td>282.0</td>\n",
       "      <td>136.94602</td>\n",
       "      <td>45.08462</td>\n",
       "      <td>32.92146</td>\n",
       "      <td>46.80068</td>\n",
       "      <td>569.82500</td>\n",
       "      <td>12.63901</td>\n",
       "      <td>0.15987</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>134.29096</td>\n",
       "      <td>0.22255</td>\n",
       "      <td>0.15824</td>\n",
       "      <td>0.32242</td>\n",
       "      <td>2.08047</td>\n",
       "      <td>1.82356</td>\n",
       "      <td>0.26769</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94-92cm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50303 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              File Name     Pos.Z  Obj.N       T.Ar    Obj.Ar  \\\n",
       "1      1s17_5-3cm_20um_rec_voi_1040.bmp  21.41504  149.0   93.99345  30.53807   \n",
       "2      1s17_5-3cm_20um_rec_voi_1039.bmp  21.39445  151.0   93.99345  30.49222   \n",
       "3      1s17_5-3cm_20um_rec_voi_1038.bmp  21.37386  153.0   93.99345  30.61306   \n",
       "4      1s17_5-3cm_20um_rec_voi_1037.bmp  21.35327  152.0   93.99345  30.81860   \n",
       "5      1s17_5-3cm_20um_rec_voi_1036.bmp  21.33267  138.0   93.99345  31.11482   \n",
       "..                                  ...       ...    ...        ...       ...   \n",
       "897  1s17_94-92cm_20um_rec_voi_0044.bmp   0.90602  262.0  136.94602  45.29154   \n",
       "898  1s17_94-92cm_20um_rec_voi_0043.bmp   0.88543  258.0  136.94602  45.42197   \n",
       "899  1s17_94-92cm_20um_rec_voi_0042.bmp   0.86484  253.0  136.94602  45.48435   \n",
       "900  1s17_94-92cm_20um_rec_voi_0041.bmp   0.84425  262.0  136.94602  45.32307   \n",
       "901  1s17_94-92cm_20um_rec_voi_0040.bmp   0.82366  282.0  136.94602  45.08462   \n",
       "\n",
       "    Obj.Ar/T.Ar      T.Pm     Obj.Pm Obj.Pm/Obj.Ar Av.Obj.Ar  ... MMI(min)  \\\n",
       "1      32.48957  36.26756  339.58716      11.12013   0.20495  ...      NaN   \n",
       "2      32.44079  36.26756  339.39845      11.13066   0.20194  ...      NaN   \n",
       "3      32.56936  36.26756  343.01012      11.20470   0.20009  ...      NaN   \n",
       "4      32.78803  36.26756  341.37986      11.07707   0.20275  ...      NaN   \n",
       "5      33.10318  36.26756  340.59603      10.94642   0.22547  ...      NaN   \n",
       "..          ...       ...        ...           ...       ...  ...      ...   \n",
       "897    33.07255  46.80068  562.56584      12.42099   0.17287  ...      NaN   \n",
       "898    33.16779  46.80068  561.18839      12.35500   0.17605  ...      NaN   \n",
       "899    33.21334  46.80068  563.00421      12.37798   0.17978  ...      NaN   \n",
       "900    33.09557  46.80068  564.17989      12.44796   0.17299  ...      NaN   \n",
       "901    32.92146  46.80068  569.82500      12.63901   0.15987  ...      NaN   \n",
       "\n",
       "     T.Or(phi)      Ecc  St.Th(pl) St.Sp(pl) St.Li.Dn(pl)       FD      i.Pm  \\\n",
       "1    166.54467  0.36757    0.17985   0.37372      1.80644  1.54754  12.48324   \n",
       "2    166.41858  0.39134    0.17968   0.37420      1.80544  1.69069  12.83122   \n",
       "3    165.30597  0.40363    0.17850   0.36955      1.82465  1.55996  13.28716   \n",
       "4    164.40286  0.41216    0.18055   0.37011      1.81598  1.67885  13.16008   \n",
       "5    163.18175  0.41304    0.18271   0.36923      1.81181  1.68291  12.80607   \n",
       "..         ...      ...        ...       ...          ...      ...       ...   \n",
       "897  137.80450  0.19643    0.16102   0.32584      2.05397  1.76204   0.08237   \n",
       "898  136.29174  0.22195    0.16188   0.32618      2.04894  1.82027   0.12355   \n",
       "899  137.22689  0.23615    0.16158   0.32491      2.05557  1.82110   0.18532   \n",
       "900  136.84910  0.22729    0.16067   0.32480      2.05986  1.82226   0.22651   \n",
       "901  134.29096  0.22255    0.15824   0.32242      2.08047  1.82356   0.26769   \n",
       "\n",
       "    Unnamed: 39 Scan Depth  \n",
       "1           NaN      5-3cm  \n",
       "2           NaN      5-3cm  \n",
       "3           NaN      5-3cm  \n",
       "4           NaN      5-3cm  \n",
       "5           NaN      5-3cm  \n",
       "..          ...        ...  \n",
       "897         NaN    94-92cm  \n",
       "898         NaN    94-92cm  \n",
       "899         NaN    94-92cm  \n",
       "900         NaN    94-92cm  \n",
       "901         NaN    94-92cm  \n",
       "\n",
       "[50303 rows x 41 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main(files, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}