main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data management imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as kungfupanda                    # pandas for data manipulation and markdown\n",
    "from pandas import DataFrame                    # export\n",
    "\n",
    "import argparse                                 # For command line arguments when calling py script with flags\n",
    "import pickle                                   # for saving/loading json records and file \n",
    "                                                # modification date history\n",
    "import json\n",
    "\n",
    "from questionDataclass import questionDataclass as Question"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OS and directory management imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir                          # for file retrieval and path calculations\n",
    "from os.path import isfile, join\n",
    "from os import stat\n",
    "\n",
    "from os.path import isdir                       # for creation of topic markdown folder if \n",
    "from os import mkdir                            # not present\n",
    "\n",
    "from os import getcwd                           # gets current working DIR for calculating git \n",
    "                                                # root of submissions folder \n",
    "\n",
    "from os import chdir                            # for changing the working directory to ensure\n",
    "from os.path import abspath, dirname            # relative paths used are from this script's\n",
    "import sys                                      # location rather than the calling location\n",
    "                                                # e.g. if you call `python someFolder/main.py`\n",
    "                                                #      then it will still work."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Environment variable imports + file log and creation time imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import getenv, environ                  # for environment variables\n",
    "from dotenv import load_dotenv, find_dotenv     # for config purposes (.env file)\n",
    "\n",
    "import subprocess                               # tracing git log history for ctimes and mtimes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os.path import getmtime, getctime          # retreiving file creation/modification times\n",
    "from datetime import datetime, timedelta\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "QOL and anti-reundancy imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Set, Dict, List, Tuple       # misc. QOL imports\n",
    "from collections import defaultdict\n",
    "from icecream import ic                         # for debugging / outputs\n",
    "\n",
    "import re                                       # for regex file name matching / question number matching\n",
    "from functools import cache                     # for redundancy protection\n",
    "\n",
    "# TQDM import done separately below after checking if this is a .py or .ipynb file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Script Configuration\n",
    "#### `.env` variables and `working directories`\n",
    "1. Loads `env` variables for reference.\n",
    "    1. Tries to retrieve it from `../` if found (prioritizing template).\n",
    "    2. If failure, use the `.env` found in the current script directory (in the updater).\n",
    "2. If is a script run, denotes it as such for script flag references and ensures working directory is the script's location rather than the calling directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default .env activated from script directory (.readme_updater/)\n"
     ]
    }
   ],
   "source": [
    "# loading env variables\n",
    "print('Default .env activated from script directory (.readme_updater/)')\n",
    "load_dotenv(find_dotenv(), override=True)\n",
    "\n",
    "if '.env' in listdir('../') :\n",
    "    print('.env found in ../ directory. Overriding default...')\n",
    "    load_dotenv(find_dotenv('../.env'), override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working directory already set to script location. No need for adjustment\n"
     ]
    }
   ],
   "source": [
    "# NOTE: if the script is being run from a jupyter notebook, then it should\n",
    "# already be in the correct directory.\n",
    "IS_NOTEBOOK = True\n",
    "try:\n",
    "    if 'ipykernel' not in sys.modules:\n",
    "        print('Working directory being set to script location.')\n",
    "        IS_NOTEBOOK = False\n",
    "        chdir(dirname(abspath(__file__)))\n",
    "    else :\n",
    "        print('Working directory already set to script location. No need for adjustment')\n",
    "except NameError:\n",
    "    print('NameError')\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import env_config as config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TQDM import based off if current running script is a jupyter notebook\n",
    "# or a python script\n",
    "\n",
    "if IS_NOTEBOOK :\n",
    "    print('Importing tqdm.notebook')\n",
    "    from tqdm.notebook import tqdm\n",
    "else :\n",
    "    print('Importing tqdm (non-notebook)')\n",
    "    from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# README_ABS_DIR will get confirmed in if name==main prior to running\n",
    "README_ABS_DIR      = getcwd().replace('\\\\', '/')\n",
    "NOTEBOOK_ABS_DIR    = README_ABS_DIR\n",
    "MAIN_DIRECTORY      = NOTEBOOK_ABS_DIR[NOTEBOOK_ABS_DIR.rfind('/')+1:]\n",
    "\n",
    "print(f'{NOTEBOOK_ABS_DIR = }')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Configed separately so that values are static and global in all cases with less \n",
    "chance of error e.g. forgetting the global keyboard declaration.\n",
    "\n",
    "Also allows for us to establish the values dependent on the .env AFTER\n",
    "we have a chance to modify them e.g. if there's a user input for a different \n",
    "directory for solution files.\n",
    "'''\n",
    "def set_env_linked_constants() -> None :\n",
    "    config.README_PATH                      = getenv('README_PATH')\n",
    "    config.LEETCODE_PATH_FROM_README        = getenv('QUESTIONS_PATH_FROM_README')\n",
    "    config.LEETCODE_PATH_REFERENCE          = join(config.README_PATH, \n",
    "                                                   config.LEETCODE_PATH_FROM_README)\n",
    "    \n",
    "    config.QUESTION_DATA_FOLDER             = join(getenv('SUBMODULE_DATA_PATH'), \n",
    "                                                   getenv('LEETCODE_QUESTION_DETAILS'))\n",
    "    config.SUBMODULE_DATA_PATH              = getenv('SUBMODULE_DATA_PATH')    \n",
    "    \n",
    "    config.HISTORY_PATH                     = join(getenv('USER_DATA_PATH'), \n",
    "                                                   getenv('FILE_MODIFICATION_NAME'))\n",
    "    \n",
    "    config.DAILIES_DATA_PATH                = join(getenv('SUBMODULE_DATA_PATH'), \n",
    "                                                   getenv('DAILIES_FOLDER'), \n",
    "                                                   getenv('DAILIES_FILE'))\n",
    "    config.WEEKLIES_DATA_PATH               = join(getenv('SUBMODULE_DATA_PATH'), \n",
    "                                                   getenv('DAILIES_FOLDER'), \n",
    "                                                   getenv('WEEKLIES_FILE'))\n",
    "\n",
    "    config.LISTSDIR                         = getenv('LISTS_LOCATION')\n",
    "    config.QUESTIONS_FOLDER_FROM_README     = getenv('QUESTIONS_PATH_FROM_README')\n",
    "    config.QUESTIONS_FOLDER                 = join(config.README_PATH, \n",
    "                                                   config.QUESTIONS_FOLDER_FROM_README)\n",
    "\n",
    "    config.MARKDOWN_PATH                    = getenv('QUESTION_MARKDOWNS_PATH_FROM_README')\n",
    "    config.MARKDOWN_TO_SUBMISSIONS          = '../' * config.MARKDOWN_PATH.count('/') + \\\n",
    "                                              config.QUESTIONS_FOLDER_FROM_README\n",
    "\n",
    "    config.QUESTION_DATA_FOLDER_PATH        = getenv('QUESTION_DATA_PATH')\n",
    "    config.QUESTION_TOPICS_FILE             = getenv('LEETCODE_QUESTION_TOPICS')\n",
    "    config.QUESTION_DETAILS_FILE            = getenv('LEETCODE_QUESTION_DETAILS')\n",
    "\n",
    "    config.BY_TOPIC_FOLDER_PATH             = getenv('TOPIC_MARKDOWN_PATH_IN_MARKDOWNS_FOLDER')\n",
    "    \n",
    "    config.TOPIC_FOLDER                     = getenv('TOPIC_MARKDOWN_PATH_IN_MARKDOWNS_FOLDER')\n",
    "\n",
    "    # For each topic case\n",
    "    config.NOTEBOOK_PATH                    = join(config.README_PATH, \n",
    "                                                   config.MARKDOWN_PATH, \n",
    "                                                   config.TOPIC_FOLDER)\n",
    "\n",
    "    # For the overal hosting markdown\n",
    "    config.OVERALL_FILE_NOTEBOOK_PATH       = join(config.README_PATH, \n",
    "                                                   config.MARKDOWN_PATH, \n",
    "                                                   'Topics.md')\n",
    "    config.OVERALL_FILE_README_PATH         = join(config.MARKDOWN_PATH, \n",
    "                                                   'Topics.md')\n",
    "\n",
    "    config.DIFFICULTY_MARKDOWNS_PATH        = config.MARKDOWN_PATH\n",
    "    config.DAILY_URL                        = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Methods\n",
    "\n",
    "AddCase $\\rightarrow$ takes information for a new question file and formats it accordingly for a row.\n",
    "\n",
    "UpdateLanguage $\\rightarrow$ if a question already has a solution, this is called instead to insert the new file link to the existing row details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ALL_GIT_CM_TIMES = {}\n",
    "USE_GIT_DATES = False\n",
    "\n",
    "with open('question_data/language_equivs.json') as f :\n",
    "    LANGUAGE_EQUIVS = json.load(f)\n",
    "\n",
    "COLUMNS = [\n",
    "            '#',\n",
    "            'Title', \n",
    "            'Level',\n",
    "            'Cats',\n",
    "            'Solution',\n",
    "            'Languages',\n",
    "            'Date Complete'\n",
    "          ]\n",
    "\n",
    "TYPE_CLARIFICATION = {\n",
    "                      '#':                  int,\n",
    "                      'Title':              str, \n",
    "                      'Level':              str,\n",
    "                      'Cats':               str,\n",
    "                      'Solution':           str,\n",
    "                      'Languages':          str,\n",
    "                      'Date Complete':      str\n",
    "                    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def individualCTimeViaGit(cmd: List[str]) -> Tuple[datetime, datetime] :\n",
    "    process = subprocess.Popen(cmd,\n",
    "                               shell=False,\n",
    "                               stdin=None,\n",
    "                               stdout=subprocess.PIPE,\n",
    "                               stderr=subprocess.PIPE)\n",
    "    result        = process.stdout.readlines()\n",
    "    modifiedTimes = []\n",
    "    \n",
    "    for line in result:\n",
    "        temp = line.decode(\"utf-8\").replace('\\n', '')\n",
    "    \n",
    "        # In case of a redundant '\\n' at the end of an output\n",
    "        if temp :\n",
    "            modifiedTimes.append(temp)\n",
    "    \n",
    "    try :\n",
    "        creationDate = datetime.strptime(time.ctime(int(min(modifiedTimes))), '%a %b %d %H:%M:%S %Y')\n",
    "        modifiedDate = datetime.strptime(time.ctime(int(max(modifiedTimes))), '%a %b %d %H:%M:%S %Y')\n",
    "        \n",
    "    except ValueError as ve:\n",
    "        print(f'Error in parsing {cmd}')\n",
    "        print(f'{modifiedTimes}')\n",
    "        print(ve)\n",
    "        exit()\n",
    "    \n",
    "    return (creationDate, modifiedDate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def getAllCTimesViaGit(paths: List[str]) -> Dict[str, Tuple[datetime, datetime]] :\n",
    "    '''\n",
    "    WARNING: DO NOT USE LOCALLY. SLOW IF RAN LOCALLY.\n",
    "    \n",
    "    GITHUB ACTIONS ARE ABLE TO PERFORM THIS QUICKLY (~10s for the script for ~700 files) \n",
    "    BUT A LOCAL RUN OF `-g` CAN TAKE UPWARDS OF 10 MINUTES FOR THE SAMEN NUMBER OF FILES.\n",
    "\n",
    "    To avoid having to constantly swap directories, this function parses all the ctimes and mtimes \n",
    "    in one block of time. This gets activated with the `-g` flag. Default otherwise is to use the \n",
    "    regular `getctime` and `getmtime` functions locally which is much much faster. This only exists \n",
    "    to compensate for the inability for ctime and mtime checking with git actions.\n",
    "    '''\n",
    "    print(f'Beginning parsing of git logs for file creation and modification dates...')\n",
    "    print(f'Script path: {getcwd() = }')\n",
    "    chdir('../')\n",
    "    print(f'README path: {getcwd() = }')\n",
    "\n",
    "    cmd = r\"git log -M --format=%ct --reverse --\".split()\n",
    "    # cmd = r\"git log -M --follow --format=%ct --reverse --\".split()\n",
    "    # cmd = r\"git log --follow --format=%ct --reverse --\".split()\n",
    "    output = {}\n",
    "    \n",
    "    oldest_date = datetime.now()\n",
    "\n",
    "    # if getenv(\"GITHUB_ACTIONS\") == 'true' :\n",
    "    #     print('\\n')\n",
    "    #     print('/' + '=' * 15 + '\\\\')\n",
    "    #     print(' ', end='')\n",
    "    #     pqBarsPrinted = 0\n",
    "    #     for i, path in enumerate(paths) :\n",
    "    #         path = join(LEETCODE_PATH_FROM_README, path)\n",
    "    #         output[path] = individualCTimeViaGit(cmd + [path])\n",
    "            \n",
    "    #         if output[path][0] < oldest_date :\n",
    "    #             oldest_date = output[path][0]\n",
    "            \n",
    "    #         curChunk = int((i / len(paths)) * 15)\n",
    "    #         if curChunk > pqBarsPrinted :\n",
    "    #             print('=' * (curChunk - pqBarsPrinted), end='')\n",
    "    #             pqBarsPrinted = curChunk\n",
    "                \n",
    "    #     print((15 - pqBarsPrinted) * '=', '\\n\\n')\n",
    "    # else :\n",
    "    \n",
    "    \n",
    "    with tqdm(total=len(paths)) as pbar :\n",
    "    # with tqdm(total=len(paths), position=0, leave=True) as pbar :\n",
    "        for i, path in enumerate(paths) :\n",
    "            path = join(config.LEETCODE_PATH_FROM_README, path)\n",
    "            output[path] = individualCTimeViaGit(cmd + [path])\n",
    "            \n",
    "            if output[path][0] < oldest_date :\n",
    "                oldest_date = output[path][0]\n",
    "            \n",
    "            pbar.update(1)\n",
    "\n",
    "    config._oldest_date = oldest_date.replace(hour=0, minute=0, second=0, microsecond=0)\n",
    "\n",
    "    # Usually I'd avoid using global for this but this is a personal project so it should be fine.\n",
    "    _ALL_GIT_CM_TIMES.update(output)\n",
    "    print(f'{_ALL_GIT_CM_TIMES = }')\n",
    "    \n",
    "    chdir(MAIN_DIRECTORY)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def getCtimesMtimesGitHistory(path: str) -> Tuple[datetime, datetime] :\n",
    "    '''\n",
    "    WARNING: DO NOT USE LOCALLY. SLOW IF RAN LOCALLY RELATIVE TO THE REGULAR CTIME FUNCTION.\n",
    "\n",
    "    IF RUNNING LOCALLY, RUN (getCtimeMtimesMain) I.E. WITHOUT THE `-g` FLAG.\n",
    "\n",
    "    The cost for a single file isn't significant however when you reach ~100+ files, \n",
    "    the cumulative wait can go into the minutes compared to the seconds it would take \n",
    "    with the regular `getctime` and `getmtime` functions (without the `-g` flag)\n",
    "    '''\n",
    "    path = path[path.find('/') + 1:]\n",
    "    chdir('../')\n",
    "    cmd = r\"git log --follow --format=%ct --reverse --\".split() + [f'{path}']\n",
    "\n",
    "    cmDates = individualCTimeViaGit(cmd)\n",
    "\n",
    "    chdir(MAIN_DIRECTORY)\n",
    "\n",
    "    return cmDates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def getCtimeMtimesMain(path: str) -> Tuple[datetime, datetime] :\n",
    "    '''\n",
    "    Returns the a tuple containing the datetime objs of \n",
    "    (create date and time, modification date and time)\n",
    "\n",
    "    @param useGitDates: bool = False\n",
    "        If true, it will track the creation/modification dates of the file \n",
    "        according to the git history. This is mainly to counter the issue in \n",
    "        GitHub actions where the file creation date is the time of the action.\n",
    "    '''\n",
    "    \n",
    "    if USE_GIT_DATES :\n",
    "        return getCtimesMtimesGitHistory(path)\n",
    "    \n",
    "    creation_date = time.ctime(getctime(path))\n",
    "    modification_date = time.ctime(getmtime(path))\n",
    "\n",
    "    creation_date = datetime.strptime(creation_date, \"%a %b %d %H:%M:%S %Y\")\n",
    "    modification_date = datetime.strptime(modification_date, \"%a %b %d %H:%M:%S %Y\")\n",
    "\n",
    "    # I've sometimes encountered weird meta data issues so just as a precaution\n",
    "    if creation_date > modification_date :\n",
    "        return (modification_date, creation_date)\n",
    "    \n",
    "    return (creation_date, modification_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCtimeMtimes(path: str, \n",
    "                   *, \n",
    "                   preCalculated: Dict[str, Tuple[datetime, datetime]] = None) -> Tuple[datetime, datetime] :\n",
    "    # Due to readme realtive and script relative paths\n",
    "    readme_path = path if ('../' not in path) else path[path.find('../') + len('../'):]\n",
    "    if _ALL_GIT_CM_TIMES and readme_path in _ALL_GIT_CM_TIMES :\n",
    "        return _ALL_GIT_CM_TIMES[readme_path]\n",
    "    \n",
    "    if preCalculated and readme_path in preCalculated :\n",
    "        return preCalculated[readme_path]\n",
    "\n",
    "    return getCtimeMtimesMain(path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addCase(level:              str,\n",
    "            number:             int, \n",
    "            title:              str, \n",
    "            categories:         Set[str],\n",
    "            language:           str,\n",
    "            notebook_path:      str,\n",
    "            readme_path:        str,\n",
    "            fileLatestTimes:    dict,\n",
    "            contestTitle:       str=None,\n",
    "            contestQNo:         str=None) -> dict :\n",
    "    '''\n",
    "    Takes the data found on a question not encountered before and \n",
    "    converts it into a callable dictionary with all the relevant \n",
    "    information\n",
    "\n",
    "    ### Parameters (Required) :\n",
    "    level : str\n",
    "        Difficulty indicator of the question (e, m, h)\n",
    "    number : int\n",
    "        The official LeetCode question number\n",
    "    title : str\n",
    "        The title of the question (colloquial name)\n",
    "    categories : Set[str]\n",
    "        The categories the question falls under (e.g. Contest, Daily, etc.)\n",
    "    language : str\n",
    "        The programming language used to solve the question\n",
    "    notebook_path : str\n",
    "        The path from the main.py/ipynb script to the code file in question\n",
    "    readme_path : str\n",
    "        The path from the README.md file to be exported to the code file in question\n",
    "    fileLatestTimes : dict\n",
    "        A dictionary containing the latest modification times of all files\n",
    "        in the repository\n",
    "    \n",
    "    ### Parameters (Optional) :\n",
    "    contestTitle : str\n",
    "        The title of the contest the question was a part of if applicable\n",
    "    contestQNo : str\n",
    "        The question number in the contest if applicable (e.g. q1, q2, etc.)\n",
    "\n",
    "    ### Returns :\n",
    "    output : dict\n",
    "        A dictionary containing all the relevant information for the question\n",
    "        to be used in the final output\n",
    "    '''\n",
    "\n",
    "    creation_date, modification_date = getCtimeMtimes(notebook_path)\n",
    "    fileLatestTimes[readme_path]     = modification_date\n",
    "\n",
    "    try :\n",
    "        fileSize = stat(notebook_path).st_size\n",
    "    except FileNotFoundError as fnfe :\n",
    "        fileSize = 0\n",
    "        print(fnfe)\n",
    "    \n",
    "\n",
    "    if not categories :\n",
    "        categories = set()\n",
    "\n",
    "    match level[0].lower() :\n",
    "        case 'e' :\n",
    "            level = 'Easy'\n",
    "        case 'm' : \n",
    "            level = 'Medium'\n",
    "        case 'h' :\n",
    "            level = 'Hard'\n",
    "        case _ :\n",
    "            level = 'Unknown'\n",
    "\n",
    "    output = {\n",
    "                'level':                level,\n",
    "                'number':               number,\n",
    "                'title':                title, \n",
    "                'categories':           categories,\n",
    "                'contestTitle':         contestTitle,\n",
    "                'contestQNo':           contestQNo,\n",
    "                'date_done':            creation_date,          # First time completed\n",
    "                'date_modified':        modification_date,      # Most recent date\n",
    "                'solution':             '',\n",
    "                'solutions':            {language: [readme_path]},\n",
    "                'languages':            set([language]),\n",
    "                'bytes':                fileSize\n",
    "             }\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateQuestion(orig:               dict, \n",
    "                   *,\n",
    "                   language:           str,\n",
    "                   categories:         Set[str],\n",
    "                   notebook_path:      str,\n",
    "                   readme_path:        str,\n",
    "                   fileLatestTimes:    dict,\n",
    "                   contestTitle:       str=None,\n",
    "                   contestQNo:         str=None) -> dict :\n",
    "    '''\n",
    "    Takes question data of a question that's already been encountered and \n",
    "    updates the relevant dictionary with the new information found. Similar \n",
    "    to addCase but for questions that have already been encountered.\n",
    "\n",
    "    ### Parameters :\n",
    "    orig : dict\n",
    "        The original dictionary containing all the relevant information from previous encounters\n",
    "    \n",
    "    All other parameters are the same as addCase and are optional in order to update them.\n",
    "        \n",
    "    ### Returns :\n",
    "    orig : dict\n",
    "        The updated dictionary containing all the relevant information from previous encounters\n",
    "    '''\n",
    "    \n",
    "    # Another question file found\n",
    "    if language and language not in orig['languages'] :\n",
    "        orig['languages'].add(language)\n",
    "\n",
    "    if contestTitle :\n",
    "        orig['contestTitle'] = contestTitle\n",
    "        \n",
    "    if contestQNo :\n",
    "        orig['contestQNo'] = contestQNo\n",
    "          \n",
    "    if categories :\n",
    "        orig['categories'] |= categories\n",
    "\n",
    "    if notebook_path and readme_path :\n",
    "        creation_date, modification_date = getCtimeMtimes(notebook_path)\n",
    "        \n",
    "        if creation_date < orig['date_done'] :\n",
    "            orig['date_done'] = creation_date\n",
    "        if modification_date > orig['date_modified'] :\n",
    "            orig['date_modified'] = modification_date\n",
    "            fileLatestTimes[readme_path] = modification_date\n",
    "\n",
    "        if language not in orig['solutions'] :\n",
    "            orig['solutions'][language] = []\n",
    "        orig['solutions'][language].append(readme_path)\n",
    "\n",
    "        try :\n",
    "            fileSize = stat(notebook_path).st_size\n",
    "        except FileNotFoundError as fnfe :\n",
    "            fileSize = 0\n",
    "            print(fnfe)\n",
    "        orig['bytes'] += fileSize\n",
    "  \n",
    "    return orig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pickle Processes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def retrieveQuestionDetails() -> dict :\n",
    "    '''\n",
    "    Retrieves the question details (i.e. title, acRates, difficulties, etc.) from\n",
    "    the relevant `.pkl` file containing parsed official LeetCode json data.\n",
    "\n",
    "    ### Returns :\n",
    "    questionDetailsDict : dict[int, details]\n",
    "        A dictionary containing the question details matched to the question's assigned number\n",
    "    '''\n",
    "\n",
    "    print(f'{getcwd() = }')\n",
    "    \n",
    "    print(f'{listdir(config.SUBMODULE_DATA_PATH) = }')\n",
    "    print(f'Question details path: {config.QUESTION_DATA_FOLDER = }')\n",
    "\n",
    "    if not isfile(config.QUESTION_DATA_FOLDER) :\n",
    "        print('\\nError in parsing official question data. leetcode.pkl not found. Exiting...')\n",
    "        print()\n",
    "        exit()\n",
    "    else : \n",
    "        print('\\nFiles found. Importing now...\\n')\n",
    "\n",
    "\n",
    "    # schema: key=int(questionNumber)   val=(title, titleSlug, paidOnly, difficulty, acRate)\n",
    "    with open(join(config.QUESTION_DATA_FOLDER), 'rb') as fp:\n",
    "        questionDetailsDict = pickle.load(fp)\n",
    "\n",
    "    print(f'{questionDetailsDict = }')\n",
    "\n",
    "    return questionDetailsDict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def writeRecentFileTimes(fileLatestTimes: dict) -> bool :\n",
    "    '''Pickles the newly found most recent modification times of each question for reference in future runs'''\n",
    "    \n",
    "    with open(config.HISTORY_PATH, 'wb') as fp:\n",
    "        pickle.dump(fileLatestTimes, fp)\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getRecentFileTimes() -> dict :\n",
    "    '''Retrieves the pickled data from previous cases of `writeRecentFileTimes()`'''\n",
    "\n",
    "    if isfile(config.HISTORY_PATH) :\n",
    "        with open(config.HISTORY_PATH, 'rb') as fp:\n",
    "            return pickle.load(fp)\n",
    "        \n",
    "    return {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Daily and Weekly Challenges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getDailies(firstDate: datetime = None) -> List[Tuple[datetime, int]] :\n",
    "    '''\n",
    "    Retrieves the daily questions from the official LeetCode json data\n",
    "    and returns them as a set of strings\n",
    "\n",
    "    ### Returns :\n",
    "    dailies : List[Tuple[date, questionNo]]\n",
    "    '''\n",
    "    if not firstDate :\n",
    "        firstDate = config._oldest_date\n",
    "    \n",
    "    print('Oldest date found:', firstDate)\n",
    "    \n",
    "    with open(config.DAILIES_DATA_PATH, 'rb') as fp:\n",
    "        dailies = json.load(fp)\n",
    "    \n",
    "    output = []\n",
    "\n",
    "    for k in list(dailies.keys()) :\n",
    "        newK = datetime.strptime(k, '%Y-%m-%d')\n",
    "        if newK < firstDate :\n",
    "            continue\n",
    "\n",
    "        output.append((newK, int(dailies[k]['question']['questionFrontendId'])))\n",
    "\n",
    "    return sorted(output, key=lambda x: x[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getWeeklies(firstDate: datetime = None) -> List[Tuple[datetime, int]] :\n",
    "    '''\n",
    "    Retrieves the weekly premium questions from the official LeetCode json data\n",
    "    and returns them as a set of strings\n",
    "\n",
    "    ### Returns :\n",
    "    weeklies : List[Tuple[date, questionNo]]\n",
    "    '''\n",
    "    if not firstDate :\n",
    "        firstDate = config._oldest_date\n",
    "    \n",
    "    print('Oldest date found:', firstDate)\n",
    "    \n",
    "    with open(config.WEEKLIES_DATA_PATH, 'rb') as fp:\n",
    "        weeklies = json.load(fp)\n",
    "    \n",
    "    output = []\n",
    "\n",
    "    for k in list(weeklies.keys()) :\n",
    "        newK = datetime.strptime(k, '%Y-%m-%d')\n",
    "        if newK < firstDate :\n",
    "            continue\n",
    "\n",
    "        output.append((newK, int(weeklies[k]['question']['questionFrontendId'])))\n",
    "\n",
    "    return sorted(output, key=lambda x: x[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseQuestionsForDailies(questionData: dict) -> Dict :\n",
    "    '''\n",
    "    Parses the official LeetCode json data for the daily and weekly premium questions\n",
    "    and returns them as a dictionary of question numbers to question objects\n",
    "\n",
    "    ### Returns :\n",
    "    dailies : Dict[int, Question]\n",
    "        A dictionary containing the daily questions as question objects\n",
    "    weeklies : Dict[int, Question]\n",
    "        A dictionary containing the weekly premium questions as question objects\n",
    "    '''\n",
    "    dailies = getDailies()\n",
    "    dailiesDict = {}\n",
    "\n",
    "    # I've added ~12 hours of leeway due to potential to forget to commit\n",
    "    for date, qNo in dailies :\n",
    "        if qNo in questionData and questionData[qNo]['date_done'] <= date + timedelta(days=1, hours=12) :\n",
    "            dailiesDict[date] = questionData[qNo].copy()\n",
    "            dailiesDict[date]['date_done'] = date\n",
    "            questionData[qNo]['categories'].add('Daily')\n",
    "\n",
    "    return dailiesDict\n",
    "\n",
    "\n",
    "# TODO: calendar that's clickable to go to that daily question's page\n",
    "# ISSUE: markdown will negate any spacing by the calendar package \n",
    "#        so we have to export to a markdown table\n",
    "\n",
    "import calendar\n",
    "\n",
    "# Input data dict for the day of that month and the string it should be instead\n",
    "# def calendar_template_maker(year: int, month: int, input_data: Dict[int, str]) -> str :\n",
    "#     output_cal = []\n",
    "    \n",
    "#     month_range = calendar.monthrange(year, month)\n",
    "#     first_day = calendar.\n",
    "    \n",
    "    \n",
    "\n",
    "# Output: [current month, all months]\n",
    "def get_dailies_month_maps(dailiesDict: Dict[datetime, Question]) -> Tuple[str, str] :\n",
    "    earliest_date = min(dailiesDict.keys())\n",
    "    \n",
    "    outputs = []\n",
    "    \n",
    "    # Sorted by year then month then day\n",
    "    \n",
    "    daily_data_mapped = {}\n",
    "    for date, question_data in dailiesDict.items() :\n",
    "        curr = daily_data_mapped\n",
    "        if date.year not in curr :\n",
    "            daily_data_mapped[date.year] = {}\n",
    "        curr = daily_data_mapped[date.year]\n",
    "        \n",
    "        if date.month not in curr :\n",
    "            curr[date.month] = {}\n",
    "        curr = curr[date.month]\n",
    "        \n",
    "        curr[date.day] = question_data\n",
    "        \n",
    "    calendar.setfirstweekday(calendar.SUNDAY)\n",
    "    \n",
    "    for year in range(earliest_date.year, datetime.now().year + 1) :\n",
    "        if year == earliest_date.year :\n",
    "            start_month = earliest_date.month\n",
    "        else :\n",
    "            start_month = 1\n",
    "        \n",
    "        if year == datetime.now().year :\n",
    "            end_month = datetime.now().month\n",
    "        else :\n",
    "            end_month = 12\n",
    "            \n",
    "        for month in range(start_month, end_month + 1) :\n",
    "            this_month = []\n",
    "            curr = daily_data_mapped[year][month]\n",
    "            cal = calendar.month(year, month)\n",
    "            \n",
    "            # print(curr.keys())\n",
    "            \n",
    "            # for day, question_data in curr.items() :\n",
    "            #     cal = re.sub(rf'\\b{day}\\b', f'[{day}](<{question_data[\"solution\"]}>)', cal)\n",
    "            \n",
    "            cal = cal.split('\\n')\n",
    "            # print(cal)\n",
    "            # print()\n",
    "            this_month.append(f'### {cal[0].strip()}')  # month header\n",
    "            days = cal[1].split()\n",
    "            cal = cal[2:]\n",
    "            \n",
    "            # (' ' if len(y) == 1 else '') + \n",
    "            cal = [[(y if int(y) not in curr else f'[{y}](<{curr[int(y)][\"solution\"]}>)') for y in x.split()] for x in cal]\n",
    "            if len(cal[0]) < 7 :\n",
    "                cal[0] = [''] * (7 - len(cal[0])) + cal[0]\n",
    "            \n",
    "            if not cal[-1] :\n",
    "                cal.pop()\n",
    "            \n",
    "            df = kungfupanda.DataFrame(cal, columns=days)\n",
    "            \n",
    "            markdown_cal =df.to_markdown(index=False)\n",
    "            \n",
    "            col_alignments = sorted(re.findall(r':{1}-{1}-*', markdown_cal), key=lambda x: len(x), reverse=True)\n",
    "            for ca in col_alignments :\n",
    "                markdown_cal = markdown_cal.replace(ca, '-' * (len(ca) - 1) + ':')\n",
    "            \n",
    "            this_month.append(markdown_cal)\n",
    "            this_month.append('\\n')\n",
    "            outputs.append('\\n'.join(this_month))\n",
    "        \n",
    "    return (outputs[-1], '\\n'.join(outputs))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseQuestionsForWeeklies(questionData: dict) -> Dict[int, Question] :\n",
    "    '''\n",
    "    Parses the official LeetCode json data for the daily and weekly premium questions\n",
    "    and returns them as a dictionary of question numbers to question objects\n",
    "\n",
    "    ### Returns :\n",
    "    dailies : Dict[int, Question]\n",
    "        A dictionary containing the daily questions as question objects\n",
    "    weeklies : Dict[int, Question]\n",
    "        A dictionary containing the weekly premium questions as question objects\n",
    "    '''\n",
    "    weeklies = getWeeklies()\n",
    "    weekliesDict = {}\n",
    "\n",
    "    # I've given ~1 day of leeway for the weeklies in case you forget to commit\n",
    "    for date, qNo in weeklies :\n",
    "        if qNo in questionData and questionData[qNo]['date_done'] <= date + timedelta(days=8) :\n",
    "            weekliesDict[date] = questionData[qNo].copy()\n",
    "            weekliesDict[date]['date_done'] = date\n",
    "            questionData[qNo]['categories'].add('Weekly Premium')\n",
    "\n",
    "    return weekliesDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parsing Files\n",
    "Question file parsing occurs here. It organizes it into 3 different lists, separated by difficulty and sorted by question number afterwards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse one leetcode answer file in the submissions folder\n",
    "def parseCase(leetcodeFile:         str,  # file name\n",
    "              questionData:         dict, # dictionary of question data\n",
    "              fileLatestTimes:      dict,\n",
    "              reprocessMarkdown:    set,\n",
    "              *,\n",
    "              questionDetailsDict:  dict = None,\n",
    "              subFolderPath:        str = '',\n",
    "              altTitle:             str = '',\n",
    "              contest:              str = None,\n",
    "              contestQNo:           str = None) -> bool:\n",
    "\n",
    "    if not questionDetailsDict :\n",
    "        questionDetailsDict = retrieveQuestionDetails()\n",
    "\n",
    "    path = join(config.LEETCODE_PATH_FROM_README, subFolderPath, leetcodeFile).replace(\"\\\\\", \"/\")\n",
    "\n",
    "    try :\n",
    "        number      = int(re.search(\"\\d{1,4}\", leetcodeFile).group())   # Takes the first full number as the question\n",
    "        level       = questionDetailsDict[number].level                 # number and uses that as reference\n",
    "                                                                        # e.g. 'e123 v1.py' becomes 123\n",
    "    except (AttributeError) as ae :\n",
    "        print(f'Error in parsing {leetcodeFile}: {ae.name} encountered while trying to extract question number int(...).',\n",
    "                '\\nparseCase(...)',\n",
    "                '\\nSkipping')\n",
    "        return False\n",
    "    except (KeyError) as ke :\n",
    "        print(f'Error in parsing {leetcodeFile}: {ke} encountered while trying to extract question level from questionDetailsDict.',\n",
    "                '\\nparseCase(...)',\n",
    "                '\\nAttempting to pull from the name...')\n",
    "        level = leetcodeFile[0].lower()\n",
    "        \n",
    "        if level in ['e', 'm', 'h'] :\n",
    "            print(f'Level found: {level}')\n",
    "        else :\n",
    "            print(f'Level not found \\\"{level}\\\". Defaulting to \"Unknown\"')\n",
    "            level = 'Unknown'\n",
    "\n",
    "\n",
    "    creationtime, modificationtime = getCtimeMtimes(join(config.README_PATH, path))\n",
    "\n",
    "    if path not in fileLatestTimes or max(creationtime, modificationtime) > fileLatestTimes[path] :\n",
    "        reprocessMarkdown.add(number)\n",
    "        fileLatestTimes[path] = max(creationtime, modificationtime)\n",
    "        \n",
    "\n",
    "    if number in questionDetailsDict :\n",
    "        title   = f'[{questionDetailsDict[number].title}](<https://leetcode.com/problems/{questionDetailsDict[number].slug}>)'\n",
    "    else :\n",
    "        title   = f'Question {number}'\n",
    "    categories  = set()\n",
    "    language    = leetcodeFile[leetcodeFile.rfind('.') + 1:]\n",
    "\n",
    "    if len(altTitle) > 0 :\n",
    "        title = altTitle + ' - ' + title\n",
    "\n",
    "    # Question is from a contest folder\n",
    "    if contest :\n",
    "        temp = re.findall('q\\d{1}', leetcodeFile)                       # Checking if file name has a question number (e.g. q1 of the contest)\n",
    "        if not len(temp) == 0 :\n",
    "            contestQNo = temp[0]\n",
    "\n",
    "        categories.add('Contest')\n",
    "\n",
    "    if number in questionData :                                     # If solution already found for this question\n",
    "        questionData[number] = updateQuestion(questionData[number], \n",
    "                                              language=language, \n",
    "                                              categories=categories, \n",
    "                                              notebook_path=join(config.README_PATH, path), \n",
    "                                              readme_path=path,\n",
    "                                              contestTitle=contest,\n",
    "                                              contestQNo=contestQNo,\n",
    "                                              fileLatestTimes=fileLatestTimes)\n",
    "    else :\n",
    "        questionData[number] = addCase(level=level, \n",
    "                                       number=number, \n",
    "                                       title=title,\n",
    "                                       categories=categories, \n",
    "                                       language=language, \n",
    "                                       notebook_path=join(config.README_PATH, path), \n",
    "                                       readme_path=path,\n",
    "                                       contestTitle=contest,\n",
    "                                       contestQNo=contestQNo,\n",
    "                                       fileLatestTimes=fileLatestTimes)\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def getCodeFiles() -> List[str] :\n",
    "    return [\n",
    "        x for x in listdir(config.LEETCODE_PATH_REFERENCE)\n",
    "        \n",
    "        if isfile(join(config.LEETCODE_PATH_REFERENCE, x))\n",
    "            and not x.endswith('.txt')\n",
    "            and not x.endswith('.md')\n",
    "            and not x.endswith('.gitignore')\n",
    "            and '.' in x\n",
    "    ]\n",
    "\n",
    "@cache\n",
    "def getContestFolders() -> List[str] :\n",
    "    return [\n",
    "        x for x in listdir(config.LEETCODE_PATH_REFERENCE)\n",
    "        if not isfile(join(config.LEETCODE_PATH_REFERENCE, x))\n",
    "    ]\n",
    "\n",
    "@cache\n",
    "def getContextFiles(*, contestFolders: List[str] = None) -> List[str] :\n",
    "    if not contestFolders :\n",
    "        contestFolders = getContestFolders()\n",
    "    \n",
    "    output = [\n",
    "        x for x in listdir(config.LEETCODE_PATH_REFERENCE)\n",
    "        \n",
    "        if isfile(join(config.LEETCODE_PATH_REFERENCE, x)) \n",
    "            and (x.endswith('.txt') \n",
    "                    or x.endswith('.md') \n",
    "                    or '.' not in x)\n",
    "            and not x.endswith('.gitignore')\n",
    "    ]\n",
    "    \n",
    "    for folder in contestFolders :\n",
    "        output.extend([\n",
    "            join(folder, y) for y in listdir(join(config.LEETCODE_PATH_REFERENCE, folder))\n",
    "            \n",
    "            if isfile(join(config.LEETCODE_PATH_REFERENCE, folder, y))\n",
    "                and (y.endswith('.txt') \n",
    "                        or y.endswith('.md')\n",
    "                        or '.' not in y)\n",
    "                and not y.endswith('.gitignore')\n",
    "        ])\n",
    "    return output\n",
    "\n",
    "def getContestFiles(contestFolders: List[str]) -> List[Tuple[str, str]] :\n",
    "    contestLeetcodeFiles = []\n",
    "\n",
    "    for contestFolder in contestFolders :\n",
    "        contestLeetcodeFiles.extend([\n",
    "            (contestFolder, fileName) for fileName in listdir(join(config.LEETCODE_PATH_REFERENCE, \n",
    "                                                                   contestFolder))\n",
    "            \n",
    "            if isfile(join(config.LEETCODE_PATH_REFERENCE, contestFolder, fileName))\n",
    "                and not fileName.endswith('.txt')\n",
    "                and not fileName.endswith('.md')\n",
    "                and '.' in fileName\n",
    "                and not fileName.endswith('.gitignore')\n",
    "        ])\n",
    "    \n",
    "    return contestLeetcodeFiles\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sort TXT Context\n",
    "If .txt notes are placed, this adds them to their respective entry."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseContextFiles(txtFiles: str, \n",
    "                      questionData: dict,\n",
    "                      fileLatestTimes: dict, \n",
    "                      reprocessMarkdown: Set[int]) -> None:\n",
    "    for fileName in txtFiles :\n",
    "        try :\n",
    "            if '\\\\' in fileName :\n",
    "                number = int(re.search(\"\\d{1,4}\", fileName[fileName.rfind('\\\\'):]).group())\n",
    "            elif '/' in fileName :\n",
    "                number = int(re.search(\"\\d{1,4}\", fileName[fileName.rfind('/'):]).group())\n",
    "            else :\n",
    "                number = int(re.search(\"\\d{1,4}\", fileName).group())\n",
    "        except AttributeError as ae :\n",
    "            print(f'Error in parsing {fileName}: Attribute Error encountered while trying to extract question number int(...).',\n",
    "                  '\\nparseContextFiles(...)',\n",
    "                  '\\nSkipping')\n",
    "            continue\n",
    "    \n",
    "        if number not in questionData :\n",
    "            print(f'Error. No question solution found for context file ({fileName = })')\n",
    "            continue\n",
    "        \n",
    "        questionData[number]['contextFile'] = join(config.LEETCODE_PATH_FROM_README, fileName)\n",
    "        path = join(config.LEETCODE_PATH_REFERENCE, fileName)\n",
    "        \n",
    "        creationtime, modificationtime = getCtimeMtimes(path)\n",
    "        if path not in fileLatestTimes or max(creationtime, modificationtime) > fileLatestTimes[path] :\n",
    "            fileLatestTimes[path] = max(creationtime, modificationtime)\n",
    "            reprocessMarkdown.add(number)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# List-Based Categories\n",
    "Updating `Category` columns based on the lists in the `Lists` directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def getLists() -> List[str] :\n",
    "\n",
    "    listFileNames = [\n",
    "        x for x in listdir(config.LISTSDIR) \n",
    "        \n",
    "        if isfile(join(config.LISTSDIR, x)) \n",
    "            and not x.startswith('.')\n",
    "            and not x == 'README.md'\n",
    "    ]\n",
    "    \n",
    "    print(listFileNames)\n",
    "    return listFileNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Format for lists file is as follows:\n",
    "\n",
    "        [Question #]. [Question Name]\n",
    "\n",
    "        [Easy, Med., Hard]\n",
    "        Topic1\n",
    "        Topic2\n",
    "        Topic3\n",
    "        ...\n",
    "'''\n",
    "\n",
    "@cache\n",
    "def getList(fileName, filePath) -> Set[int] :\n",
    "    output = set() # can change to dict later if we want to output category info\n",
    "\n",
    "    count = 0\n",
    "    with open(filePath, 'r') as file :\n",
    "        lines = file.readlines()\n",
    "        for line in lines :\n",
    "            if re.match(r'\\d{1,4}\\.', line) :\n",
    "                count += 1\n",
    "                output.add(int(line[:line.find('.')]))\n",
    "    \n",
    "    return output\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processListData(questionData: dict,\n",
    "                    *,\n",
    "                    listFileNames: List[str] = None) -> dict :\n",
    "    \n",
    "    if not listFileNames :\n",
    "        listFileNames = getLists()\n",
    "    \n",
    "    listData = {}\n",
    "    for file in listFileNames :\n",
    "        listData[file] = getList(file, join(config.LISTSDIR, file))\n",
    "        for q in listData[file] :\n",
    "            if q in questionData :\n",
    "                questionData[q]['categories'].add(file)\n",
    "                \n",
    "    return listData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question Topic Grouping\n",
    "Parses the questions in `questionData` and adds their numbers to appropriate lists so that they can be parsed into their own lists as well as counted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCompletedQuestionsTopicLists(questionData: dict,\n",
    "                                    *,\n",
    "                                    questionTopicsDict: dict = None) -> defaultdict :\n",
    "    \n",
    "    if not questionTopicsDict :\n",
    "        questionTopicsDict = retrieveQuestionDetails()\n",
    "    \n",
    "    completedTopicLists = defaultdict(set)\n",
    "\n",
    "    for question in questionData.keys() :\n",
    "        # Shouldn't occur but just in case\n",
    "        if question not in questionTopicsDict :\n",
    "            continue\n",
    "        for topic in questionTopicsDict[question].topics :\n",
    "            completedTopicLists[topic].add(question)\n",
    "\n",
    "    return completedTopicLists\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Individual Markdown Generation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MARKDOWN_TO_SUBMISSIONS\n",
    "def generate_markdown(questionNo: int, \n",
    "                      questionData: dict,\n",
    "                      *,\n",
    "                      questionDetailsDict: dict,\n",
    "                      export: bool = False) -> str :\n",
    "    \n",
    "    if not questionDetailsDict :\n",
    "        questionDetailsDict = retrieveQuestionDetails()\n",
    "    \n",
    "    if questionNo in questionData :\n",
    "        questionData = questionData[questionNo]\n",
    "\n",
    "    title = questionData[\"title\"]\n",
    "\n",
    "    # Only if title has already been modified and matched to a LeetCode url\n",
    "    # E.g. some contest files will be unmatched\n",
    "    if '[' in questionData[\"title\"] :\n",
    "        title = title[title.find('[') + 1:title.find(']')]\n",
    "\n",
    "    title = f'{questionNo}. {title}'\n",
    "    \n",
    "    generate_file_name = f'_{title}.md'\n",
    "    generate_path = join(config.README_PATH, config.MARKDOWN_PATH, generate_file_name)\n",
    "    output_path = join(config.MARKDOWN_PATH, generate_file_name)\n",
    "    \n",
    "    questionData['solution'] = generate_file_name\n",
    "\n",
    "    if not export :\n",
    "        return generate_path\n",
    "\n",
    "    with open(generate_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# {questionNo}. {questionData[\"title\"]}\\n\\n')\n",
    "\n",
    "        date_done = questionData['date_done']\n",
    "        date_modified = questionData['date_modified']\n",
    "        \n",
    "        f.write(f'*All prompts are owned by LeetCode. To view the prompt, click the title link above.*\\n\\n')\n",
    "        \n",
    "        if questionData['contestTitle'] and questionData['contestQNo']:\n",
    "            f.write(f'*Completed during {questionData[\"contestTitle\"]} ({questionData[\"contestQNo\"]})*\\n\\n')\n",
    "\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(f'> *First completed : {date_done:%B %d, %Y}*\\n>\\n')\n",
    "        f.write(f'> *Last updated : {date_modified:%B %d, %Y}*\\n')\n",
    "\n",
    "        f.write('\\n------\\n\\n')\n",
    "\n",
    "        tpcs = 'N/A' if questionNo not in questionDetailsDict or len(questionDetailsDict[questionNo].topics) == 0 \\\n",
    "                     else ', '.join([f'[{x}](<{join(config.BY_TOPIC_FOLDER_PATH, x)}.md>)' for x in questionDetailsDict[questionNo].topics])\n",
    "        \n",
    "        f.write(f'> **Related Topics** : **{tpcs}**\\n>\\n')\n",
    "\n",
    "        acrate = 'Unknown' if questionNo not in questionDetailsDict else f'{round(questionDetailsDict[questionNo].acRate, 2)} %'\n",
    "        f.write(f'> **Acceptance Rate** : **{acrate}**\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "\n",
    "        if 'contextFile' in questionData:\n",
    "            with open(join(config.README_PATH, questionData['contextFile']), 'r') as contextFile:\n",
    "                f.write('> ' + contextFile.read().replace('\\n', '\\n> '))\n",
    "            f.write(f'\\n\\n------\\n\\n')\n",
    "        \n",
    "\n",
    "        f.write(f'## Solutions\\n\\n')\n",
    "        for lang, solutions in questionData['solutions'].items() :\n",
    "            solutions.sort()\n",
    "            for solution in solutions :\n",
    "                name = solution[solution.find('/') + 1:]\n",
    "                f.write(f'- [{name}](<{join(config.README_PATH, solution)}>)\\n')\n",
    "\n",
    "        for lang, solutions in questionData['solutions'].items() :\n",
    "            if lang.lower() in LANGUAGE_EQUIVS :\n",
    "                lang = LANGUAGE_EQUIVS[lang.lower()]\n",
    "            else :\n",
    "                print()\n",
    "                print(f'Lang equiv not found: {lang = }')\n",
    "            f.write(f'### {lang}\\n')\n",
    "            for solution in solutions :\n",
    "                name = solution.rfind('/') + 1\n",
    "                f.write(f'#### [{solution[name:]}](<{join(config.README_PATH, solution)}>)\\n')\n",
    "                f.write(f'```{lang}\\n')\n",
    "                with open(join(config.README_PATH, solution), 'r', encoding='utf-8') as solutionFile:\n",
    "                    fileData = solutionFile.read()\n",
    "                    if '# @lc code=start' in fileData :\n",
    "                        lcStart = '# @lc code=start'\n",
    "                        lcEnd   = '# @lc code=end'\n",
    "                        fileData = fileData[fileData.find(lcStart) + len(lcStart):fileData.rfind(lcEnd)]\n",
    "                    f.write(fileData)\n",
    "                f.write('\\n```\\n\\n')\n",
    "\n",
    "    return output_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processMarkdownGeneration(questionData: dict,\n",
    "                              reprocessMarkdown: Set[int],\n",
    "                              *,\n",
    "                              questionDetailsDict: dict = None) -> None :\n",
    "    \n",
    "    if not questionDetailsDict :\n",
    "        questionDetailsDict = retrieveQuestionDetails()\n",
    "    \n",
    "    # Create a folder to avoid errors if it doesn't already exist\n",
    "    markdownFolder = join(config.README_PATH, config.MARKDOWN_PATH)\n",
    "    if not isdir(markdownFolder) :\n",
    "        mkdir(markdownFolder)\n",
    "\n",
    "    for questionNo, dta in questionData.items() :\n",
    "        if questionNo in reprocessMarkdown :\n",
    "            generate_markdown(questionNo, \n",
    "                              questionData, \n",
    "                              questionDetailsDict=questionDetailsDict, \n",
    "                              export=True)\n",
    "        else : # In order to assign the markdown paths\n",
    "            generate_markdown(questionNo, \n",
    "                              questionData, \n",
    "                              questionDetailsDict=questionDetailsDict, \n",
    "                              export=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataFrames\n",
    "Conversion into DataFrames and declaration of respective column headers occurs here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertDataToMatrix(questionData: dict,\n",
    "                        *,\n",
    "                        sortBy: str = 'number',\n",
    "                        includeDate: bool = False,\n",
    "                        includeQuestions: set[int] = set(),\n",
    "                        relativeFolderAdjustment: int = 0,\n",
    "                        includeMarkdownFolder: bool = False) -> List[list] :\n",
    "    \n",
    "    dataframe_array = []\n",
    "\n",
    "    for question in questionData.values() :\n",
    "        # If it's not an empty set and the value isn't in there, skip\n",
    "        if includeQuestions and question['number'] not in includeQuestions :\n",
    "            continue\n",
    "\n",
    "        if sortBy == 'number' and includeMarkdownFolder :\n",
    "            solution_path = join(config.MARKDOWN_PATH, question['solution'])    \n",
    "        else :\n",
    "            solution_path = question['solution']\n",
    "        \n",
    "        solution_path = '../' * abs(relativeFolderAdjustment) + solution_path\n",
    "\n",
    "        title_to_use = question['title']\n",
    "        \n",
    "        if question['contestTitle'] and question['contestQNo'] :\n",
    "            title_to_use = f'{question[\"contestTitle\"]} - {question[\"contestQNo\"]} - {title_to_use}'\n",
    "\n",
    "        currentRow = [question['number'],\n",
    "                      title_to_use, \n",
    "                      question['level'], \n",
    "                      ', '.join(sorted(list(question['categories']))), \n",
    "                      f'[solution](<{solution_path}>)', \n",
    "                      ', '.join(sorted(list(question['languages'])))]\n",
    "        \n",
    "        if includeDate :\n",
    "            currentRow.append(question['date_done'].strftime('%b %d, %Y'))\n",
    "        \n",
    "        dataframe_array.append(currentRow)\n",
    "\n",
    "    # Usually, x[0] is the question number but the qustionData set might be keyed by the date if it's a daily case\n",
    "    if not questionData :\n",
    "        pass\n",
    "    elif sortBy == 'date_done' and type(list(questionData.keys())[0]) == datetime :\n",
    "        dataframe_array.sort(key=lambda x: datetime.strptime(x[-1], '%b %d, %Y'))\n",
    "    else :\n",
    "        dataframe_array.sort(key=lambda x: questionData.get(x[0])[sortBy])\n",
    "\n",
    "    return dataframe_array\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertQuestionDataToDataframe(questionData: dict,\n",
    "                                   *,\n",
    "                                   sortBy: str = 'number',\n",
    "                                   includeDate:  bool = False,\n",
    "                                   includeQuestions: set[int] = set(),\n",
    "                                   relativeFolderAdjustment: int = 0,\n",
    "                                   includeMarkdownFolder: bool = False) -> DataFrame :\n",
    "    \n",
    "    questionData = convertDataToMatrix(questionData, \n",
    "                                       sortBy=sortBy, \n",
    "                                       includeDate=includeDate, \n",
    "                                       includeQuestions=includeQuestions,\n",
    "                                       relativeFolderAdjustment=relativeFolderAdjustment,\n",
    "                                       includeMarkdownFolder=includeMarkdownFolder)\n",
    "    \n",
    "    # Protects against empty cases (e.g. if you have no daily files)\n",
    "    dfQuestions = kungfupanda.DataFrame()\n",
    "    if questionData :\n",
    "        dfQuestions = kungfupanda.DataFrame(data=questionData, columns=COLUMNS[:len(questionData[0])])\n",
    "    \n",
    "    return dfQuestions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# List & Other Markdowns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sorted by Most Recent\n",
    "Using creation dates of code files only; not modification dates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: Reversed due to default sorting being in ascending order\n",
    "def byRecentQuestionDataDataframe(questionData: dict) -> DataFrame :\n",
    "    return convertQuestionDataToDataframe(questionData, sortBy='date_done', includeDate=True).iloc[::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sorted by Amount of Code\n",
    "Questions with more files on the question and longer submissions will be prioritized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def byCodeLengthDataDataframe(questionData: dict) -> DataFrame :\n",
    "    return convertQuestionDataToDataframe(questionData, sortBy='bytes', includeDate=True).iloc[::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generation of Markdowns for Each Related Topic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def questionTopicDataframes(questionData: dict,\n",
    "                            *,\n",
    "                            topicGroupings: defaultdict) -> List[Tuple[str, int, DataFrame]] : # [topic, number of questions, dataframe]\n",
    "    if not topicGroupings :\n",
    "        topicGroupings = getCompletedQuestionsTopicLists(questionData)\n",
    "    \n",
    "    output = []\n",
    "    for topic, qs in topicGroupings.items() :\n",
    "        output.append((topic, \n",
    "                       len(qs), \n",
    "                       convertQuestionDataToDataframe(questionData,\n",
    "                                                      includeDate=True,\n",
    "                                                      includeQuestions=qs,\n",
    "                                                      relativeFolderAdjustment= \\\n",
    "                                                        -getenv('TOPIC_MARKDOWN_PATH_IN_MARKDOWNS_FOLDER').count('/'))))\n",
    "\n",
    "    output.sort(key=lambda x: x[1], reverse=True)\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Topic based markdown generation and any of the large list markdowns in general sometimes suddenly show massive edits for what should be a regular usual question update. This is likely due to the dataframe.to_markdown method increasing the width of the table due to a larger than before seen input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topicBasedMarkdowns(questionData: dict,\n",
    "                         *,\n",
    "                         topicGroupings: defaultdict) -> List[Tuple[str, str]] :    # path of all outputs\n",
    "                                                                                    # list[0]  = overall mardown\n",
    "                                                                                    # list[1:] = order of count\n",
    "                                                                                    # doesn't include markdown 'markdown/'\n",
    "    if not topicGroupings :\n",
    "        topicGroupings = getCompletedQuestionsTopicLists(questionData)\n",
    "\n",
    "    topicDataframes = questionTopicDataframes(questionData=questionData, topicGroupings=topicGroupings)\n",
    "\n",
    "    if not isdir(config.NOTEBOOK_PATH) :\n",
    "        mkdir(config.NOTEBOOK_PATH)\n",
    "\n",
    "    output = [config.OVERALL_FILE_README_PATH]\n",
    "    with open(config.OVERALL_FILE_NOTEBOOK_PATH, 'w', encoding='utf-8') as topic_file :\n",
    "        topic_file.write('# Topics\\n\\n')\n",
    "        topic_file.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        topic_file.write('------\\n\\n')\n",
    "\n",
    "        for topic, cnt, df in topicDataframes :\n",
    "            file_name = f'{topic}.md'\n",
    "            notebook_path = join(config.NOTEBOOK_PATH, file_name)\n",
    "            readme_path = join(config.TOPIC_FOLDER, file_name)\n",
    "            with open(notebook_path, 'w', encoding='utf-8') as f :\n",
    "                url = f'https://leetcode.com/tag/{topic.replace(\" \", \"-\")}/'\n",
    "                f.write(f'# [{topic}](<{url}>) ({cnt} completed)\\n\\n')\n",
    "                f.write(f'*[Back to top](<../../README.md>)*\\n\\n')\n",
    "                f.write('------\\n\\n')\n",
    "                f.write(df.to_markdown(index=False))\n",
    "        \n",
    "            topic_file.write(f'- [{topic}](<{readme_path}>) ({cnt} completed)\\n')\n",
    "            output.append((topic, readme_path))\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Markdowns for Easy/Medium/Hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def generateDifficultyLevelMarkdowns(questionData: dict) -> Tuple[Tuple[int, str], \n",
    "                                                                  Tuple[int, str], \n",
    "                                                                  Tuple[int, str]] :\n",
    "    '''\n",
    "    ### Returns:\n",
    "    - Tuple[Easy, Medium, Hard]\n",
    "        - Tuple[int, str] : (count, path from readme)\n",
    "    '''\n",
    "    \n",
    "    easyQuestions   = {}\n",
    "    mediumQuestions = {}\n",
    "    hardQuestions   = {}\n",
    "    \n",
    "    for q, d in questionData.items() :\n",
    "        lvlIndicator = d['level'][0].lower()\n",
    "        match lvlIndicator :\n",
    "            case 'e' :\n",
    "                easyQuestions[q] = d\n",
    "            case 'm' :\n",
    "                mediumQuestions[q] = d\n",
    "            case 'h' :\n",
    "                hardQuestions[q] = d\n",
    "            case _ :\n",
    "                print(f'Error identifying level of {q = }')\n",
    "    \n",
    "    easyMarkdown    = convertQuestionDataToDataframe(easyQuestions, \n",
    "                                                     includeDate=True, \n",
    "                                                     includeMarkdownFolder=False)\n",
    "    mediumMarkdown  = convertQuestionDataToDataframe(mediumQuestions, \n",
    "                                                     includeDate=True, \n",
    "                                                     includeMarkdownFolder=False)\n",
    "    hardMarkdown    = convertQuestionDataToDataframe(hardQuestions, \n",
    "                                                     includeDate=True, \n",
    "                                                     includeMarkdownFolder=False)\n",
    "    \n",
    "    \n",
    "    easy_path   = join(config.DIFFICULTY_MARKDOWNS_PATH, 'Easy.md')\n",
    "    medium_path = join(config.DIFFICULTY_MARKDOWNS_PATH, 'Medium.md')\n",
    "    hard_path   = join(config.DIFFICULTY_MARKDOWNS_PATH, 'Hard.md')\n",
    "    \n",
    "    with open('../' + easy_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# Easy Questions ({len(easyQuestions)})\\n\\n')\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(easyMarkdown.to_markdown(index=False))\n",
    "        \n",
    "    with open('../' + medium_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# Medium Questions ({len(mediumQuestions)})\\n\\n')\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(mediumMarkdown.to_markdown(index=False))\n",
    "        \n",
    "    with open('../' + hard_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# Hard Questions ({len(hardQuestions)})\\n\\n')\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(hardMarkdown.to_markdown(index=False))\n",
    "    \n",
    "    return ((len(easyQuestions),    easy_path),\n",
    "            (len(mediumQuestions),  medium_path),\n",
    "            (len(hardQuestions),    hard_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dailies, Recents, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def miscMarkdownGenerations(questionData:   dict,\n",
    "                            *,\n",
    "                            code_length:    bool = False,\n",
    "                            recent:         bool = False,\n",
    "                            daily:          bool = False,\n",
    "                            weekly:         bool = False) -> str : # output path\n",
    "    \n",
    "    df          = None\n",
    "    fileName    = None\n",
    "    header_data = None\n",
    "    details     = None\n",
    "    tail        = None\n",
    "\n",
    "    if code_length :\n",
    "        df = byCodeLengthDataDataframe(questionData)\n",
    "        fileName    = 'Questions_By_Code_Length.md'\n",
    "        header_data = '# Questions By Code Length\\n\\n'\n",
    "        details     = 'Calculations are based on the code files\\'s byte sizes.\\n\\n'\n",
    "    elif recent :\n",
    "        df = byRecentQuestionDataDataframe(questionData)\n",
    "        fileName    = 'Questions_By_Recent.md'\n",
    "        header_data = '# Most Recently Solved Questions\\n\\n'\n",
    "        details     = 'Calculations are based on the date of the first solve.\\n\\n'\n",
    "    elif daily :\n",
    "        dailyQuestionData = parseQuestionsForDailies(questionData)\n",
    "\n",
    "        print(f'{dailyQuestionData = }')\n",
    "        print(f'{config._oldest_date = }')\n",
    "\n",
    "        df = byRecentQuestionDataDataframe(dailyQuestionData)\n",
    "        fileName    = 'Daily_Questions.md'\n",
    "        # header_data = f'# [Daily Questions](<{DAILY_URL}>)\\n\\n'\n",
    "        header_data = f'# Daily Questions\\n\\n'\n",
    "        \n",
    "        curr_month, months = get_dailies_month_maps(dailyQuestionData)\n",
    "        \n",
    "        details     = 'Dates are for the date I completed the ' + \\\n",
    "                      'question so due to the my time zone and how it lines up with ' + \\\n",
    "                      'UTC, it may be off by a day.\\n\\n' + \\\n",
    "                      curr_month + '\\n\\n'\n",
    "        tail        = months\n",
    "    elif weekly :\n",
    "        weeklyQuestionData = parseQuestionsForWeeklies(questionData)\n",
    "    \n",
    "        df          = byRecentQuestionDataDataframe(weeklyQuestionData)\n",
    "        fileName    = 'Weekly_Questions.md'\n",
    "        # header_data = f'# [Daily Questions](<{DAILY_URL}>)\\n\\n'\n",
    "        header_data = f'# Weekly Premium Questions\\n\\n'\n",
    "        details     = 'Dates are for the date I completed the ' + \\\n",
    "                      'question so due to the my time zone and how it lines up with ' + \\\n",
    "                      'UTC, it may be off by a day.\\n\\n'\n",
    "    else :\n",
    "        print('Error. No markdown generation specified.')\n",
    "        print()\n",
    "        return ''\n",
    "\n",
    "    output_path = join(config.MARKDOWN_PATH, fileName)\n",
    "    readme_path = join(config.README_PATH, config.MARKDOWN_PATH, fileName)\n",
    "\n",
    "    with open(readme_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(header_data)\n",
    "        f.write(f'*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write(details)\n",
    "        f.write(df.to_markdown(index=False))\n",
    "        \n",
    "        if tail :\n",
    "            f.write(f'\\n\\n{tail}')\n",
    "\n",
    "    return output_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Outputing to README File\n",
    "Takes all the above and overwrites the current [README.md](README.md) file with the data calculated above.\n",
    "\n",
    "Inputs values in order of:\n",
    "- Profile link\n",
    "- Stats\n",
    "- Stat clarification\n",
    "- Question link tables Easy-Medium-Hard\n",
    "\n",
    "Uses the built-in DataFrame `.to_markdown()` for outputting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exportPrimaryReadme(dfQuestions:        DataFrame,\n",
    "                        *,\n",
    "                        difficultyBasedMarkdowns: Tuple[Tuple[int, str], \n",
    "                                                        Tuple[int, str], \n",
    "                                                        Tuple[int, str]] = None,\n",
    "                        additionalSorts:    List[str] = [],\n",
    "                        topicLinks:         List[Tuple[str, str]] = []) -> None :\n",
    "    readmePath = join(config.README_PATH, 'README.md')\n",
    "    print(readmePath)\n",
    "\n",
    "    # No. Questions Solved\n",
    "    qSolvedHeader = f'{len(dfQuestions.index)} solved'\n",
    "    \n",
    "    print(difficultyBasedMarkdowns)\n",
    "    # if difficultyBasedMarkdowns :\n",
    "    #     qSolvedHeader += f' - [{difficultyBasedMarkdowns[0][0]}e](<{difficultyBasedMarkdowns[0][1]}>), ' + \\\n",
    "    #                         f'[{difficultyBasedMarkdowns[1][0]}m](<{difficultyBasedMarkdowns[1][1]}>), ' + \\\n",
    "    #                         f'[{difficultyBasedMarkdowns[2][0]}h](<{difficultyBasedMarkdowns[2][1]}>)'\n",
    "    \n",
    "    with open(readmePath, 'w') as file :\n",
    "        username = getenv('LEETCODE_USERNAME')\n",
    "        file.write(f'# **[LeetCode Records](https://leetcode.com/u/{username}/)** ({qSolvedHeader})\\n\\n')\n",
    "        file.write(f'<!-- This readme was generated using [WikiLeet](<https://github.com/Zanger67/WikiLeet>) -->\\n\\n')\n",
    "        file.write(f'> My LeetCode Profile: [{username}](https://leetcode.com/u/{username}/)\\n')\n",
    "        \n",
    "        # if difficultyBasedMarkdowns :\n",
    "        #     file.write(f'> [{difficultyBasedMarkdowns[0][0]} easy](<{difficultyBasedMarkdowns[0][1]}>), ' + \\\n",
    "        #                f'[{difficultyBasedMarkdowns[1][0]} medium](<{difficultyBasedMarkdowns[1][1]}>), ' + \\\n",
    "        #                f'[{difficultyBasedMarkdowns[2][0]} hard](<{difficultyBasedMarkdowns[2][1]}>)')\n",
    "        \n",
    "        file.write('\\n\\n')\n",
    "\n",
    "        file.write('## About this Repo\\n\\n')\n",
    "        file.write('This repo is a collection of my LeetCode solutions, primarily written in Python, Java, and C. ' + \n",
    "                   'On any page, `click the main title` to be redirected to the official `LeetCode` page for the ' + \n",
    "                   'question, topic, list, etc. See the `Additional Categories` section for pages that group' + \n",
    "                   ' questions by different criteria -- e.g. by their *related topics*.')\n",
    "        file.write('\\n\\n\\n')\n",
    "\n",
    "        file.write('------\\n\\n')\n",
    "\n",
    "        file.write('## Category Notes\\n')\n",
    "        file.write('1. **Daily** - Daily challenge questions that were done on the day of\\n')\n",
    "        file.write('2. **Weekly Premium** - Weekly premium questions that were done on week of\\n')\n",
    "        file.write('3. **Contest** - Questions that I completed during a live contest\\n')\n",
    "        # file.write('4. **Favourite** - Questions that I liked and wanted to keep a record of\\n')\n",
    "        file.write('\\n\\n')\n",
    "\n",
    "        file.write('------\\n\\n')\n",
    "\n",
    "        file.write('## Additional Categories Stats\\n')\n",
    "\n",
    "        for altSorts in additionalSorts :\n",
    "            file.write(altSorts)\n",
    "            file.write('\\n\\n')\n",
    "        \n",
    "        if topicLinks :\n",
    "            file.write('------\\n\\n')\n",
    "            file.write(', '.join([f'[{topic}](<{join(config.MARKDOWN_PATH, link)}>)' for topic, link in topicLinks[1:]]))       \n",
    "            file.write('\\n\\n')\n",
    "            file.write('------\\n\\n')\n",
    "        \n",
    "        file.write('\\n\\n')\n",
    "\n",
    "        file.write('## Questions\\n')\n",
    "        file.write(dfQuestions.to_markdown(index=False))\n",
    "        \n",
    "        \n",
    "        file.write('\\n\\n')\n",
    "        file.write('<p align=\"right\"><i>This README was generated using <a href=\"https://github.com/Zanger67/WikiLeet\">WikiLeet</a></i></p>\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recalculateAll: forces recalcualtion markdowns for each question irregardless if its\n",
    "#                 source files have been modified or not\n",
    "def main(*, recalculateAll: bool = False, noRecord: bool = False) -> None :\n",
    "    leetcodeFiles           = getCodeFiles()\n",
    "    additionalInfoFiles     = getContextFiles()     # For later use when generating the individual readme files\n",
    "\n",
    "    contestFolders          = getContestFolders()\n",
    "    contestLeetcodeFiles    = getContestFiles(contestFolders)\n",
    "\n",
    "    if USE_GIT_DATES :\n",
    "        getAllCTimesViaGit(additionalInfoFiles \n",
    "                           + leetcodeFiles \n",
    "                           + [join(x[0], x[1]) for x in contestLeetcodeFiles])\n",
    "\n",
    "\n",
    "    questionDetailsDict     = retrieveQuestionDetails()\n",
    "\n",
    "    leetcodeFiles.sort()\n",
    "    contestLeetcodeFiles.sort()\n",
    "\n",
    "\n",
    "    # Files for leetcode questions found\n",
    "    print(f'Total of {len(leetcodeFiles)} files found.')\n",
    "\n",
    "    # Files in contest folders found\n",
    "    print(f'Total of {len(contestLeetcodeFiles)} contest files found.')\n",
    "\n",
    "\n",
    "    # Parsing primary files\n",
    "    fileLatestTimes = getRecentFileTimes() if (not recalculateAll and not noRecord) else {}\n",
    "\n",
    "    reprocessMarkdown = set()\n",
    "    questionData = {}\n",
    "\n",
    "    # Parsing primary files\n",
    "    print('Parsing code files...')\n",
    "    for leetcodeFile in leetcodeFiles :\n",
    "        parseCase(leetcodeFile=leetcodeFile,\n",
    "                  questionData=questionData,\n",
    "                  fileLatestTimes=fileLatestTimes, \n",
    "                  reprocessMarkdown=reprocessMarkdown,\n",
    "                  questionDetailsDict=questionDetailsDict)\n",
    "        \n",
    "    # Parsing contest files & folforders\n",
    "    print('Parsing contest files...')\n",
    "    for leetcodeContestFile in contestLeetcodeFiles :\n",
    "        contestFolder, leetcodeFile = leetcodeContestFile\n",
    "        parseCase(leetcodeFile=leetcodeFile,\n",
    "                  questionData=questionData,\n",
    "                  fileLatestTimes=fileLatestTimes,\n",
    "                  reprocessMarkdown=reprocessMarkdown, \n",
    "                  subFolderPath=contestFolder, \n",
    "                  questionDetailsDict=questionDetailsDict,\n",
    "                  contest=contestFolder)\n",
    "        \n",
    "\n",
    "    # Parsing additional information files\n",
    "    print('Parsing additional information/context files...')\n",
    "    parseContextFiles(txtFiles=additionalInfoFiles, \n",
    "                      questionData=questionData,\n",
    "                      fileLatestTimes=fileLatestTimes,\n",
    "                      reprocessMarkdown=reprocessMarkdown)\n",
    "    \n",
    "    # Identifying members of lists\n",
    "    print('Sorting questions to their lists...')\n",
    "    processListData(questionData=questionData)\n",
    "\n",
    "    # Generating markdowns for each individual question\n",
    "    print('Generating markdowns for each individual question...')\n",
    "    processMarkdownGeneration(questionData=questionData, \n",
    "                              reprocessMarkdown=reprocessMarkdown, \n",
    "                              questionDetailsDict=questionDetailsDict)\n",
    "\n",
    "    \n",
    "    # Produces a markdown where questions are sorted by the amount of code\n",
    "    # written for the question\n",
    "    # code_length_md_path = exportCodeLengthMarkdown(questionData)\n",
    "    print('Generating category lists...')\n",
    "    dailyQuestions      = miscMarkdownGenerations(questionData, daily=True)\n",
    "    weeklyQuestions     = miscMarkdownGenerations(questionData, weekly=True)\n",
    "    byCodeLength        = miscMarkdownGenerations(questionData, code_length=True)\n",
    "    byRecentlySolved    = miscMarkdownGenerations(questionData, recent=True)\n",
    "    altSorts            = [f'- [Daily Questions](<{dailyQuestions}>)',\n",
    "                           f'- [Weekly Questions](<{weeklyQuestions}>)',\n",
    "                           f'- [Questions By Code Length](<{byCodeLength}>)',\n",
    "                           f'- [Questions By Recent](<{byRecentlySolved}>)']\n",
    "    \n",
    "\n",
    "    difficultyBasedMarkdowns = generateDifficultyLevelMarkdowns(questionData)\n",
    "    \n",
    "\n",
    "    completedQsTopicGroupings = getCompletedQuestionsTopicLists(questionData)\n",
    "    topicMarkdownLinks = topicBasedMarkdowns(questionData, topicGroupings=completedQsTopicGroupings)\n",
    "    altSorts.append(f'- [Grouped by Topic](<{topicMarkdownLinks[0]}>)')\n",
    "\n",
    "\n",
    "    # Exporting the primary README.md file\n",
    "    print('Exporting primary README.md file...')\n",
    "    dfQuestions = convertQuestionDataToDataframe(questionData, \n",
    "                                                 includeDate=False, \n",
    "                                                 includeMarkdownFolder=True)\n",
    "    exportPrimaryReadme(dfQuestions, \n",
    "                        additionalSorts=altSorts, \n",
    "                        topicLinks=topicMarkdownLinks, \n",
    "                        difficultyBasedMarkdowns=difficultyBasedMarkdowns)\n",
    "\n",
    "\n",
    "    print(f'Number of individual questions updated/added: {len(reprocessMarkdown)}')\n",
    "\n",
    "\n",
    "    if not noRecord :\n",
    "        print('Pickling most recent modification times for future reference...')\n",
    "        writeRecentFileTimes(fileLatestTimes)           # restore for next use\n",
    "\n",
    "\n",
    "    print('All processes complete. Exiting...')\n",
    "    return questionData, reprocessMarkdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__' :\n",
    "    '''\n",
    "    ### Flags\n",
    "    `-r` : \n",
    "        Recalculate all markdown files irregardless of whether there are modified or new code files for that question or not\n",
    "    `-n` :\n",
    "        Don't use the previous modified dates and don't store them (in effect, the same as `-r` but it doesn't save the \n",
    "        new modification dates). Primarily for use with GitHub actions.\n",
    "    `-g` :\n",
    "        Uses the repository's git log history for each file to trace the creation and last modification dates of each file \n",
    "        rather than use the default `getctime()` and `getmtime()` of each file. GitHub actions seem to default the ctime \n",
    "        and mtimes to time.now due to not tracking the actual mtime ctime metadata.\n",
    "\n",
    "        WARNING: Only for use with GitHub actions as this ends up being very slow due to low subprocess speends.\n",
    "    '''\n",
    "    recalcaulateAll = False\n",
    "    noRecord = False\n",
    "    \n",
    "\n",
    "    if not IS_NOTEBOOK :\n",
    "        parser = argparse.ArgumentParser()\n",
    "        \n",
    "\n",
    "        parser.add_argument(\"-r\", \n",
    "                            help=\"Recompile all markdown files\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "        \n",
    "        parser.add_argument(\"-n\", \n",
    "                            help=\"Don't use the previous modified dates and don't store them\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "        parser.add_argument(\"-norecord\", \n",
    "                            help=\"Don't use the previous modified dates and don't store them\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "        parser.add_argument(\"-g\", \n",
    "                            help=\"Use Git repo's dates for determining if a file has been modified and created (WARNING SLOW)\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "\n",
    "        parser.add_argument('-user', \n",
    "                            type=str, \n",
    "                            default='', \n",
    "                            help='LeetCode Username',\n",
    "                            required=False)\n",
    "        parser.add_argument(\"-dir\",\n",
    "                            type=str,\n",
    "                            default='',\n",
    "                            help=\"Solutions directory name; default of \\\"my-submissions/\\\"\",\n",
    "                            required=False)\n",
    "        \n",
    "        if parser.parse_args().user :\n",
    "            environ['LEETCODE_USERNAME'] = parser.parse_args().user\n",
    "        \n",
    "        if parser.parse_args().dir :\n",
    "            print('dir found')\n",
    "            environ['QUESTIONS_PATH_FROM_README'] = parser.parse_args().dir\n",
    "            \n",
    "            if not environ['QUESTIONS_PATH_FROM_README'].endswith('/') :\n",
    "                print('added / to dir')\n",
    "                environ['QUESTIONS_PATH_FROM_README'] = f\"{environ['QUESTIONS_PATH_FROM_README']}/\"\n",
    "        \n",
    "        print(environ['QUESTIONS_PATH_FROM_README'])\n",
    "        \n",
    "        recalcaulateAll = parser.parse_args().r\n",
    "        noRecord = parser.parse_args().norecord or parser.parse_args().n\n",
    "\n",
    "        USE_GIT_DATES = parser.parse_args().g\n",
    "\n",
    "    \n",
    "\n",
    "    set_env_linked_constants()\n",
    "    print(f'\\t\\t{config.QUESTION_DATA_FOLDER = }')\n",
    "\n",
    "    print('No record'.ljust(20), 'on' if noRecord else 'off')\n",
    "    print('Recalculate'.ljust(20), 'on' if recalcaulateAll else 'off')\n",
    "    print('Use Git dates'.ljust(20), 'on' if USE_GIT_DATES else 'off')\n",
    "    print()\n",
    "\n",
    "\n",
    "    main(recalculateAll=recalcaulateAll, noRecord=noRecord)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}