dvc-2-data-versioning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial: Versioning Data and Model \n",
    "\n",
    "ML REPA School course: **Machine Learning experiments reproducibility and engineering with DVC**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checkout branch `tutorial`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:15:19.846622Z",
     "start_time": "2020-10-18T15:15:19.712024Z"
    }
   },
   "outputs": [],
   "source": [
    "!git checkout -b tutorial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize DVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:15:22.447971Z",
     "start_time": "2020-10-18T15:15:20.931295Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:15:24.239785Z",
     "start_time": "2020-10-18T15:15:24.117061Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls .dvc/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:15:24.635490Z",
     "start_time": "2020-10-18T15:15:24.507207Z"
    }
   },
   "outputs": [],
   "source": [
    "!git status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:15:25.056535Z",
     "start_time": "2020-10-18T15:15:24.781962Z"
    }
   },
   "outputs": [],
   "source": [
    "!git add .dvc\n",
    "!git commit -m \"DVC init\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How data versioning works?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:15:47.480522Z",
     "start_time": "2020-10-18T15:15:45.737444Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc status"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add a file under DVC control"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:16:27.065704Z",
     "start_time": "2020-10-18T15:16:11.987014Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get data \n",
    "\n",
    "!dvc get https://github.com/iterative/dataset-registry \\\n",
    "          get-started/data.xml -o data/data.xml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:16:29.468276Z",
     "start_time": "2020-10-18T15:16:27.075386Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!dvc add data/data.xml -v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:16:31.394516Z",
     "start_time": "2020-10-18T15:16:31.343632Z"
    }
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "git add data/.gitignore data/data.xml.dvc\n",
    "git commit -m \"Add raw data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:16:32.468331Z",
     "start_time": "2020-10-18T15:16:32.345543Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls .dvc/cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:16:33.479169Z",
     "start_time": "2020-10-18T15:16:33.354465Z"
    }
   },
   "outputs": [],
   "source": [
    "!du -sh .dvc/cache/*/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:16:41.380231Z",
     "start_time": "2020-10-18T15:16:34.361689Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat .dvc/cache/a3/04afb96060aad90176268345e10355"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:17:43.999993Z",
     "start_time": "2020-10-18T15:17:43.874843Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat data/data.xml.dvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Add a directory under version control"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:18:08.421307Z",
     "start_time": "2020-10-18T15:18:08.287161Z"
    }
   },
   "outputs": [],
   "source": [
    "!git checkout -b cats-dogs-v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:24:45.820399Z",
     "start_time": "2020-10-18T15:23:51.225739Z"
    }
   },
   "outputs": [],
   "source": [
    "# Download cata-dogs data sets ()\n",
    "\n",
    "!dvc get --rev cats-dogs-v1 \\\n",
    "          https://github.com/iterative/dataset-registry \\\n",
    "          use-cases/cats-dogs -o datadir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:24:52.197475Z",
     "start_time": "2020-10-18T15:24:52.070344Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!ls datadir/data/train/cats "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:25:13.071321Z",
     "start_time": "2020-10-18T15:25:07.323409Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!dvc add datadir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:25:23.606458Z",
     "start_time": "2020-10-18T15:25:23.483275Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat datadir.dvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:25:30.651688Z",
     "start_time": "2020-10-18T15:25:30.523021Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!git status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:25:36.921831Z",
     "start_time": "2020-10-18T15:25:36.522609Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!git add .gitignore datadir.dvc\n",
    "!git commit -m \"Add datadir\"\n",
    "!git tag -a cats-dogs-v1 -m \"Create data version v1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tracking changes "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Track data status changes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:25:41.450987Z",
     "start_time": "2020-10-18T15:25:39.690913Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc status"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Updating Tracked Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:26:37.102374Z",
     "start_time": "2020-10-18T15:26:36.973380Z"
    }
   },
   "outputs": [],
   "source": [
    "!git checkout -b cats-dogs-v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:27:53.130784Z",
     "start_time": "2020-10-18T15:26:39.760171Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc get --rev cats-dogs-v2 \\\n",
    "          https://github.com/iterative/dataset-registry \\\n",
    "          use-cases/cats-dogs -o datadir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:30:28.136882Z",
     "start_time": "2020-10-18T15:30:26.283857Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc status "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:30:26.276329Z",
     "start_time": "2020-10-18T15:30:17.631112Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!dvc add datadir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:28:37.323660Z",
     "start_time": "2020-10-18T15:28:37.193900Z"
    }
   },
   "outputs": [],
   "source": [
    "!git status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:32:14.693738Z",
     "start_time": "2020-10-18T15:32:14.305147Z"
    }
   },
   "outputs": [],
   "source": [
    "!git add datadir.dvc\n",
    "!git commit -m \"Change data\"\n",
    "!git tag -a cats-dogs-v2 -m \"Create data version v2\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Switching versions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checkout to the initital branch `tutorial`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:45:05.783162Z",
     "start_time": "2020-10-18T15:45:03.730332Z"
    }
   },
   "outputs": [],
   "source": [
    "!git checkout tutorial\n",
    "!dvc checkout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:45:05.914496Z",
     "start_time": "2020-10-18T15:45:05.790564Z"
    }
   },
   "outputs": [],
   "source": [
    "# No `datadir` directory there! \n",
    "\n",
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Switch to the first version of the Cats&Dogs data (branch cats-dogs-v1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:45:09.907395Z",
     "start_time": "2020-10-18T15:45:09.775878Z"
    }
   },
   "outputs": [],
   "source": [
    "# Checkout to the cats-dogs-v1\n",
    "\n",
    "!git checkout cats-dogs-v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:45:44.584467Z",
     "start_time": "2020-10-18T15:45:44.462400Z"
    }
   },
   "outputs": [],
   "source": [
    "# Still - No `datadir` directory there! Why? \n",
    "\n",
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:46:43.546157Z",
     "start_time": "2020-10-18T15:46:42.066300Z"
    }
   },
   "outputs": [],
   "source": [
    "# DVC can see the status and why there is no 'datadir' over there\n",
    "\n",
    "!dvc status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:47:39.039524Z",
     "start_time": "2020-10-18T15:47:35.348041Z"
    }
   },
   "outputs": [],
   "source": [
    "# To bring the 'datadir' back we need to do `dvc checkout` \n",
    "\n",
    "!dvc checkout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:47:49.038869Z",
     "start_time": "2020-10-18T15:47:47.137611Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls\n",
    "!dvc status"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store and share data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup your remote storage (local)\n",
    "\n",
    "**IMPORTANT**: we use `/tmp` folder in examples in this course just for simplicity! DON NOT use `/tmp` folder to store files for a long term! This folder is cleaned up frequently by your system. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:54:16.733710Z",
     "start_time": "2020-10-18T15:54:15.808018Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create new remote\n",
    "\n",
    "!mkdir -p /tmp/dvc\n",
    "!dvc remote add -d local /tmp/dvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:54:20.048272Z",
     "start_time": "2020-10-18T15:54:19.921063Z"
    }
   },
   "outputs": [],
   "source": [
    "# As you can see, .dvc/config is changed\n",
    "\n",
    "!git status -s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:54:30.253559Z",
     "start_time": "2020-10-18T15:54:30.128066Z"
    }
   },
   "outputs": [],
   "source": [
    "# Check config file \n",
    "\n",
    "!cat .dvc/config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:54:33.762382Z",
     "start_time": "2020-10-18T15:54:33.509102Z"
    }
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "git add .\n",
    "git commit -m \"Add remote storage\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Push data to remote storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:54:43.408833Z",
     "start_time": "2020-10-18T15:54:38.409287Z"
    }
   },
   "outputs": [],
   "source": [
    "# Push data to remote\n",
    "\n",
    "!dvc push -v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:54:56.929427Z",
     "start_time": "2020-10-18T15:54:56.807582Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat datadir.dvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:55:47.568504Z",
     "start_time": "2020-10-18T15:55:47.446645Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls /tmp/dvc/b6/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:55:57.293273Z",
     "start_time": "2020-10-18T15:55:57.148304Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat /tmp/dvc/b6/923e1e4ad16ea1a7e2b328842d56a2.dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-07T16:39:19.987379Z",
     "start_time": "2020-06-07T16:39:19.972195Z"
    }
   },
   "source": [
    "## Retrieve data from remote storage "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:58:33.668273Z",
     "start_time": "2020-10-18T15:58:33.332981Z"
    }
   },
   "outputs": [],
   "source": [
    "# For example - Remove local cached file\n",
    "\n",
    "!rm -rf .dvc/cache\n",
    "!rm -rf datadir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:58:45.487828Z",
     "start_time": "2020-10-18T15:58:45.365274Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:59:02.218759Z",
     "start_time": "2020-10-18T15:58:47.069686Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc pull -v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:59:02.353837Z",
     "start_time": "2020-10-18T15:59:02.228723Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-09T07:57:55.231135Z",
     "start_time": "2019-06-09T07:57:55.190232Z"
    },
    "scrolled": true
   },
   "source": [
    "# Data Access"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find a dataset\n",
    "\n",
    "> You can use dvc list to explore a DVC repository hosted on any Git server. For example, let's see what's in the use-cases/ directory of out dataset-registry repo:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T15:59:54.303384Z",
     "start_time": "2020-10-18T15:59:52.031402Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc list https://github.com/iterative/dataset-registry use-cases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dvc get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T16:01:42.569767Z",
     "start_time": "2020-10-18T16:00:07.561196Z"
    }
   },
   "outputs": [],
   "source": [
    "# dvc get = just download dataset\n",
    "\n",
    "!dvc get https://github.com/iterative/dataset-registry \\\n",
    "          use-cases/cats-dogs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T16:01:42.707218Z",
     "start_time": "2020-10-18T16:01:42.577558Z"
    }
   },
   "outputs": [],
   "source": [
    "# DVC doesn't control cats-dogs/ folder. There is no cats-dogs.dvc \n",
    "\n",
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dvc import "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:25:48.351814Z",
     "start_time": "2020-10-18T18:25:44.602900Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# dvc import = download dataset and add it under DVC control = dvc get + dvc add\n",
    "\n",
    "!dvc import git@github.com:iterative/example-get-started \\\n",
    "             data/data.xml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:26:07.578907Z",
     "start_time": "2020-10-18T18:26:07.452584Z"
    }
   },
   "outputs": [],
   "source": [
    "# New data.xml file and data.xml.dvc appeared \n",
    "\n",
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:26:43.378443Z",
     "start_time": "2020-10-18T18:26:43.258886Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat data.xml.dvc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dvc import-url\n",
    "\n",
    "To illustrate these examples we will be using the project explained in Example: Tracking a remote file https://dvc.org/doc/command-reference/import-url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:27:51.258398Z",
     "start_time": "2020-10-18T18:27:27.521335Z"
    }
   },
   "outputs": [],
   "source": [
    "!dvc import-url https://data.dvc.org/get-started/data.xml \\\n",
    "                 data/data.xml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:27:51.394124Z",
     "start_time": "2020-10-18T18:27:51.272975Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat data.xml.dvc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Special section"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What is DVC-file?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data file internals\n",
    "\n",
    "\n",
    ">    If you take a look at the DVC-file, you will see that only outputs are defined in outs. \n",
    "    In this file, only one output is defined. The output contains the data file path in the repository and md5 cache.\n",
    "    This md5 cache determines a location of the actual content file in DVC cache directory .dvc/cache\n",
    "    >> Output from DVC-files defines the relationship between the data file path in a repository and the path in a cache directory. See also DVC File Format\n",
    "\n",
    "\n",
    "\n",
    "(c) dvc.org https://dvc.org/doc/tutorial/define-ml-pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:29:17.651228Z",
     "start_time": "2020-10-18T18:29:17.531261Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat datadir.dvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:29:19.569662Z",
     "start_time": "2020-10-18T18:29:19.380033Z"
    }
   },
   "outputs": [],
   "source": [
    "!du -sh .dvc/cache/*/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:29:21.084408Z",
     "start_time": "2020-10-18T18:29:20.933031Z"
    }
   },
   "outputs": [],
   "source": [
    "!tree .dvc/cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:29:21.633128Z",
     "start_time": "2020-10-18T18:29:21.492563Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat .dvc/cache/b6/923e1e4ad16ea1a7e2b328842d56a2.dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:29:30.517415Z",
     "start_time": "2020-10-18T18:29:24.588343Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat .dvc/cache/a3/04afb96060aad90176268345e10355"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Review Files and Directories created by DVC\n",
    "\n",
    "> Once initialized in a project, DVC populates its installation directory (.dvc/) with the internal files and directories needed for DVC operation: https://dvc.org/doc/user-guide/dvc-files-and-directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:33:35.571643Z",
     "start_time": "2020-10-18T18:33:35.450507Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls -la .dvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:33:46.854782Z",
     "start_time": "2020-10-18T18:33:46.734941Z"
    }
   },
   "outputs": [],
   "source": [
    "!cat .dvc/.gitignore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:32:53.122608Z",
     "start_time": "2020-10-18T18:32:52.995780Z"
    }
   },
   "outputs": [],
   "source": [
    "!tree .dvc/plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:33:04.095909Z",
     "start_time": "2020-10-18T18:33:03.970704Z"
    }
   },
   "outputs": [],
   "source": [
    "!tree .dvc/tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-09T08:12:06.692580Z",
     "start_time": "2019-06-09T08:12:04.239910Z"
    }
   },
   "source": [
    "## Explore Structure of cache directory\n",
    "\n",
    "> There are two ways in which the data is stored in cache: As a single file (eg. data.csv), or a directory of files.\n",
    ">\n",
    ">\n",
    ">For the first case, we calculate the file hash, a 32 characters long string (usually MD5). The first two characters are used to name the directory inside .dvc/cache, and the rest become the file name of the cached file.\n",
    ">\n",
    ">\n",
    ">>Note that file hashes are calculated from file contents only. 2 or more files with different names but the same contents can exist in the workspace and be tracked by DVC, but only one copy is stored in the cache. This helps avoid data duplication in cache and remotes.\n",
    "#### (c) dvc.org https://dvc.org/doc/user-guide/dvc-files-and-directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:32:35.781350Z",
     "start_time": "2020-10-18T18:32:35.659339Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls .dvc/cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T18:35:28.590046Z",
     "start_time": "2020-10-18T18:35:28.437790Z"
    }
   },
   "outputs": [],
   "source": [
    "!tree .dvc/cache"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dvc-venv",
   "language": "python",
   "name": "dvc-venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "231px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}