RecordLinkageNCVR_Anonlink.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Privacy Preserving Record Linkage of the NCVR dataset using Anonlink"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import json\n",
    "import pprint\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "import anonlink\n",
    "from clkhash.clk import generate_clk_from_csv\n",
    "from clkhash import schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/ncvr_optimal_schema.json', 'r') as f:\n",
    "    linkage_schema = schema.from_json_file(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfa = pd.read_csv('data/ncvoter-20140619-temporal-balanced-ratio-1to1-a.csv', encoding='latin-1')\n",
    "dfa.drop_duplicates(subset='voter_id', keep='last', inplace=True)\n",
    "dfb = pd.read_csv('data/ncvoter-20140619-temporal-balanced-ratio-1to1-b.csv', encoding='latin-1')\n",
    "dfb.drop_duplicates(subset='voter_id', keep='last', inplace=True)\n",
    "dfa = dfa.reset_index(drop=True)[:100000]\n",
    "dfb = dfb.reset_index(drop=True)[:100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_csv = io.StringIO()\n",
    "dfa.drop(columns=['voter_id']).to_csv(a_csv, index=False)\n",
    "a_csv.seek(0)\n",
    "b_csv = io.StringIO()\n",
    "dfb.drop(columns=['voter_id']).to_csv(b_csv, index=False)\n",
    "b_csv.seek(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "generating CLKs: 100%|██████████| 100k/100k [00:29<00:00, 3.39kclk/s, mean=626, std=14.1] \n",
      "generating CLKs: 100%|██████████| 100k/100k [00:33<00:00, 2.96kclk/s, mean=626, std=14.1] \n"
     ]
    }
   ],
   "source": [
    "secret = \"password1234\"\n",
    "clks_a = generate_clk_from_csv(a_csv, secret, linkage_schema)\n",
    "clks_b = generate_clk_from_csv(b_csv, secret, linkage_schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mapping_from_clks(clks_a, clks_b, threshold):\n",
    "    results_candidate_pairs = anonlink.candidate_generation.find_candidate_pairs(\n",
    "            [clks_a, clks_b],\n",
    "            anonlink.similarities.dice_coefficient,\n",
    "            threshold\n",
    "    )\n",
    "    solution = anonlink.solving.greedy_solve(results_candidate_pairs)\n",
    "    print('Found {} matches'.format(len(solution)))\n",
    "    # each entry in `solution` looks like this: '((0, 4039), (1, 2689))'.\n",
    "    # The format is ((dataset_id, row_id), (dataset_id, row_id))\n",
    "    # As we only have two parties in this example, we can remove the dataset_ids.\n",
    "    # Also, turning the solution into a set will make it easier to assess the\n",
    "    # quality of the matching.\n",
    "    return set((a, b) for ((_, a), (_, b)) in solution)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 66706 matches\n",
      "comparing and solving took 156.66797494888306 seconds.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "found_matches = mapping_from_clks(clks_a, clks_b, 0.86)\n",
    "print(f'comparing and solving took {time.time()-start} seconds.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge = pd.merge(dfa['voter_id'].reset_index(), dfb['voter_id'].reset_index(), how='inner', on='voter_id')\n",
    "\n",
    "true_matches = set((row[0], row[1]) for row in merge.drop(columns=['voter_id']).itertuples(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.988, Recall: 0.992\n"
     ]
    }
   ],
   "source": [
    "def describe_matching_quality(found_matches, true_matches):\n",
    "           \n",
    "    tp = len(found_matches & true_matches)\n",
    "    fp = len(found_matches - true_matches)\n",
    "    fn = len(true_matches - found_matches)\n",
    "\n",
    "    precision = tp / (tp + fp)\n",
    "    recall = tp / (tp + fn)\n",
    "\n",
    "    print('Precision: {:.3f}, Recall: {:.3f}'.format(precision, recall))\n",
    "\n",
    "describe_matching_quality(found_matches, true_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}