Bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "orig_nbformat": 2,
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "Bert.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "615df31b294f4e5db6fd4245af0e4910": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cf077f31073b4ebfb34ba96fffa37488",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_41b1f38fce734396ab3e86af3c7d67e6",
              "IPY_MODEL_33fbe6d26e7b4d0c9f36c88878a71df4"
            ]
          }
        },
        "cf077f31073b4ebfb34ba96fffa37488": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41b1f38fce734396ab3e86af3c7d67e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f6bd180fefe64268b5fa90a546202365",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1965,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1965,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c40c2fe9f3884a20a980c32bba12500d"
          }
        },
        "33fbe6d26e7b4d0c9f36c88878a71df4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cbe06faa1b904e939c622d72beb1a006",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1965/1965 [06:50&lt;00:00,  4.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5ded6d8a94ec4556921510dadc8883cc"
          }
        },
        "f6bd180fefe64268b5fa90a546202365": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c40c2fe9f3884a20a980c32bba12500d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cbe06faa1b904e939c622d72beb1a006": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5ded6d8a94ec4556921510dadc8883cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6e084d14ab1244049bc43b7694e74d33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0d203e1636ad40ac9446b5e48658f196",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7a687d063e2740c2ad226dda7dd1f0cb",
              "IPY_MODEL_6b978d7054de41349a50f7a1b8ecc8bd"
            ]
          }
        },
        "0d203e1636ad40ac9446b5e48658f196": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a687d063e2740c2ad226dda7dd1f0cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2a077ff629a7424aad5efc2014c5d26b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1965,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1965,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de14bbb9235f41c4a30599e5d6d56412"
          }
        },
        "6b978d7054de41349a50f7a1b8ecc8bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_635cb103aa744102863b5152efa6c4b1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1965/1965 [06:50&lt;00:00,  4.79it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0fb95afbbd514378885076a21e72941c"
          }
        },
        "2a077ff629a7424aad5efc2014c5d26b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de14bbb9235f41c4a30599e5d6d56412": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "635cb103aa744102863b5152efa6c4b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0fb95afbbd514378885076a21e72941c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f5d1e197f724e51b69602a25a1bfb93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_caccb34b06fc4984a317a6dde1ba5171",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4bae3ac7cd9547afbb1a4b8e8c8fa769",
              "IPY_MODEL_a267a1f162334904aab2ded3cd1353b0"
            ]
          }
        },
        "caccb34b06fc4984a317a6dde1ba5171": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4bae3ac7cd9547afbb1a4b8e8c8fa769": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_37a8a93bdc5e46d48689da55875a39f2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1965,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1965,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fbe69d3d78ed4ebc8fa70dd229dcfe5f"
          }
        },
        "a267a1f162334904aab2ded3cd1353b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e860570a266d4a0e9fae5f9e8ad32964",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1965/1965 [06:49&lt;00:00,  4.80it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f72aac9ddb64334af729a4d41d65e1e"
          }
        },
        "37a8a93bdc5e46d48689da55875a39f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fbe69d3d78ed4ebc8fa70dd229dcfe5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e860570a266d4a0e9fae5f9e8ad32964": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f72aac9ddb64334af729a4d41d65e1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bdcc77c338d540c28ea7d0beefee0d36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ce47873fc5654c2fb16b74dcfe9b71dc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_327526bec8664aeab68500b88ca60a5f",
              "IPY_MODEL_bad702d26e6f47f5aaf4a42f58e5e9c3"
            ]
          }
        },
        "ce47873fc5654c2fb16b74dcfe9b71dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "327526bec8664aeab68500b88ca60a5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_43ebcea774134f53b9d85c9d4c8bd456",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1965,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1965,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a3df0740c0ca436c8d04131717a49ba6"
          }
        },
        "bad702d26e6f47f5aaf4a42f58e5e9c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0caa4584c44a4086937732057b2b7979",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1965/1965 [02:07&lt;00:00, 15.40it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bbfd36197fd84e5d80e8b9ddb8cebd02"
          }
        },
        "43ebcea774134f53b9d85c9d4c8bd456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a3df0740c0ca436c8d04131717a49ba6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0caa4584c44a4086937732057b2b7979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bbfd36197fd84e5d80e8b9ddb8cebd02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HUb3MP1ODEZH",
        "colab_type": "text"
      },
      "source": [
        "# Relation Classification based on Bert"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uzepGlr-DEZJ",
        "colab_type": "text"
      },
      "source": [
        "## For Colab Environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltBUinCeDEZK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 853
        },
        "outputId": "8df6624e-ed27-4986-a809-37392f5313d5"
      },
      "source": [
        "try:\n",
        "    import google.colab\n",
        "    IN_COLAB = True\n",
        "except:\n",
        "    IN_COLAB = False\n",
        "if IN_COLAB:\n",
        "    !git clone https://github.com/Molin-L/RLRC.git\n",
        "    !pip install transformers\n",
        "    !pip install wandb\n",
        "    !wandb login f8b9d4c1a91d9a60c0dfe9934d5d550598750c33\n",
        "    %cd RLRC"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'RLRC' already exists and is not an empty directory.\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: tokenizers==0.8.1.rc1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.6/dist-packages (0.9.4)\n",
            "Requirement already satisfied: sentry-sdk>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (0.16.4)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (2.23.0)\n",
            "Requirement already satisfied: subprocess32>=3.5.3 in /usr/local/lib/python3.6/dist-packages (from wandb) (3.5.4)\n",
            "Requirement already satisfied: GitPython>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (3.1.7)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (0.4.0)\n",
            "Requirement already satisfied: PyYAML>=3.10 in /usr/local/lib/python3.6/dist-packages (from wandb) (3.13)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from wandb) (2.8.1)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (5.4.8)\n",
            "Requirement already satisfied: configparser>=3.8.1 in /usr/local/lib/python3.6/dist-packages (from wandb) (5.0.0)\n",
            "Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (7.352.0)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (7.1.2)\n",
            "Requirement already satisfied: watchdog>=0.8.3 in /usr/local/lib/python3.6/dist-packages (from wandb) (0.10.3)\n",
            "Requirement already satisfied: shortuuid>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (1.0.1)\n",
            "Requirement already satisfied: gql==0.2.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (0.2.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from sentry-sdk>=0.4.0->wandb) (2020.6.20)\n",
            "Requirement already satisfied: urllib3>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from sentry-sdk>=0.4.0->wandb) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.6/dist-packages (from GitPython>=1.0.0->wandb) (4.0.5)\n",
            "Requirement already satisfied: pathtools>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from watchdog>=0.8.3->wandb) (0.1.2)\n",
            "Requirement already satisfied: graphql-core<2,>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from gql==0.2.0->wandb) (1.1)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.6/dist-packages (from gql==0.2.0->wandb) (2.3)\n",
            "Requirement already satisfied: smmap<4,>=3.0.1 in /usr/local/lib/python3.6/dist-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb) (3.0.4)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
            "\u001b[32mSuccessfully logged in to Weights & Biases!\u001b[0m\n",
            "/content/RLRC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2w9NtaqNDEZP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchtext.data import Field, TabularDataset, BucketIterator, Iterator\n",
        "import torchtext\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import copy\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm.notebook import tqdm\n",
        "from utils import format_time\n",
        "from transformers import BertTokenizer, BertForSequenceClassification, DistilBertTokenizer, DistilBertForSequenceClassification, AdamW, get_linear_schedule_with_warmup\n",
        "import time\n",
        "import wandb\n",
        "import random\n",
        "import RLRC_dataloader\n",
        "from RLRC_Bert_model import RC_BERT\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g2qfSmQfDEZS",
        "colab_type": "text"
      },
      "source": [
        "## Load Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eNziKtBaDEZT",
        "colab_type": "text"
      },
      "source": [
        "## Bert Model Setting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "klgFE0NnDEZU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "ff3022b3-111e-434b-f973-b1d42cf986f3"
      },
      "source": [
        "print('Loading pretrained tokenizer...')\n",
        "pretrain_model = [\"bert-base-uncased\", \"distilbert-base-uncased\"]\n",
        "pretrain_model = pretrain_model[1]  \n",
        "additional_special_tokens = ['<e1>', '</e1>', '<e2>', '</e2>']\n",
        "tokenizer = DistilBertTokenizer.from_pretrained(pretrain_model, do_lower_case=False)\n",
        "print(len(tokenizer))\n",
        "tokenizer.add_tokens(['<e1>', '</e1>', '<e2>', '</e2>'])\n",
        "print(len(tokenizer))\n",
        "e1_id = tokenizer.convert_tokens_to_ids('<e1>')\n",
        "e2_id = tokenizer.convert_tokens_to_ids('<e2>')\n",
        "\n",
        "print(e1_id, e2_id)\n",
        "assert e1_id != e2_id != 1"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading pretrained tokenizer...\n",
            "30522\n",
            "30526\n",
            "30522 30524\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZecRrRoGEBKq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b694cc74-e455-4684-fc74-dc6fc732e1ab"
      },
      "source": [
        "if IN_COLAB:\n",
        "    !mkdir data"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mkdir: cannot create directory ‘data’: File exists\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PWxrSWotDEZc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "1374a09e-17e8-4e54-d240-f96a37410d41"
      },
      "source": [
        "wandb.init(\n",
        "        project=\"RLRC_BERT\",\n",
        "        config={\n",
        "            'pretrain_model': \"distilbert-base-uncased\",\n",
        "            'num_classes': 53,\n",
        "            'lr': 0.001,\n",
        "            'dropout': 0.5,\n",
        "            'epochs': 3\n",
        "        }\n",
        "    )\n",
        "wb_config = wandb.config"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/molin/RLRC_BERT\" target=\"_blank\">https://app.wandb.ai/molin/RLRC_BERT</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/molin/RLRC_BERT/runs/3bhyh7al\" target=\"_blank\">https://app.wandb.ai/molin/RLRC_BERT/runs/3bhyh7al</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3kaA2ztAFa5_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "6f07c221-8614-4224-bcdc-a8461827f887"
      },
      "source": [
        "train_dataloader, validation_dataloader = RLRC_dataloader.get_dataloader()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load Bert data from .pkl files...\n",
            "\t62,859 training samples\n",
            "\t6,985 validation samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WTjMPlMBDEZf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss,\n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5bTYgX0OPgR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GEkTHeIsDEZi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "615df31b294f4e5db6fd4245af0e4910",
            "cf077f31073b4ebfb34ba96fffa37488",
            "41b1f38fce734396ab3e86af3c7d67e6",
            "33fbe6d26e7b4d0c9f36c88878a71df4",
            "f6bd180fefe64268b5fa90a546202365",
            "c40c2fe9f3884a20a980c32bba12500d",
            "cbe06faa1b904e939c622d72beb1a006",
            "5ded6d8a94ec4556921510dadc8883cc",
            "6e084d14ab1244049bc43b7694e74d33",
            "0d203e1636ad40ac9446b5e48658f196",
            "7a687d063e2740c2ad226dda7dd1f0cb",
            "6b978d7054de41349a50f7a1b8ecc8bd",
            "2a077ff629a7424aad5efc2014c5d26b",
            "de14bbb9235f41c4a30599e5d6d56412",
            "635cb103aa744102863b5152efa6c4b1",
            "0fb95afbbd514378885076a21e72941c",
            "1f5d1e197f724e51b69602a25a1bfb93",
            "caccb34b06fc4984a317a6dde1ba5171",
            "4bae3ac7cd9547afbb1a4b8e8c8fa769",
            "a267a1f162334904aab2ded3cd1353b0",
            "37a8a93bdc5e46d48689da55875a39f2",
            "fbe69d3d78ed4ebc8fa70dd229dcfe5f",
            "e860570a266d4a0e9fae5f9e8ad32964",
            "2f72aac9ddb64334af729a4d41d65e1e"
          ]
        },
        "outputId": "e8f615a3-f767-4762-d0da-d5abad099ca5"
      },
      "source": [
        "config = wb_config\n",
        "# print(config)\n",
        "model = RC_BERT(config)\n",
        "optimizer = AdamW(\n",
        "    model.parameters(),\n",
        "    lr=wb_config['lr'],\n",
        "    eps=1e-8\n",
        ")\n",
        "total_steps = len(train_dataloader) * config['epochs']\n",
        "print(total_steps)\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps=0,  # Default value in run_glue.py\n",
        "                                            num_training_steps=total_steps)\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")\n",
        "model.to(device)\n",
        "epochs = config['epochs']\n",
        "wandb.watch(model)\n",
        "for epoch_i in range(config['epochs']):\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "    t0 = time.time()\n",
        "    total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "    model.train()\n",
        "    for step, batch in enumerate(tqdm(train_dataloader)):\n",
        "        # Progress update every 40 batches.\n",
        "        batch_counts += 1\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "\n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(\n",
        "                step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the\n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because\n",
        "        # accumulating the gradients is \"convenient while training RNNs\".\n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()\n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # The documentation for this `model` function is here:\n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        # It returns different numbers of parameters depending on what arguments\n",
        "        # arge given and what flags are set. For our useage here, it returns\n",
        "        # the loss (because we provided labels) and the \"logits\"--the model\n",
        "        # outputs prior to activation.\n",
        "        logits = model(b_input_ids, b_input_mask)\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        batch_loss += loss.item()\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = batch_loss / batch_counts\n",
        "\n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "\n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using\n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():\n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which\n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here:\n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "            # values prior to applying an activation function like the softmax.\n",
        "            logits = model(b_input_ids,\n",
        "                            attention_mask=b_input_mask,\n",
        "                            )\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5895\n",
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n",
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "615df31b294f4e5db6fd4245af0e4910",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1965.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch    40  of  1,965.    Elapsed: 0:00:09.\n",
            "  Batch    80  of  1,965.    Elapsed: 0:00:17.\n",
            "  Batch   120  of  1,965.    Elapsed: 0:00:26.\n",
            "  Batch   160  of  1,965.    Elapsed: 0:00:34.\n",
            "  Batch   200  of  1,965.    Elapsed: 0:00:43.\n",
            "  Batch   240  of  1,965.    Elapsed: 0:00:51.\n",
            "  Batch   280  of  1,965.    Elapsed: 0:00:59.\n",
            "  Batch   320  of  1,965.    Elapsed: 0:01:07.\n",
            "  Batch   360  of  1,965.    Elapsed: 0:01:16.\n",
            "  Batch   400  of  1,965.    Elapsed: 0:01:24.\n",
            "  Batch   440  of  1,965.    Elapsed: 0:01:33.\n",
            "  Batch   480  of  1,965.    Elapsed: 0:01:41.\n",
            "  Batch   520  of  1,965.    Elapsed: 0:01:49.\n",
            "  Batch   560  of  1,965.    Elapsed: 0:01:58.\n",
            "  Batch   600  of  1,965.    Elapsed: 0:02:06.\n",
            "  Batch   640  of  1,965.    Elapsed: 0:02:14.\n",
            "  Batch   680  of  1,965.    Elapsed: 0:02:23.\n",
            "  Batch   720  of  1,965.    Elapsed: 0:02:31.\n",
            "  Batch   760  of  1,965.    Elapsed: 0:02:39.\n",
            "  Batch   800  of  1,965.    Elapsed: 0:02:48.\n",
            "  Batch   840  of  1,965.    Elapsed: 0:02:56.\n",
            "  Batch   880  of  1,965.    Elapsed: 0:03:04.\n",
            "  Batch   920  of  1,965.    Elapsed: 0:03:13.\n",
            "  Batch   960  of  1,965.    Elapsed: 0:03:21.\n",
            "  Batch 1,000  of  1,965.    Elapsed: 0:03:30.\n",
            "  Batch 1,040  of  1,965.    Elapsed: 0:03:38.\n",
            "  Batch 1,080  of  1,965.    Elapsed: 0:03:46.\n",
            "  Batch 1,120  of  1,965.    Elapsed: 0:03:55.\n",
            "  Batch 1,160  of  1,965.    Elapsed: 0:04:03.\n",
            "  Batch 1,200  of  1,965.    Elapsed: 0:04:11.\n",
            "  Batch 1,240  of  1,965.    Elapsed: 0:04:20.\n",
            "  Batch 1,280  of  1,965.    Elapsed: 0:04:28.\n",
            "  Batch 1,320  of  1,965.    Elapsed: 0:04:36.\n",
            "  Batch 1,360  of  1,965.    Elapsed: 0:04:45.\n",
            "  Batch 1,400  of  1,965.    Elapsed: 0:04:53.\n",
            "  Batch 1,440  of  1,965.    Elapsed: 0:05:01.\n",
            "  Batch 1,480  of  1,965.    Elapsed: 0:05:10.\n",
            "  Batch 1,520  of  1,965.    Elapsed: 0:05:18.\n",
            "  Batch 1,560  of  1,965.    Elapsed: 0:05:27.\n",
            "  Batch 1,600  of  1,965.    Elapsed: 0:05:35.\n",
            "  Batch 1,640  of  1,965.    Elapsed: 0:05:43.\n",
            "  Batch 1,680  of  1,965.    Elapsed: 0:05:52.\n",
            "  Batch 1,720  of  1,965.    Elapsed: 0:06:00.\n",
            "  Batch 1,760  of  1,965.    Elapsed: 0:06:08.\n",
            "  Batch 1,800  of  1,965.    Elapsed: 0:06:16.\n",
            "  Batch 1,840  of  1,965.    Elapsed: 0:06:25.\n",
            "  Batch 1,880  of  1,965.    Elapsed: 0:06:33.\n",
            "  Batch 1,920  of  1,965.    Elapsed: 0:06:42.\n",
            "  Batch 1,960  of  1,965.    Elapsed: 0:06:50.\n",
            "\n",
            "\n",
            "  Average training loss: 1.67\n",
            "  Training epcoh took: 0:06:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.50\n",
            "  Validation Loss: 1.65\n",
            "  Validation took: 0:00:14\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6e084d14ab1244049bc43b7694e74d33",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1965.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch    40  of  1,965.    Elapsed: 0:00:09.\n",
            "  Batch    80  of  1,965.    Elapsed: 0:00:17.\n",
            "  Batch   120  of  1,965.    Elapsed: 0:00:25.\n",
            "  Batch   160  of  1,965.    Elapsed: 0:00:34.\n",
            "  Batch   200  of  1,965.    Elapsed: 0:00:42.\n",
            "  Batch   240  of  1,965.    Elapsed: 0:00:50.\n",
            "  Batch   280  of  1,965.    Elapsed: 0:00:59.\n",
            "  Batch   320  of  1,965.    Elapsed: 0:01:07.\n",
            "  Batch   360  of  1,965.    Elapsed: 0:01:15.\n",
            "  Batch   400  of  1,965.    Elapsed: 0:01:24.\n",
            "  Batch   440  of  1,965.    Elapsed: 0:01:32.\n",
            "  Batch   480  of  1,965.    Elapsed: 0:01:40.\n",
            "  Batch   520  of  1,965.    Elapsed: 0:01:49.\n",
            "  Batch   560  of  1,965.    Elapsed: 0:01:57.\n",
            "  Batch   600  of  1,965.    Elapsed: 0:02:05.\n",
            "  Batch   640  of  1,965.    Elapsed: 0:02:14.\n",
            "  Batch   680  of  1,965.    Elapsed: 0:02:22.\n",
            "  Batch   720  of  1,965.    Elapsed: 0:02:30.\n",
            "  Batch   760  of  1,965.    Elapsed: 0:02:39.\n",
            "  Batch   800  of  1,965.    Elapsed: 0:02:47.\n",
            "  Batch   840  of  1,965.    Elapsed: 0:02:55.\n",
            "  Batch   880  of  1,965.    Elapsed: 0:03:04.\n",
            "  Batch   920  of  1,965.    Elapsed: 0:03:12.\n",
            "  Batch   960  of  1,965.    Elapsed: 0:03:20.\n",
            "  Batch 1,000  of  1,965.    Elapsed: 0:03:29.\n",
            "  Batch 1,040  of  1,965.    Elapsed: 0:03:37.\n",
            "  Batch 1,080  of  1,965.    Elapsed: 0:03:46.\n",
            "  Batch 1,120  of  1,965.    Elapsed: 0:03:54.\n",
            "  Batch 1,160  of  1,965.    Elapsed: 0:04:02.\n",
            "  Batch 1,200  of  1,965.    Elapsed: 0:04:11.\n",
            "  Batch 1,240  of  1,965.    Elapsed: 0:04:19.\n",
            "  Batch 1,280  of  1,965.    Elapsed: 0:04:27.\n",
            "  Batch 1,320  of  1,965.    Elapsed: 0:04:36.\n",
            "  Batch 1,360  of  1,965.    Elapsed: 0:04:44.\n",
            "  Batch 1,400  of  1,965.    Elapsed: 0:04:52.\n",
            "  Batch 1,440  of  1,965.    Elapsed: 0:05:01.\n",
            "  Batch 1,480  of  1,965.    Elapsed: 0:05:09.\n",
            "  Batch 1,520  of  1,965.    Elapsed: 0:05:17.\n",
            "  Batch 1,560  of  1,965.    Elapsed: 0:05:26.\n",
            "  Batch 1,600  of  1,965.    Elapsed: 0:05:34.\n",
            "  Batch 1,640  of  1,965.    Elapsed: 0:05:43.\n",
            "  Batch 1,680  of  1,965.    Elapsed: 0:05:51.\n",
            "  Batch 1,720  of  1,965.    Elapsed: 0:05:59.\n",
            "  Batch 1,760  of  1,965.    Elapsed: 0:06:08.\n",
            "  Batch 1,800  of  1,965.    Elapsed: 0:06:16.\n",
            "  Batch 1,840  of  1,965.    Elapsed: 0:06:24.\n",
            "  Batch 1,880  of  1,965.    Elapsed: 0:06:32.\n",
            "  Batch 1,920  of  1,965.    Elapsed: 0:06:41.\n",
            "  Batch 1,960  of  1,965.    Elapsed: 0:06:49.\n",
            "\n",
            "\n",
            "  Average training loss: 1.65\n",
            "  Training epcoh took: 0:06:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.50\n",
            "  Validation Loss: 1.65\n",
            "  Validation took: 0:00:14\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1f5d1e197f724e51b69602a25a1bfb93",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1965.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch    40  of  1,965.    Elapsed: 0:00:08.\n",
            "  Batch    80  of  1,965.    Elapsed: 0:00:17.\n",
            "  Batch   120  of  1,965.    Elapsed: 0:00:25.\n",
            "  Batch   160  of  1,965.    Elapsed: 0:00:34.\n",
            "  Batch   200  of  1,965.    Elapsed: 0:00:42.\n",
            "  Batch   240  of  1,965.    Elapsed: 0:00:50.\n",
            "  Batch   280  of  1,965.    Elapsed: 0:00:59.\n",
            "  Batch   320  of  1,965.    Elapsed: 0:01:07.\n",
            "  Batch   360  of  1,965.    Elapsed: 0:01:15.\n",
            "  Batch   400  of  1,965.    Elapsed: 0:01:24.\n",
            "  Batch   440  of  1,965.    Elapsed: 0:01:32.\n",
            "  Batch   480  of  1,965.    Elapsed: 0:01:40.\n",
            "  Batch   520  of  1,965.    Elapsed: 0:01:49.\n",
            "  Batch   560  of  1,965.    Elapsed: 0:01:57.\n",
            "  Batch   600  of  1,965.    Elapsed: 0:02:05.\n",
            "  Batch   640  of  1,965.    Elapsed: 0:02:14.\n",
            "  Batch   680  of  1,965.    Elapsed: 0:02:22.\n",
            "  Batch   720  of  1,965.    Elapsed: 0:02:30.\n",
            "  Batch   760  of  1,965.    Elapsed: 0:02:39.\n",
            "  Batch   800  of  1,965.    Elapsed: 0:02:47.\n",
            "  Batch   840  of  1,965.    Elapsed: 0:02:55.\n",
            "  Batch   880  of  1,965.    Elapsed: 0:03:04.\n",
            "  Batch   920  of  1,965.    Elapsed: 0:03:12.\n",
            "  Batch   960  of  1,965.    Elapsed: 0:03:20.\n",
            "  Batch 1,000  of  1,965.    Elapsed: 0:03:29.\n",
            "  Batch 1,040  of  1,965.    Elapsed: 0:03:37.\n",
            "  Batch 1,080  of  1,965.    Elapsed: 0:03:46.\n",
            "  Batch 1,120  of  1,965.    Elapsed: 0:03:54.\n",
            "  Batch 1,160  of  1,965.    Elapsed: 0:04:02.\n",
            "  Batch 1,200  of  1,965.    Elapsed: 0:04:11.\n",
            "  Batch 1,240  of  1,965.    Elapsed: 0:04:19.\n",
            "  Batch 1,280  of  1,965.    Elapsed: 0:04:27.\n",
            "  Batch 1,320  of  1,965.    Elapsed: 0:04:36.\n",
            "  Batch 1,360  of  1,965.    Elapsed: 0:04:44.\n",
            "  Batch 1,400  of  1,965.    Elapsed: 0:04:52.\n",
            "  Batch 1,440  of  1,965.    Elapsed: 0:05:01.\n",
            "  Batch 1,480  of  1,965.    Elapsed: 0:05:09.\n",
            "  Batch 1,520  of  1,965.    Elapsed: 0:05:17.\n",
            "  Batch 1,560  of  1,965.    Elapsed: 0:05:25.\n",
            "  Batch 1,600  of  1,965.    Elapsed: 0:05:34.\n",
            "  Batch 1,640  of  1,965.    Elapsed: 0:05:42.\n",
            "  Batch 1,680  of  1,965.    Elapsed: 0:05:50.\n",
            "  Batch 1,720  of  1,965.    Elapsed: 0:05:59.\n",
            "  Batch 1,760  of  1,965.    Elapsed: 0:06:07.\n",
            "  Batch 1,800  of  1,965.    Elapsed: 0:06:15.\n",
            "  Batch 1,840  of  1,965.    Elapsed: 0:06:24.\n",
            "  Batch 1,880  of  1,965.    Elapsed: 0:06:32.\n",
            "  Batch 1,920  of  1,965.    Elapsed: 0:06:40.\n",
            "  Batch 1,960  of  1,965.    Elapsed: 0:06:49.\n",
            "\n",
            "\n",
            "  Average training loss: 1.65\n",
            "  Training epcoh took: 0:06:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.50\n",
            "  Validation Loss: 1.65\n",
            "  Validation took: 0:00:14\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gTzGx34fnnlp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "outpath = './out/bert_model.pth'"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mTq-lAlMoPij",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.save(model.state_dict(), outpath)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UMqxp_8arsBz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "36ff5040-7fe1-4b64-9f61-e77f4383c8c8"
      },
      "source": [
        "predict_result = []\n",
        "model.eval()\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RC_BERT(\n",
              "  (Bert): DistilBertModel(\n",
              "    (embeddings): Embeddings(\n",
              "      (word_embeddings): Embedding(30526, 768)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (transformer): Transformer(\n",
              "      (layer): ModuleList(\n",
              "        (0): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (1): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (2): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (3): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (4): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (5): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): Sequential(\n",
              "    (0): Linear(in_features=768, out_features=250, bias=True)\n",
              "    (1): ReLU()\n",
              "    (2): Linear(in_features=250, out_features=53, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mPFX_YFNv3I0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "bdcc77c338d540c28ea7d0beefee0d36",
            "ce47873fc5654c2fb16b74dcfe9b71dc",
            "327526bec8664aeab68500b88ca60a5f",
            "bad702d26e6f47f5aaf4a42f58e5e9c3",
            "43ebcea774134f53b9d85c9d4c8bd456",
            "a3df0740c0ca436c8d04131717a49ba6",
            "0caa4584c44a4086937732057b2b7979",
            "bbfd36197fd84e5d80e8b9ddb8cebd02"
          ]
        },
        "outputId": "81e2ed11-77d0-43b8-cfc0-97e5bc80982f"
      },
      "source": [
        "true_y = []\n",
        "pred_y = None\n",
        "\n",
        "for batch in tqdm(train_dataloader):\n",
        "    b_input_ids = batch[0].to(device)\n",
        "    b_input_mask = batch[1].to(device)\n",
        "    b_labels = batch[2].to(device)\n",
        "\n",
        "    # Tell pytorch not to bother with constructing the compute graph during\n",
        "    # the forward pass, since this is only needed for backprop (training).\n",
        "    with torch.no_grad():\n",
        "        logits = model(b_input_ids, b_input_mask)\n",
        "    logits = logits.detach().cpu()\n",
        "    label_ids = b_labels.to('cpu').numpy()\n",
        "    batch_pred_np = logits.numpy()\n",
        "    if len(true_y) == 0:\n",
        "        true_y = label_ids\n",
        "        pred_y = batch_pred_np\n",
        "    else:\n",
        "        true_y = np.concatenate((true_y, label_ids), axis=None)\n",
        "        pred_y = np.concatenate((pred_y, batch_pred_np), axis=None)\n",
        "\n"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bdcc77c338d540c28ea7d0beefee0d36",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1965.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e1LBtr34v4Vi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.save('./out/pred_y.npy', pred_y, allow_pickle=True)\n",
        "np.save('./out/true_y.npy', true_y, allow_pickle=True)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3sslWLOVzTPM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "28600a68-22e3-475a-f0a5-8e447b71024e"
      },
      "source": [
        "!ls ./data -lh"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 405M\n",
            "-rw-r--r-- 1 root root  69M Aug 14 13:04 bert_attention_masks.pkl\n",
            "-rw-r--r-- 1 root root  69M Aug 14 13:04 bert_input_ids.pkl\n",
            "-rw-r--r-- 1 root root 547K Aug 14 13:04 bert_labels.pkl\n",
            "-rw-r--r-- 1 root root 254M Aug 14 15:41 model.pth\n",
            "-rw-r--r-- 1 root root  13M Aug 14 16:34 pred_y.npy\n",
            "-rw-r--r-- 1 root root 492K Aug 14 16:34 true_y.npy\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4eCMhi-0uMW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}