SF_annotation_ChatGPT.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "euhQlpp4BB95"
      },
      "source": [
        "## Some preliminary steps (installations, imports, etc)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Td0MLE6CQIJ0"
      },
      "outputs": [],
      "source": [
        "!pip install openai -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9MoX4mus8gtv"
      },
      "outputs": [],
      "source": [
        "\n",
        "!pip install transformers -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dLRc0HQzBHiV"
      },
      "outputs": [],
      "source": [
        "!pip install python-Levenshtein -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BhAtbNPbYujd"
      },
      "outputs": [],
      "source": [
        "pip install git+https://github.com/Kpetyxova/DeepPavlov.git@feat/topic_shift"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MKn4DR0EYuje"
      },
      "outputs": [],
      "source": [
        "pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "djv1dDlfQRxA"
      },
      "outputs": [],
      "source": [
        "import openai\n",
        "import time\n",
        "from transformers import pipeline\n",
        "import pandas as pd\n",
        "import xml.etree.ElementTree as ET\n",
        "from Levenshtein import distance as lev\n",
        "import json\n",
        "from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score\n",
        "import random\n",
        "from collections import Counter\n",
        "import numpy as np\n",
        "from tqdm.notebook import tqdm\n",
        "import sys\n",
        "import os\n",
        "import re\n",
        "from deeppavlov import build_model\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w7Fjjklx8t_C"
      },
      "outputs": [],
      "source": [
        "pipe = pipeline(model=\"facebook/bart-large-mnli\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P_KVES-jYxMj"
      },
      "outputs": [],
      "source": [
        "model = build_model('superglue_topic_shift.json')"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "MBjUKr0TLkty"
      },
      "source": [
        "# Working with datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p5DWH85sd_Ld"
      },
      "outputs": [],
      "source": [
        "def make_dialog_dict(filename, sep='\\t', filetype='common'):\n",
        "    if filetype == 'common':\n",
        "        dialogs = pd.read_csv(filename, sep=sep)\n",
        "        d_list = dialogs[['INPUT:dialog_id', 'INPUT:utterances']].values.tolist()\n",
        "        fin_dict = {}\n",
        "        for item in d_list:\n",
        "            # split the source_data string into a list of utterances\n",
        "            utterances = re.findall(r'<span.*?>(.*?)</span>:\\s*\"(.*?)\"', item[1])\n",
        "\n",
        "            # initialize the target_data list\n",
        "            target_data = []\n",
        "\n",
        "            # iterate through the utterances and convert them to dictionaries\n",
        "            for i, (speaker, utterance) in enumerate(utterances):\n",
        "                # extract the speaker number from the class name\n",
        "                speaker_num = int(re.search(r'speaker_(\\d+)', speaker).group(1))\n",
        "\n",
        "                # create the dictionary and append it to the target_data list\n",
        "                target_data.append({\"speaker\": speaker_num, \"utterance\": utterance.strip().replace('\\\\', '')})\n",
        "            fin_dict[str(item[0])] = target_data\n",
        "    elif filetype == 'gold':\n",
        "        dialogs = pd.read_csv(filename, sep=sep)\n",
        "        d_list = dialogs[['dialogue_id', 'speaker', 'text']].values.tolist()\n",
        "        fin_dict = {}\n",
        "        for id, speaker, text in d_list:\n",
        "            if 'O' in speaker:\n",
        "                speaker = '1'\n",
        "            elif 'X' in speaker:\n",
        "                speaker = '2'\n",
        "            if str(id) not in fin_dict:\n",
        "                fin_dict[str(id)] = [{\"speaker\": speaker, \"utterance\": text.strip()}]\n",
        "            else:\n",
        "                fin_dict[str(id)] += [{\"speaker\": speaker, \"utterance\": text.strip()}]\n",
        "\n",
        "    # print the target_data list\n",
        "    return fin_dict"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r3jMHDVoDAPL"
      },
      "source": [
        "##Loading XML trees"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qrGhVwP3vWGq"
      },
      "outputs": [],
      "source": [
        "tree = ET.parse('files/questions.drawio.xml')\n",
        "root = tree.getroot()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oRzRz4BlImWg"
      },
      "outputs": [],
      "source": [
        "tree_extra = ET.parse('files/questions_extra.drawio.xml')\n",
        "root_extra = tree_extra.getroot()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6R0Xl5Pn1UcA"
      },
      "source": [
        "# Loading instructions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TISxW4H91TWe"
      },
      "outputs": [],
      "source": [
        "instructions = pd.read_csv('files/instructions.tsv', sep='\\t')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_w9crAfkp3dJ"
      },
      "outputs": [],
      "source": [
        "import re #добавила кастомную чистку инструкций от тегов (Ника)\n",
        "CLEANR = re.compile('<.*?>')\n",
        "DASH = re.compile('(?:([^>])<\\/span><br>(<br>)?(<\\/span>)*(<span class=\"text_attention\">)*)|(?:([^>])<\\/span>(<p>)+(<\\/span>)*(<span class=\"text_attention\">)*)')\n",
        "\n",
        "def cleanhtml(raw_html):\n",
        "  cleantext = \"\"\n",
        "  if type(raw_html) == str:\n",
        "      raw_html = raw_html.replace('\\n', '')\n",
        "      raw_html = re.sub(DASH, '\\g<1> -- ', raw_html)\n",
        "      raw_html = raw_html.replace('<br>', '\\n').replace('<span class=\"text_attention\">', '\\n\\n')\n",
        "      cleantext = re.sub(CLEANR, '', raw_html)\n",
        "  return cleantext"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AhjQAuyBp4pV"
      },
      "outputs": [],
      "source": [
        "instructions['instruction'] = instructions['instruction'].apply(cleanhtml)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Zlr85UK-Dixv"
      },
      "source": [
        "# Functions for annotation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ttNwZ2ClNPCW"
      },
      "outputs": [],
      "source": [
        "annotation_log = \"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lFpQWY_hzvBl"
      },
      "outputs": [],
      "source": [
        "def fill_annotation_log(additional_text):\n",
        "  global annotation_log\n",
        "  annotation_log = annotation_log + \"\"\"\n",
        "\"\"\"\n",
        "  annotation_log = annotation_log + additional_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ua3DABro80ZD"
      },
      "outputs": [],
      "source": [
        "def preprocess_answer(right_answer_text, possible_answers):\n",
        "  output = pipe(right_answer_text, candidate_labels=possible_answers)\n",
        "  return output['labels'][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G-PoiMU97QIV"
      },
      "outputs": [],
      "source": [
        "def select_answer(right_answer_text, possible_answers, target_nodes):\n",
        "  cands = []\n",
        "  if 'label' in right_answer_text:\n",
        "    right_answer_text = right_answer_text.replace('.', '')\n",
        "  for i in range(len(target_nodes)):\n",
        "    if lev(right_answer_text, possible_answers[i])<=2:\n",
        "      cands.append((lev(right_answer_text, possible_answers[i]), (target_nodes[i], i)))\n",
        "  if cands:\n",
        "    sorted_cands = min(cands, key=lambda x: x[0])\n",
        "    return sorted_cands[1]\n",
        "\n",
        "  processed_right_answer_text = preprocess_answer(right_answer_text, possible_answers)\n",
        "  for i in range(len(target_nodes)):\n",
        "    if possible_answers[i]==processed_right_answer_text:\n",
        "      return target_nodes[i], i"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZHhB_nYYQhE6"
      },
      "outputs": [],
      "source": [
        "def chatGPT_annotate(dialog, question, level_recursion=0):\n",
        "  if level_recursion > 10:\n",
        "      return 'Tried to get response 10 times and failed. Something went really wrong.'\n",
        "  try:\n",
        "    response = openai.ChatCompletion.create(\n",
        "        model=chatGPT_settings['model'],\n",
        "        temperature = chatGPT_settings['temperature'],\n",
        "        messages=[\n",
        "            {\"role\": \"system\", \"content\": dialog},\n",
        "            {\"role\": \"user\", \"content\": question},\n",
        "        ]\n",
        "    )\n",
        "  except:\n",
        "      print(\"Something went wrong. Let's sleep (15 seconds) and try again\")\n",
        "      time.sleep(15)\n",
        "      response = chatGPT_annotate(dialog, question, level_recursion=level_recursion+1)\n",
        "  return response"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3CFIXJ3uqF4F"
      },
      "source": [
        "## Masking"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H-Q8i_aNqFax"
      },
      "outputs": [],
      "source": [
        "ambiguous_labels = ['Enhance', 'Extend', 'Elaborate', 'Fact', 'Opinion', 'Command']\n",
        "full_labels_list = ['React.Rejoinder.Support.Response.Resolve', 'React.Rejoinder.Confront.Challenge.Detach',\n",
        "                    'React.Rejoinder.Confront.Response.Refute', 'React.Rejoinder.Confront.Challenge.Counter', 'Open.Attend', 'Open.Command',\n",
        "                    'Open.Demand.Opinion', 'Open.Demand.Fact', 'Open.Give.Fact', 'Open.Give.Opinion', 'Sustain.Continue.Monitor',\n",
        "                    'React.Respond.Support.Register', 'React.Respond.Command', 'React.Respond.Support.Engage', 'React.Respond.Support.Reply.Accept',\n",
        "                    'Sustain.Continue.Command', 'React.Rejoinder.Support.Track.Check', 'React.Rejoinder.Support.Track.Confirm',\n",
        "                    'React.Rejoinder.Support.Track.Clarify', 'React.Rejoinder.Support.Track.Probe', 'Sustain.Continue.Prolong.Enhance',\n",
        "                    'Sustain.Continue.Prolong.Extend', 'Sustain.Continue.Prolong.Elaborate', 'React.Rejoinder.Confront.Challenge.Rebound',\n",
        "                    'React.Rejoinder.Confront.Response.Re-challenge', 'React.Respond.Support.Reply.Acknowledge',\n",
        "                    'React.Respond.Support.Reply.Affirm', 'React.Respond.Support.Reply.Agree', 'React.Respond.Confront.Reply.Disavow',\n",
        "                    'React.Respond.Confront.Reply.Disagree', 'React.Respond.Confront.Reply.Contradict', 'React.Respond.Support.Develop.Extend',\n",
        "                    'React.Respond.Support.Develop.Enhance', 'React.Respond.Support.Develop.Elaborate', 'Detailedanswer']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YpvuDBJvtaWf"
      },
      "outputs": [],
      "source": [
        "def make_label_to_mask_dict():\n",
        "    global ambiguous_labels, full_labels_list\n",
        "    dict_names = {}\n",
        "    for n, full_name in enumerate(full_labels_list):\n",
        "        short_label = '.'.join(full_name.split('.')[-2:]) if any(amb in full_name for amb in ambiguous_labels) else full_name.split('.')[-1]\n",
        "        dict_names[short_label] = {'mask': f'label_{n}', 'full_label': full_name}\n",
        "    return dict_names"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iwOjFJfDtpg7"
      },
      "outputs": [],
      "source": [
        "label_to_mask = make_label_to_mask_dict()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZVGRg1XzzI2d"
      },
      "outputs": [],
      "source": [
        "def process_text(text, function='mask'):\n",
        "    \"\"\"\n",
        "    function = mask, unmask, full_label\n",
        "    \"\"\"\n",
        "    global label_to_mask\n",
        "    text_split = text.replace('Detailed answer', 'Detailedanswer').replace('\\n', ' \\n').split(' ')\n",
        "    for index, word in enumerate(text_split):\n",
        "        if function=='unmask':\n",
        "            for value in label_to_mask.values():\n",
        "                if re.search(rf\"{value['mask']}([^a-zA-Z]|\\b)\", word):\n",
        "                    text_split[index] = re.sub(r'[.a-zA-Z_0-9]+', value['full_label'], word)\n",
        "                    text_split[index] = text_split[index].replace('Detailedanswer', 'Detailed answer')\n",
        "        else:\n",
        "            for key in label_to_mask.keys():\n",
        "                if re.search(rf\"{key}([^a-zA-Z]|\\b)\", word):\n",
        "                    if function=='mask':\n",
        "                        text_split[index] = re.sub(r'[.a-zA-Z]+', label_to_mask[key]['mask'], word)\n",
        "                    elif function=='get_full_label':\n",
        "                        text_split[index] = re.sub(r'[.a-zA-Z]+', label_to_mask[key]['full_label'], word)\n",
        "    return ' '.join(text_split)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SnkcVnkyBIUC"
      },
      "source": [
        "## Creating a prompt\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W_AMMZjwsEGC"
      },
      "outputs": [],
      "source": [
        "def create_prompt(dialog_json, current_node_question, possible_answers, current_utterance_num, instruction_id):\n",
        "  global chatGPT_settings\n",
        "  if current_utterance_num == 0:\n",
        "    curr_utt = dialog_json[int(current_utterance_num)]\n",
        "    curr_utt = f\"speaker_{curr_utt['speaker']}: {curr_utt['utterance']}\"\n",
        "    start = f\"\"\"DIALOG:\n",
        "(Start the dialog)\n",
        "CURRENT UTTERANCE: {curr_utt}\"\"\"\n",
        "  else:\n",
        "    len_context = chatGPT_settings['context_length']\n",
        "    prev_context = dialog_json[max(int(current_utterance_num)-chatGPT_settings['context_length'], 0):int(current_utterance_num)]\n",
        "    prev_context = [f\"speaker_{x['speaker']}: {x['utterance']}\" for x in prev_context]\n",
        "    prev_join = '\\n'.join(prev_context)\n",
        "    curr_utt = dialog_json[int(current_utterance_num)]\n",
        "    curr_utt = f\"speaker_{curr_utt['speaker']}: {curr_utt['utterance']}\"\n",
        "    start = f\"\"\"DIALOG\n",
        "PREVIOUS CONTEXT:\n",
        "{prev_join}\n",
        "CURRENT UTTERANCE:\n",
        "{curr_utt}\"\"\"\n",
        "  prompt_system = f\"\"\"TASK: This is part of the dialog is between 2 speakers. Answer QUESTION about CURRENT UTTERANCE. You must analyze relations between CURRENT UTTERANCE and PREVIOUS CONTEXT, not taking anything before it into account.\"\"\"\n",
        "  instruction = \"\"\n",
        "  instruction_after_q = \"\"\n",
        "  prev_utt = \"\"\n",
        "  if chatGPT_settings['instruction'] == True:\n",
        "    row = instructions[instructions['id'] == instruction_id]\n",
        "    if not row.empty:\n",
        "        instruction_text = row['instruction'].iloc[0]\n",
        "        if type(instruction_text) == str:\n",
        "            if len(instruction_text.split()) < 30:\n",
        "                instruction_after_q = instruction_text\n",
        "            else:\n",
        "                instruction = f\"\"\"INSTRUCTION: {instruction_text} \"\"\"\n",
        "        else:\n",
        "            instruction = ''\n",
        "        if 'Miscellaneous' in instruction:\n",
        "            current_node_question = current_node_question.replace('previous sentence', f'previous sentence ({prev_context[-1]})')\n",
        "            current_node_question = current_node_question.replace('This sentence', f'This sentence ({curr_utt})')\n",
        "  possible_answers = [x.strip() for x in possible_answers]\n",
        "  if chatGPT_settings['explanation']:\n",
        "    prompt_user = f\"\"\"{instruction}\\n{start}\\nQUESTION: {current_node_question} {instruction_after_q}\n",
        "POSSIBLE ANSWERS: {', '.join(possible_answers)}\\nProvide an explanation. If you are not sure, choose a most likely option. You must always select an option ({' or '.join(possible_answers)}). Structure your answer the following way:\n",
        "'Reasoning: (your reasoning). Final answer: (your final answer, {' or '.join(possible_answers)})\"\"\"\n",
        "  else:\n",
        "    prompt_user = f\"\"\"{instruction}\\n{start}\\nQUESTION: {current_node_question} {instruction_after_q}\n",
        "POSSIBLE ANSWERS: {', '.join(possible_answers)}\\nYou must always select an option. Provide only one answer without explanation. ANSWER ({' or '.join(possible_answers)}):\"\"\"\n",
        "  if chatGPT_settings['masking']:\n",
        "    prompt_system = process_text(prompt_system, function='mask')\n",
        "    prompt_user = process_text(prompt_user, function='mask')\n",
        "  return prompt_system, prompt_user"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99oOstkBnkyK"
      },
      "source": [
        "## Working with the tree"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A7R5XI4DyQyH"
      },
      "outputs": [],
      "source": [
        "def extract_node_features(current_node_path, current_node_path_object, current_node_path_object_child):\n",
        "  print(current_node_path_object_child)\n",
        "  if root.find(current_node_path):\n",
        "    current_node_question = root.find(current_node_path).attrib['value']\n",
        "    node_text = root.find(current_node_path).attrib['value']\n",
        "    style_text = root.find(current_node_path).attrib['style']\n",
        "  else:\n",
        "    current_node_question = root.find(current_node_path_object).attrib['label']\n",
        "    node_text = root.find(current_node_path_object).attrib['label']\n",
        "    style_text = root.find(current_node_path_object_child).attrib['style']\n",
        "  return current_node_question, node_text, style_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ei13ZYyIu_Y8"
      },
      "outputs": [],
      "source": [
        "def get_features_for_extra(current_node):\n",
        "    print(current_node)\n",
        "    extra_id = root_extra.find(\"diagram/mxGraphModel/root/object[@linked_parent_id='\"+str(current_node)+\"']\").attrib['id']\n",
        "    current_node_question = \"Select the most appropriate description for the current utterance\"\n",
        "    extra_answers = root_extra.findall(\"diagram/mxGraphModel/root/mxCell[@source='\"+str(extra_id)+\"']\")\n",
        "    if extra_answers:\n",
        "        possible_answers = []\n",
        "        target_nodes = [answer.attrib['target'] for answer in extra_answers]\n",
        "    for answer in extra_answers:\n",
        "        possible_answer = root_extra.find(f\"diagram/mxGraphModel/root/mxCell[@parent='{answer.attrib['id']}']\")\n",
        "        possible_answers.append(possible_answer.attrib['value'] if possible_answer else answer.attrib['value'])\n",
        "    possible_answers = [cleanhtml(x) for x in possible_answers]\n",
        "    possible_answers = [x.strip('\\n') if '\\n' in x else x for x in possible_answers]\n",
        "    return possible_answers, target_nodes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DK9-w1JcvSbR"
      },
      "outputs": [],
      "source": [
        "def get_features_for_others(current_node):\n",
        "    answers_lines = root.findall(\"diagram/mxGraphModel/root/mxCell[@source='\"+str(current_node)+\"']\")\n",
        "    answers_lines_objects = root.findall(\".//mxCell[@source='\"+str(current_node)+\"']...\")\n",
        "    answers_lines_objects_child = root.findall(\"diagram/mxGraphModel/root/object/mxCell[@source='\"+str(current_node)+\"']\")\n",
        "\n",
        "    possible_answers = []\n",
        "    target_nodes = []\n",
        "\n",
        "    if answers_lines or answers_lines_objects_child:\n",
        "      for answer in answers_lines:\n",
        "        answer_value = root.find(f\"diagram/mxGraphModel/root/mxCell[@parent='{answer.attrib['id']}']\")\n",
        "        possible_answers.append(answer_value.attrib['value'] if answer_value else answer.attrib['value'])\n",
        "        target_nodes.append(answer.attrib['target'])\n",
        "    i = 0\n",
        "    for answer_line in answers_lines_objects_child:\n",
        "        target_nodes.append(answer_line.attrib['target'])\n",
        "        possible_answers.append(root.find(\".//mxCell[@parent='\"+str(answers_lines_objects[i].attrib['id'])+\"']/*[@label]/*[last()]/*[@value]\"))\n",
        "        i += 1\n",
        "    possible_answers = [cleanhtml(x) for x in possible_answers]\n",
        "    possible_answers = [x.strip('\\n') if '\\n' in x else x for x in possible_answers]\n",
        "    return possible_answers, target_nodes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wQQPftabwWSK"
      },
      "outputs": [],
      "source": [
        "def chain_answer(dialog_json, current_node, current_utterance_num):\n",
        "  global final_answers, logs_real_time, prev_utterance_type\n",
        "  if logs_real_time:\n",
        "      stdout_meaning = sys.stdout\n",
        "  else:\n",
        "      stdout_meaning = open(os.devnull, 'w')\n",
        "  LAST_NODE_COLOR = '#dae8fc'\n",
        "  EXTRA_NODE_COLOR = '#e1d5e7'\n",
        "  current_node_path = \"diagram/mxGraphModel/root/mxCell[@id='\"+str(current_node)+\"']\"\n",
        "  current_node_path_object = \"diagram/mxGraphModel/root/object[@id='\"+str(current_node)+\"']\"\n",
        "  current_node_path_object_child = \"diagram/mxGraphModel/root/object[@id='\"+str(current_node)+\"']/mxCell\"\n",
        "  current_node_question, node_text, style_text = extract_node_features(current_node_path, current_node_path_object, current_node_path_object_child)\n",
        "  instruction_id = '0_'+str(current_node)\n",
        "  prev_utt = \"\"\n",
        "  if current_utterance_num-2 >= 0:\n",
        "    prev_utt =  f\"speaker_{dialog_json[current_utterance_num-2]['speaker']}: {dialog_json[current_utterance_num-2]['utterance']} \"\n",
        "  prev_utt +=  f\"speaker_{dialog_json[current_utterance_num-1]['speaker']}: {dialog_json[current_utterance_num-1]['utterance']}\"\n",
        "  curr_utt = f\"speaker_{dialog_json[current_utterance_num]['speaker']}: {dialog_json[current_utterance_num]['utterance']}\"\n",
        "  if root.find(current_node_path) or root.find(current_node_path_object):\n",
        "    print(style_text)\n",
        "    if (f'fillColor={LAST_NODE_COLOR}' in style_text):\n",
        "       final_answers.append(node_text)\n",
        "       print(\"This node is the last one in the scheme\", file=stdout_meaning)\n",
        "       print(final_answers, file=stdout_meaning)\n",
        "       fill_annotation_log(str(final_answers))\n",
        "       print(f'answer_text for {curr_utt}: {node_text}')\n",
        "       return node_text, 0\n",
        "    if (f'fillColor={EXTRA_NODE_COLOR}' in style_text):\n",
        "       possible_answers, target_nodes = get_features_for_extra(current_node)\n",
        "       if chatGPT_settings['masking']:\n",
        "         possible_answers = [process_text(answer, function = 'mask') for answer in possible_answers]\n",
        "       prompt_system, prompt_user = create_prompt(dialog_json, current_node_question, possible_answers, current_utterance_num, instruction_id)\n",
        "       print(prompt_system, file=stdout_meaning)\n",
        "       print(prompt_user, file=stdout_meaning)\n",
        "       fill_annotation_log(prompt_system)\n",
        "       fill_annotation_log(prompt_user)\n",
        "       answer_text = chatGPT_annotate(prompt_system, prompt_user).choices[0]['message']['content']\n",
        "       print(answer_text, file=stdout_meaning)\n",
        "       fill_annotation_log(answer_text)\n",
        "       if chatGPT_settings['explanation']:\n",
        "         answer_text = answer_text.split('Final answer: ')[-1]\n",
        "       next_node, right_answer_id = select_answer(answer_text, possible_answers, target_nodes)\n",
        "       node_text = process_text(possible_answers[right_answer_id], function = 'unmask')\n",
        "       final_answers.append(node_text)\n",
        "       print('This node was the last one in the extra file', file=stdout_meaning)\n",
        "       print(node_text, file=stdout_meaning)\n",
        "       fill_annotation_log(node_text)\n",
        "       print(f'answer_text for {curr_utt}: {node_text}')\n",
        "       return node_text, 0\n",
        "    else:\n",
        "      if int(current_node) == 2 and len(final_answers)!=0 and len(final_answers) > 0:\n",
        "        if final_answers[current_utterance_num-1] == 'Open.Attend':\n",
        "          if dialog_json[current_utterance_num]['speaker'] == dialog_json[current_utterance_num-1]['speaker']:\n",
        "            print('The next node is also OPEN', file=stdout_meaning)\n",
        "            return \"Yes\", 3\n",
        "      if int(current_node) == 2 and current_utterance_num == 0:\n",
        "        print('OPEN node', file=stdout_meaning)\n",
        "        return \"Yes\", 3\n",
        "      elif int(current_node) == 2:\n",
        "        print(f\"topic shift model working: {prev_utt} {curr_utt}\", file=stdout_meaning)\n",
        "        res = model(prev_utt, curr_utt)\n",
        "        if res[0] == '1':\n",
        "            return \"Yes\", 3\n",
        "        else:\n",
        "            return \"No\", 36\n",
        "      if int(current_node) == 36 and current_utterance_num != 0:\n",
        "        if dialog_json[current_utterance_num]['speaker'] != dialog_json[current_utterance_num-1]['speaker']:\n",
        "          print('Speaker changes', file=stdout_meaning)\n",
        "          return \"Yes\", 39\n",
        "        else:\n",
        "          print('Speaker is the same', file=stdout_meaning)\n",
        "          return \"No\", 173\n",
        "      if int(current_node) in [39, 173] and final_answers:\n",
        "        if 'Open' not in final_answers[-1]:\n",
        "          answer_text = prev_utterance_type\n",
        "          possible_answers, target_nodes = get_features_for_others(current_node)\n",
        "          next_node, right_answer_id = select_answer(answer_text, possible_answers, target_nodes)\n",
        "          return answer_text, next_node\n",
        "      if int(current_node) in [184, 271] and dialog_json[current_utterance_num-1]['speaker'] == dialog_json[current_utterance_num]['speaker']:\n",
        "          return \"No\", 253\n",
        "      possible_answers, target_nodes = get_features_for_others(current_node)\n",
        "      prompt_system, prompt_user = create_prompt(dialog_json, current_node_question, possible_answers, current_utterance_num, instruction_id)\n",
        "      print(prompt_system, file=stdout_meaning)\n",
        "      print(prompt_user, file=stdout_meaning)\n",
        "      fill_annotation_log(prompt_user)\n",
        "      answer_text = chatGPT_annotate(prompt_system, prompt_user).choices[0]['message']['content']\n",
        "      print(f'answer_text: {answer_text}', file=stdout_meaning)\n",
        "      fill_annotation_log(f'answer_text: {answer_text}')\n",
        "      if chatGPT_settings['explanation']:\n",
        "          answer_text = answer_text.split('Final answer: ')[-1]\n",
        "      if chatGPT_settings['masking']:\n",
        "         possible_answers = [process_text(answer, function = 'mask') for answer in possible_answers]\n",
        "      next_node, right_answer_id = select_answer(answer_text, possible_answers, target_nodes)\n",
        "      node_text = possible_answers[right_answer_id]\n",
        "      if int(current_node) == 112 and 'Other' in node_text:\n",
        "          return \"Other\", 271\n",
        "      if int(current_node) in [41, 176, 264, 145, 112]:\n",
        "          prev_utterance_type = node_text\n",
        "          print(f'just wrote prev_utterance_type: {prev_utterance_type}', file=stdout_meaning)\n",
        "      return node_text, next_node"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S0UO3Id2NPHh"
      },
      "outputs": [],
      "source": [
        "def chain(dialog_json, cur_utterance):\n",
        "  global logs_real_time\n",
        "  if logs_real_time:\n",
        "      stdout_meaning = sys.stdout\n",
        "  else:\n",
        "      stdout_meaning = open(os.devnull, 'w')\n",
        "  cur_node = 2\n",
        "  while cur_node != 0:\n",
        "    answer, cur_node = chain_answer(dialog_json, cur_node, cur_utterance)\n",
        "    print(f'{answer} {cur_node}', file=stdout_meaning)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xDsJIbFGdq2V"
      },
      "outputs": [],
      "source": [
        "#это - главная функция, которая запускает всю работу и разметку датасета\n",
        "#я ее не вызываю, потому что ChatGPT периодически обрывает API. В автоматическом режиме слетает разметка.\n",
        "def annotate_dialog(dialog_id, dialog_json):\n",
        "  global final_answers, annotation_log\n",
        "  final_answers = [] #хранятся ответы разметки для одного диалога\n",
        "  rows_list = []\n",
        "  for current_utterance_id in tqdm(range(len(dialog_json))):\n",
        "    annotation_log = \"\"\n",
        "    chain(dialog_json, current_utterance_id)\n",
        "    new_row = {'dialog_id':dialog_id, 'utterance_id':current_utterance_id, 'speaker':dialog_json[current_utterance_id]['speaker'], 'utterance':dialog_json[current_utterance_id]['utterance'], 'annotation': final_answers[-1], \"logs\": annotation_log}\n",
        "    rows_list.append(new_row)\n",
        "  results = pd.DataFrame(rows_list)\n",
        "  return results\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P8FXwz99j5BT"
      },
      "outputs": [],
      "source": [
        "def annotate_all_dialogs(list_dialogs): # 'reannot_toloka (1).tsv'\n",
        "    # list_dialogs = make_dialog_dict(filename)\n",
        "    df_dummy = pd.DataFrame({'dialog_id': pd.Series(dtype='int'),\n",
        "                            'utterance_id': pd.Series(dtype='int'),\n",
        "                            'speaker': pd.Series(dtype='str'),\n",
        "                            'utterance': pd.Series(dtype='str'),\n",
        "                            'annotation': pd.Series(dtype='str'),\n",
        "                            'logs': pd.Series(dtype='str')})\n",
        "    for id, dialog in tqdm(list_dialogs.items()):\n",
        "        df_to_merge = annotate_dialog(id, dialog)\n",
        "        df_dummy = pd.concat([df_dummy, df_to_merge])\n",
        "    return df_dummy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WQaUKUj0j6oP"
      },
      "outputs": [],
      "source": [
        "def annotate_file(filename): # 'reannot_toloka (1).tsv'\n",
        "    list_dialogs = make_dialog_dict(filename, filetype='gold', sep=',')\n",
        "    df_result = annotate_all_dialogs(list_dialogs)\n",
        "    return df_result"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bj8thsuZjQnz"
      },
      "source": [
        "# Functions for calculating metrcis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F2Qi7vJTJMcr"
      },
      "outputs": [],
      "source": [
        "def cut_labels(golden_cut):\n",
        "    short_labels = []\n",
        "    for i in range(len(golden_cut)):\n",
        "        if 'Open' in golden_cut[i]:\n",
        "            if 'Initiate' in golden_cut[i]:\n",
        "                short_labels.append(re.sub('Initiate.','', golden_cut[i]))\n",
        "            if len(golden_cut[i].split('.')) == 3:\n",
        "                short_labels.append('.'.join(golden_cut[i].split('.')[:-1]))\n",
        "            else:\n",
        "                short_labels.append(golden_cut[i])\n",
        "        elif \"Prolong\" in golden_cut[i] or \"Develop\" in golden_cut[i]:\n",
        "            short_labels.append('.'.join(golden_cut[i].split('.')[:-1]))\n",
        "        elif \"Track\" in golden_cut[i]:\n",
        "            short_labels.append('.'.join(golden_cut[i].split('.')[:-1]))\n",
        "        elif \"Reply\" in golden_cut[i]:\n",
        "            if \"Accept\" not in golden_cut[i]:\n",
        "                short_labels.append('.'.join(golden_cut[i].split('.')[:-1]))\n",
        "            else:\n",
        "                short_labels.append(golden_cut[i])\n",
        "        elif \"Challenge\" in golden_cut[i]:\n",
        "            short_labels.append('.'.join(golden_cut[i].split('.')[:-1]))\n",
        "        elif \"Confront.Response\" in golden_cut[i]:\n",
        "            short_labels.append('.'.join(golden_cut[i].split('.')[:-1]))\n",
        "        else:\n",
        "            short_labels.append(golden_cut[i])\n",
        "\n",
        "    return short_labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N-AAJR_cH9WH"
      },
      "outputs": [],
      "source": [
        "def cut_labels_list(list_of_lists):\n",
        "    for i, list_labels in enumerate(list_of_lists):\n",
        "        list_of_lists[i] = cut_labels(list_of_lists[i])\n",
        "    return list_of_lists"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_krGn1-9E96M"
      },
      "outputs": [],
      "source": [
        "def make_annotation_list(annotation_results, gold_dialogs, type_ann='toloka', type_label='full'):\n",
        "    \"\"\"\n",
        "    type_ann = toloka, good, excellent\n",
        "    \"\"\"\n",
        "    annotation = {}\n",
        "    for dialog_id in annotation_results['dialog_id'].unique():\n",
        "        annotation[dialog_id] = []\n",
        "        dialog_df = annotation_results[annotation_results['dialog_id'] == dialog_id]\n",
        "        dialog_df = dialog_df.reset_index()\n",
        "        if type_label=='cut':\n",
        "            dialog_df['annotation'] = cut_labels(dialog_df['annotation'])\n",
        "        if type_ann=='toloka':\n",
        "            for _, row in dialog_df.iterrows():\n",
        "                annotation[dialog_id].append(row['annotation'])\n",
        "        else:\n",
        "            gold_annot = gold_dialogs[str(dialog_id)][type_ann]\n",
        "            if type_label=='cut':\n",
        "                gold_annot = cut_labels_list(gold_annot)\n",
        "            for _, row in dialog_df.iterrows():\n",
        "                if row['annotation'] in gold_annot[row['utterance_id']]:\n",
        "                    annotation[dialog_id].append(row['annotation'])\n",
        "                else:\n",
        "                    annotation[dialog_id].append(gold_annot[row['utterance_id']][0])\n",
        "    return annotation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qQmbebyvWt-R"
      },
      "outputs": [],
      "source": [
        "def return_metrics(annotation_results, gold_dialogs_filepath, type_gold='good', type_label='full', how='by_one_dialog'): #\"gold_dialogs_train (1).json\"\n",
        "    \"\"\"\n",
        "    type_gold = good, excellent\n",
        "    type_label = full, cut\n",
        "    how = by_one_dialog, by_all\n",
        "    \"\"\"\n",
        "    with open(gold_dialogs_filepath, \"r\") as f:\n",
        "        gold_dialogs = json.load(f)\n",
        "    toloka_annotation = make_annotation_list(annotation_results, gold_dialogs, type_ann='toloka', type_label=type_label)\n",
        "    gold_annotation = make_annotation_list(annotation_results, gold_dialogs, type_ann=type_gold, type_label=type_label)\n",
        "    if how=='by_one_dialog':\n",
        "        for key in toloka_annotation.keys():\n",
        "            print(f\"Accuracy for dialog {key} ({type_gold}, {type_label} labels, no voting):\")\n",
        "            print(round(accuracy_score(gold_annotation[key], toloka_annotation[key]), 2))\n",
        "\n",
        "            print(f\"macro f1 for dialog {key} ({type_gold}, {type_label} labels, no voting)\")\n",
        "            print(round(f1_score(gold_annotation[key], toloka_annotation[key], average=\"macro\"), 2))\n",
        "\n",
        "            print(f\"micro for dialog {key} ({type_gold}, {type_label} labels, no voting):\")\n",
        "            print(round(f1_score(gold_annotation[key], toloka_annotation[key], average=\"micro\"), 2))\n",
        "\n",
        "            print(f\"weighted recall for dialog {key} ({type_gold}, {type_label} labels, no voting):\")\n",
        "            print(round(recall_score(gold_annotation[key], toloka_annotation[key], average=\"weighted\"), 2))\n",
        "\n",
        "            print(f\"weighted precision for dialog {key} ({type_gold}, {type_label} labels, no voting):\")\n",
        "            print(round(precision_score(gold_annotation[key], toloka_annotation[key], average=\"weighted\"), 2))\n",
        "    elif how=='by_all':\n",
        "        toloka_annotation =  sum(toloka_annotation.values(), [])\n",
        "        gold_annotation =  sum(gold_annotation.values(), [])\n",
        "        print(f\"Accuracy for all dialogs ({type_gold}, {type_label} labels, no voting):\")\n",
        "        print(round(accuracy_score(gold_annotation, toloka_annotation), 2))\n",
        "\n",
        "        print(f\"macro f1 for all dialogs ({type_gold}, {type_label} labels, no voting)\")\n",
        "        print(round(f1_score(gold_annotation, toloka_annotation, average=\"macro\"), 2))\n",
        "\n",
        "        print(f\"micro f1 for all dialogs ({type_gold}, {type_label} labels, no voting):\")\n",
        "        print(round(f1_score(gold_annotation, toloka_annotation, average=\"micro\"), 2))\n",
        "\n",
        "        print(f\"weighted recall for all dialogs ({type_gold}, {type_label} labels, no voting):\")\n",
        "        print(round(recall_score(gold_annotation, toloka_annotation, average=\"weighted\"), 2))\n",
        "\n",
        "        print(f\"weighted precision for all dialogs ({type_gold}, {type_label} labels, no voting):\")\n",
        "        print(round(precision_score(gold_annotation, toloka_annotation, average=\"weighted\"), 2))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QW4e-wYpnILj"
      },
      "source": [
        "# TO EXPERIMENT WITH EVERYTHING BY YOURSELF, GO HERE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U2J60X5R1EwM"
      },
      "outputs": [],
      "source": [
        "openai.api_key = 'YOUR_KEY' # you have to provide your own key here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_ZnWfqOknmBF"
      },
      "outputs": [],
      "source": [
        "count = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ddsnPphX5BHK"
      },
      "outputs": [],
      "source": [
        "chatGPT_settings = {\n",
        "    'model': \"gpt-3.5-turbo\",\n",
        "    'temperature': 0.9,\n",
        "    'context_length': 1,\n",
        "    'instruction': True,\n",
        "    'masking': False,\n",
        "    'explanation': False\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lQT_iqMs5BHM"
      },
      "outputs": [],
      "source": [
        "count += 1\n",
        "logs_real_time = False #True if you want to see lots of logs real-time\n",
        "final_answers = []\n",
        "prev_utterance_type = ''\n",
        "# keys = ['1'] #ids of dialogs that you want to annotate\n",
        "dialogs = make_dialog_dict('gold_standard.csv', sep=',', filetype='gold')\n",
        "dialogs_shorter = dict(list(dialogs.items())[:12]) #12 dialogs only\n",
        "# dialogs_shorter = {key: dialogs[key] for key in keys} #uncomment if you want to use ids\n",
        "annotation_results = annotate_all_dialogs(dialogs_shorter)\n",
        "annotation_results['annotation'] = annotation_results['annotation'].apply(process_text, function='get_full_label')\n",
        "annotation_results['annotation'] = annotation_results['annotation'].apply(lambda x: x.strip())\n",
        "annotation_results.to_csv(f'/content/drive/MyDrive/annotation_SIGDIAL/sf_annotation_{count}.tsv', sep='\\t')\n",
        "return_metrics(annotation_results, \"gold_standard.json\", type_gold='good', type_label='full', how='by_all')\n",
        "return_metrics(annotation_results, \"gold_standard.json\", type_gold='good', type_label='cut', how='by_all')\n",
        "return_metrics(annotation_results, \"gold_standard.json\", type_gold='excellent', type_label='full', how='by_all')\n",
        "return_metrics(annotation_results, \"gold_standard.json\", type_gold='excellent', type_label='cut', how='by_all')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}