2. seq_versions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This script:\n",
    "\n",
    "1. Creates sequences of stimuli excluding unwanted repetitions of letters and locations\n",
    "2. Statistically assesses if the sequence is good enough and deletes a sequence, if not\n",
    "3. Creates versions of each sequence with different distributions of tasks (?)\n",
    "4. Checks wether the tasks and answers are well distributed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import ast\n",
    "import collections\n",
    "import statistics\n",
    "import re\n",
    "from collections import Counter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_path_to_save = '...\\\\sequences\\\\' #! SET YOUR PATH HERE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**CREATING SEQUENCES**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid</th>\n",
       "      <th>letters</th>\n",
       "      <th>locs</th>\n",
       "      <th>encoding_stim</th>\n",
       "      <th>incorr_locs</th>\n",
       "      <th>corr_letters</th>\n",
       "      <th>incorr_letters</th>\n",
       "      <th>rotated_inds</th>\n",
       "      <th>incorr_rot</th>\n",
       "      <th>abc_ordered</th>\n",
       "      <th>incorr_abc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[array([['Л', 'Ж', 'П'],\\n       ['Т', 'Х', 'Х...</td>\n",
       "      <td>['Л', 'Ж', 'П', 'Т']</td>\n",
       "      <td>[0, 1, 2, 3]</td>\n",
       "      <td>grid_1331.png</td>\n",
       "      <td>[5, 6, 7, 8]</td>\n",
       "      <td>{'Л': 0, 'Ж': 1, 'П': 2, 'Т': 3}</td>\n",
       "      <td>{'Л': [1, 2, 3], 'Ж': [0, 2, 3], 'П': [0, 1, 3...</td>\n",
       "      <td>[2, 5, 8, 1]</td>\n",
       "      <td>[0, 3, 6, 7]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}</td>\n",
       "      <td>{'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[array([['Ж', 'Л', 'Т'],\\n       ['Х', 'Х', 'П...</td>\n",
       "      <td>['Ж', 'Л', 'Т', 'П']</td>\n",
       "      <td>[0, 1, 2, 5]</td>\n",
       "      <td>grid_1332.png</td>\n",
       "      <td>[3, 6, 7, 8]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'Т': 2, 'П': 3}</td>\n",
       "      <td>{'Ж': [1, 2, 3], 'Л': [0, 2, 3], 'Т': [0, 1, 3...</td>\n",
       "      <td>[2, 5, 8, 7]</td>\n",
       "      <td>[0, 1, 3, 6]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}</td>\n",
       "      <td>{'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[array([['П', 'Л', 'Ж'],\\n       ['Х', 'Х', 'Х...</td>\n",
       "      <td>['П', 'Л', 'Ж', 'Т']</td>\n",
       "      <td>[0, 1, 2, 6]</td>\n",
       "      <td>grid_1333.png</td>\n",
       "      <td>[3, 5, 7, 8]</td>\n",
       "      <td>{'П': 0, 'Л': 1, 'Ж': 2, 'Т': 3}</td>\n",
       "      <td>{'П': [1, 2, 3], 'Л': [0, 2, 3], 'Ж': [0, 1, 3...</td>\n",
       "      <td>[2, 5, 8, 0]</td>\n",
       "      <td>[1, 3, 6, 7]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}</td>\n",
       "      <td>{'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[array([['Ж', 'Л', 'Т'],\\n       ['Х', 'Х', 'Х...</td>\n",
       "      <td>['Ж', 'Л', 'Т', 'П']</td>\n",
       "      <td>[0, 1, 2, 7]</td>\n",
       "      <td>grid_1334.png</td>\n",
       "      <td>[3, 5, 6, 8]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'Т': 2, 'П': 3}</td>\n",
       "      <td>{'Ж': [1, 2, 3], 'Л': [0, 2, 3], 'Т': [0, 1, 3...</td>\n",
       "      <td>[2, 5, 8, 3]</td>\n",
       "      <td>[0, 1, 6, 7]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}</td>\n",
       "      <td>{'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[array([['Т', 'П', 'Ж'],\\n       ['Х', 'Х', 'Х...</td>\n",
       "      <td>['Т', 'П', 'Ж', 'Л']</td>\n",
       "      <td>[0, 1, 2, 8]</td>\n",
       "      <td>grid_1335.png</td>\n",
       "      <td>[3, 5, 6, 7]</td>\n",
       "      <td>{'Т': 0, 'П': 1, 'Ж': 2, 'Л': 3}</td>\n",
       "      <td>{'Т': [1, 2, 3], 'П': [0, 2, 3], 'Ж': [0, 1, 3...</td>\n",
       "      <td>[2, 5, 8, 6]</td>\n",
       "      <td>[0, 1, 3, 7]</td>\n",
       "      <td>{'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}</td>\n",
       "      <td>{'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                grid               letters  \\\n",
       "0  [array([['Л', 'Ж', 'П'],\\n       ['Т', 'Х', 'Х...  ['Л', 'Ж', 'П', 'Т']   \n",
       "1  [array([['Ж', 'Л', 'Т'],\\n       ['Х', 'Х', 'П...  ['Ж', 'Л', 'Т', 'П']   \n",
       "2  [array([['П', 'Л', 'Ж'],\\n       ['Х', 'Х', 'Х...  ['П', 'Л', 'Ж', 'Т']   \n",
       "3  [array([['Ж', 'Л', 'Т'],\\n       ['Х', 'Х', 'Х...  ['Ж', 'Л', 'Т', 'П']   \n",
       "4  [array([['Т', 'П', 'Ж'],\\n       ['Х', 'Х', 'Х...  ['Т', 'П', 'Ж', 'Л']   \n",
       "\n",
       "           locs  encoding_stim   incorr_locs  \\\n",
       "0  [0, 1, 2, 3]  grid_1331.png  [5, 6, 7, 8]   \n",
       "1  [0, 1, 2, 5]  grid_1332.png  [3, 6, 7, 8]   \n",
       "2  [0, 1, 2, 6]  grid_1333.png  [3, 5, 7, 8]   \n",
       "3  [0, 1, 2, 7]  grid_1334.png  [3, 5, 6, 8]   \n",
       "4  [0, 1, 2, 8]  grid_1335.png  [3, 5, 6, 7]   \n",
       "\n",
       "                       corr_letters  \\\n",
       "0  {'Л': 0, 'Ж': 1, 'П': 2, 'Т': 3}   \n",
       "1  {'Ж': 0, 'Л': 1, 'Т': 2, 'П': 3}   \n",
       "2  {'П': 0, 'Л': 1, 'Ж': 2, 'Т': 3}   \n",
       "3  {'Ж': 0, 'Л': 1, 'Т': 2, 'П': 3}   \n",
       "4  {'Т': 0, 'П': 1, 'Ж': 2, 'Л': 3}   \n",
       "\n",
       "                                      incorr_letters  rotated_inds  \\\n",
       "0  {'Л': [1, 2, 3], 'Ж': [0, 2, 3], 'П': [0, 1, 3...  [2, 5, 8, 1]   \n",
       "1  {'Ж': [1, 2, 3], 'Л': [0, 2, 3], 'Т': [0, 1, 3...  [2, 5, 8, 7]   \n",
       "2  {'П': [1, 2, 3], 'Л': [0, 2, 3], 'Ж': [0, 1, 3...  [2, 5, 8, 0]   \n",
       "3  {'Ж': [1, 2, 3], 'Л': [0, 2, 3], 'Т': [0, 1, 3...  [2, 5, 8, 3]   \n",
       "4  {'Т': [1, 2, 3], 'П': [0, 2, 3], 'Ж': [0, 1, 3...  [2, 5, 8, 6]   \n",
       "\n",
       "     incorr_rot                       abc_ordered  \\\n",
       "0  [0, 3, 6, 7]  {'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}   \n",
       "1  [0, 1, 3, 6]  {'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}   \n",
       "2  [1, 3, 6, 7]  {'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}   \n",
       "3  [0, 1, 6, 7]  {'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}   \n",
       "4  [0, 1, 3, 7]  {'Ж': 0, 'Л': 1, 'П': 2, 'Т': 3}   \n",
       "\n",
       "                                          incorr_abc  \n",
       "0  {'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...  \n",
       "1  {'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...  \n",
       "2  {'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...  \n",
       "3  {'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...  \n",
       "4  {'Ж': [1, 2], 'Л': [0, 2], 'П': [1, 3], 'Т': [...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Path to the CSV file containing ALL_GRIDS\n",
    "csv_path = '...\\\\all_grids.csv' #! SET YOUR PATH HERE\n",
    "df = pd.read_csv(csv_path, delimiter=';', index_col=0) \n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rules for seqs:\n",
    "# 1. Length - 300 rows (trials)\n",
    "# 2. Less than 3 coinciding locations in two consequtive trials\n",
    "# 3. No coinciding letters in two consequtive trials\n",
    "\n",
    "info = []\n",
    "\n",
    "# Loop for generating sequences\n",
    "for i in range(0, 15): # Set desired number of sequences. NOTE: set higher number than you need as many seqs will be excluded due to the criteria below\n",
    "    # Generate sequence name\n",
    "    sequence_name = f'sequence_{i+1}'\n",
    "    \n",
    "    # Total number of rows in the DataFrame\n",
    "    total_rows = df.shape[0]\n",
    "    \n",
    "    # Randomly shuffle DataFrame indices\n",
    "    random_indices = random.sample(range(total_rows), total_rows)\n",
    "    \n",
    "    # Retrieve random lines from DataFrame\n",
    "    random_lines = df.iloc[random_indices]\n",
    "    \n",
    "    # Initialize sequence with the first trial\n",
    "    sequence = random_lines.iloc[[0]]\n",
    "    \n",
    "    # Initialize DataFrame to store bad trials that don't meet criteria\n",
    "    bad_lines = pd.DataFrame(columns=sequence.columns)\n",
    "\n",
    "    # Initialize previous trial with the first trial\n",
    "    previous_trial = random_lines.iloc[[0]]\n",
    "\n",
    "    # Loop through each trial in the shuffled DataFrame\n",
    "    for i in range(total_rows - 1):\n",
    "\n",
    "        # Get the current trial\n",
    "        current_trial = random_lines.iloc[[i+1]]\n",
    "\n",
    "        # Extract location information from current and previous trials\n",
    "        curr_loc = ast.literal_eval(current_trial['locs'].iloc[0])\n",
    "        prev_loc = ast.literal_eval(previous_trial['locs'].iloc[0])\n",
    "        \n",
    "        # Calculate unwanted coincidences in locations\n",
    "        loc_coin = len(list(set(curr_loc) & set(prev_loc)))\n",
    "\n",
    "        # Extract letter information from current and previous trials\n",
    "        curr_let = ast.literal_eval(current_trial['letters'].iloc[0])\n",
    "        prev_let = ast.literal_eval(previous_trial['letters'].iloc[0])\n",
    "        \n",
    "        # Calculate unwanted coincidences in letters\n",
    "        lett_coin = len(list(set(curr_let) & set(prev_let)))\n",
    "\n",
    "        # Check if the current trial meets the criteria - less than 3 coinciding locations & no coinciding letters\n",
    "        if loc_coin < 3 and lett_coin == False:\n",
    "            # Append current trial to the sequence and update previous trial\n",
    "            sequence = sequence.append(current_trial)\n",
    "            previous_trial = current_trial\n",
    "        else:\n",
    "            # Append current trial to bad lines DataFrame\n",
    "            bad_lines = bad_lines.append(current_trial)\n",
    "\n",
    "        # Check if the sequence length is less than or equal to 300\n",
    "        if (i == (total_rows - 2)) and (len(sequence) <= 300):\n",
    "            # Append length of sequence to info list\n",
    "            info.append(len(sequence))\n",
    "            \n",
    "            # Update total_rows with the number of bad lines\n",
    "            total_rows = bad_lines.shape[0]\n",
    "            \n",
    "            # Randomly shuffle indices of bad lines DataFrame\n",
    "            random_indices = random.sample(range(total_rows), total_rows)\n",
    "            \n",
    "            # Retrieve random lines from bad lines DataFrame\n",
    "            random_lines = bad_lines.iloc[random_indices]\n",
    "            \n",
    "            # Set previous trial as the last trial of the current sequence\n",
    "            previous_trial = sequence.iloc[[-1]]\n",
    "\n",
    "            # Loop through each trial in the shuffled bad lines DataFrame\n",
    "            for i in range(total_rows - 1):\n",
    "                # Get the current trial\n",
    "                current_trial = random_lines.iloc[[i+1]]\n",
    "\n",
    "                # Extract location information from current and previous trials\n",
    "                curr_loc = ast.literal_eval(current_trial['locs'].iloc[0])\n",
    "                prev_loc = ast.literal_eval(previous_trial['locs'].iloc[0])\n",
    "                \n",
    "                # Calculate unwanted coincidences in locations\n",
    "                loc_coin = len(list(set(curr_loc) & set(prev_loc)))\n",
    "\n",
    "                # Extract letter information from current and previous trials\n",
    "                curr_let = ast.literal_eval(current_trial['letters'].iloc[0])\n",
    "                prev_let = ast.literal_eval(previous_trial['letters'].iloc[0])\n",
    "                \n",
    "                # Calculate unwanted coincidences in letters\n",
    "                lett_coin = len(list(set(curr_let) & set(prev_let)))\n",
    "\n",
    "                # Check if the current trial meets the criteria\n",
    "                if loc_coin < 3 and lett_coin == False:\n",
    "                    # Append current trial to the sequence and update previous trial\n",
    "                    sequence = sequence.append(current_trial)\n",
    "                    previous_trial = current_trial\n",
    "\n",
    "                # Check if the sequence length is 300 and there are no duplicates\n",
    "                if (len(sequence) == 300) and (sequence.duplicated().any() == False):\n",
    "                    # Save the first 300 trials of the sequence to a CSV file\n",
    "                    sequence[:300].to_csv(f'{seq_path_to_save}{sequence_name}.csv', sep=';', index=True, encoding='utf-8-sig')\n",
    "                    break\n",
    "\n",
    "# Print the original lengths of the sequences\n",
    "print(f'Original lengths of sequences were: {info}')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SEQUENCE STATISTICS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def good_sequence(letter_counter: collections.Counter, locs_counter: collections.Counter) -> bool:\n",
    "    # Initialize a flag to indicate whether the sequence is good\n",
    "    good_seq = True\n",
    "\n",
    "    # Calculate mean and standard deviation of letter counts and location counts\n",
    "    letters_mean = statistics.mean(letter_counter.values())\n",
    "    letters_std = statistics.stdev(letter_counter.values())\n",
    "    locs_mean = statistics.mean(locs_counter.values())\n",
    "    locs_std = statistics.stdev(locs_counter.values())\n",
    "\n",
    "    # Check if any letter count deviates significantly from the mean\n",
    "    for i in letter_counter.values():\n",
    "        if abs(i - letters_mean) >= 2 * letters_std:\n",
    "            # If deviation is larger that 2*STD, mark the sequence as not good and break the loop\n",
    "            good_seq = False\n",
    "            break\n",
    "\n",
    "    # Check if any location count deviates significantly from the mean\n",
    "    for i in locs_counter.values():\n",
    "        if abs(i - locs_mean) >= 2 * locs_std:\n",
    "            # If deviation is larger that 2*STD, mark the sequence as not good and break the loop\n",
    "            good_seq = False\n",
    "            break\n",
    "\n",
    "    # Return the flag indicating whether the sequence is good\n",
    "    return good_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of sequence_1: 300\n",
      "Length of sequence_10: 300\n",
      "Length of sequence_11: 300\n",
      "Length of sequence_12: 300\n",
      "sequence_13 is a bad sequence!\n",
      "sequence_14 is a bad sequence!\n",
      "Length of sequence_15: 300\n",
      "Length of sequence_16: 300\n",
      "sequence_18 is a bad sequence!\n",
      "sequence_19 is a bad sequence!\n",
      "Length of sequence_2: 300\n",
      "sequence_21 is a bad sequence!\n",
      "Length of sequence_22: 300\n",
      "Length of sequence_23: 300\n",
      "Length of sequence_25: 300\n",
      "sequence_26 is a bad sequence!\n",
      "Length of sequence_27: 300\n",
      "sequence_28 is a bad sequence!\n",
      "sequence_29 is a bad sequence!\n",
      "sequence_30 is a bad sequence!\n",
      "sequence_31 is a bad sequence!\n",
      "sequence_33 is a bad sequence!\n",
      "Length of sequence_34: 300\n",
      "sequence_36 is a bad sequence!\n",
      "Length of sequence_37: 300\n",
      "Length of sequence_38: 300\n",
      "Length of sequence_40: 300\n",
      "Length of sequence_41: 300\n",
      "Length of sequence_42: 300\n",
      "Length of sequence_43: 300\n",
      "Length of sequence_44: 300\n",
      "sequence_46 is a bad sequence!\n",
      "sequence_48 is a bad sequence!\n",
      "sequence_49 is a bad sequence!\n",
      "Length of sequence_50: 300\n",
      "Length of sequence_51: 300\n",
      "Length of sequence_52: 300\n",
      "Length of sequence_53: 300\n",
      "Length of sequence_55: 300\n",
      "sequence_56 is a bad sequence!\n",
      "sequence_57 is a bad sequence!\n",
      "sequence_58 is a bad sequence!\n",
      "sequence_59 is a bad sequence!\n",
      "Length of sequence_6: 300\n",
      "sequence_60 is a bad sequence!\n",
      "Length of sequence_61: 300\n",
      "Length of sequence_62: 300\n",
      "Length of sequence_63: 300\n",
      "Length of sequence_64: 300\n",
      "sequence_65 is a bad sequence!\n",
      "Length of sequence_7: 300\n",
      "Length of sequence_8: 300\n",
      "Length of sequence_9: 300\n"
     ]
    }
   ],
   "source": [
    "# Get a list of sequence files in the specified path\n",
    "sequence_files = os.listdir(seq_path_to_save)[:-1]\n",
    "\n",
    "# Iterate over each sequence file\n",
    "for file_name in sequence_files:\n",
    "    # Read the sequence CSV file into a DataFrame\n",
    "    seq_df = pd.read_csv(os.path.join(seq_path_to_save, file_name), delimiter=';', index_col=0)\n",
    "\n",
    "    # Initialize arrays to store letters and locations\n",
    "    letters_array = np.array([])\n",
    "    locs_array = np.array([])\n",
    "\n",
    "    # Extract letters and locations from each grid in the sequence\n",
    "    for grid in range(seq_df.shape[0]):\n",
    "        letters = ast.literal_eval(seq_df['letters'].iloc[grid])\n",
    "        letters_array = np.append(letters_array, [i for i in letters])\n",
    "\n",
    "        locs = ast.literal_eval(seq_df['locs'].iloc[grid])\n",
    "        locs_array = np.append(locs_array, [j for j in locs])\n",
    "\n",
    "    # Count occurrences of each letter and location\n",
    "    letter_counter = collections.Counter(letters_array)\n",
    "    locs_counter = collections.Counter(locs_array)\n",
    "    locs_counter = collections.Counter({int(k): v for k, v in locs_counter.items()})\n",
    "\n",
    "    # Check if the sequence is good based on sequence stats\n",
    "    good_seq = good_sequence(letter_counter, locs_counter)\n",
    "\n",
    "    # If the sequence is good, print its length and save statistics plots\n",
    "    if good_seq == True:\n",
    "        print(f'Length of {file_name[:-4]}: {len(seq_df)}')\n",
    "        plt.bar(letter_counter.keys(), letter_counter.values())\n",
    "        plt.savefig(f'{seq_path_to_save}stats\\{file_name[:-4]}_letters.png', bbox_inches='tight', pad_inches=0, dpi=300)\n",
    "        plt.close()\n",
    "        plt.bar(locs_counter.keys(), locs_counter.values())\n",
    "        plt.savefig(f'{seq_path_to_save}stats\\{file_name[:-4]}_locs.png', bbox_inches='tight', pad_inches=0, dpi=300)\n",
    "        plt.close()\n",
    "    # If the sequence is not good, print a message and remove the sequence file from the folder\n",
    "    else:\n",
    "        print(f'{file_name[:-4]} is a bad sequence!')\n",
    "        os.remove(os.path.join(seq_path_to_save, file_name))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**TASKS FOR EACH SEQUENCE**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Indexing of tasks:\n",
    "\n",
    "'perc': 0\n",
    "\n",
    "'pr_loc': 1\n",
    "\n",
    "'pr_let': 2\n",
    "\n",
    "'pr_rot': 3\n",
    "\n",
    "'pr_abc': 4\n",
    "\n",
    "'corr': np.nan - if perception, 0 - incorrect, 1 - correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_task_type(\n",
    "    seq_df: pd.DataFrame,\n",
    "    num_rows: int, integers: list[int],\n",
    "    percentage_per_integer: float\n",
    "            ) -> pd.DataFrame:\n",
    "\n",
    "    # Calculate the number of rows and 20% of the total rows\n",
    "    num_per_integer = int(num_rows * percentage_per_integer)\n",
    "    # Create and shuffle a list with integers 1 to 5 repeated 20% of the total rows\n",
    "    tasks_list = integers * num_per_integer\n",
    "    # Shuffle the tasks list to randomize the distribution of integers\n",
    "    random.shuffle(tasks_list)\n",
    "\n",
    "    return tasks_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rand_zero_ones(num_rows: int) -> list[int]:\n",
    "    # Calculate the number of 0 and 1 values needed based on the total number of rows\n",
    "    num_zero_ones = num_rows // 10  # Since there are 5 types of tasks, half correct (1), half incorrect (0)\n",
    "\n",
    "    # Create a list with alternating zero and one values\n",
    "    zero_ones = [0, 1] * num_zero_ones\n",
    "\n",
    "    # Shuffle the list to randomize the distribution of zero and one values\n",
    "    random.shuffle(zero_ones)\n",
    "\n",
    "    return zero_ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate correctness values for trials based on their task indices\n",
    "def set_correctness(num_rows: int, seq_df: pd.DataFrame) -> pd.DataFrame:\n",
    "    # Initialize a list to store the correctness of each trial (correct or incorrect)\n",
    "    answer_type = []\n",
    "\n",
    "    # Initialize lists for different types of tasks\n",
    "    zo_1, zo_2, zo_3, zo_4 = [], [], [], []\n",
    "\n",
    "    # Populate the lists with zero and one values representing correctness (0 for incorrect, 1 for correct)\n",
    "    for pattern in zo_1, zo_2, zo_3, zo_4:\n",
    "        pattern.extend(rand_zero_ones(num_rows))\n",
    "\n",
    "    # Assign correctness values to trials based on their task indices\n",
    "    for i in range(num_rows):\n",
    "        if seq_df['task_ind'].iloc[i] == 0:\n",
    "            # For trials with task index 0, mark the correctness as NaN\n",
    "            answer_type.append(np.nan)\n",
    "        else:\n",
    "            # For other task indices, assign correctness values based on the corresponding list\n",
    "            if seq_df['task_ind'].iloc[i] == 1:\n",
    "                answer_type.append(zo_1[0])\n",
    "                zo_1.pop(0)\n",
    "            elif seq_df['task_ind'].iloc[i] == 2:\n",
    "                answer_type.append(zo_2[0])\n",
    "                zo_2.pop(0)\n",
    "            elif seq_df['task_ind'].iloc[i] == 3:\n",
    "                answer_type.append(zo_3[0])\n",
    "                zo_3.pop(0)\n",
    "            else:\n",
    "                answer_type.append(zo_4[0])\n",
    "                zo_4.pop(0)\n",
    "    \n",
    "    # Convert NaN values to None and cast other values to integers\n",
    "    answer_type_int = [int(i) if not np.isnan(i) else i for i in answer_type]\n",
    "\n",
    "    # Return the list of correctness values\n",
    "    return answer_type_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_task(num_rows: int, seq_df: pd.DataFrame) -> pd.DataFrame:\n",
    "\n",
    "    # Create probe tasks for each trial\n",
    "    task = []\n",
    "\n",
    "    for i in range(num_rows):\n",
    "\n",
    "        if seq_df['task_ind'].iloc[i] == 0: # perception\n",
    "            task.append(np.nan)\n",
    "\n",
    "        elif seq_df['task_ind'].iloc[i] == 1: # location\n",
    "            if seq_df['answer_type'].iloc[i] == 0: # incorrect\n",
    "                good_locs = [i for i in ast.literal_eval(seq_df['incorr_locs'].iloc[i]) if i != 4]\n",
    "                loc_to_present = random.choice(good_locs)\n",
    "                task.append(loc_to_present)\n",
    "\n",
    "            elif seq_df['answer_type'].iloc[i] == 1: # correct\n",
    "                good_locs = [i for i in ast.literal_eval(seq_df['locs'].iloc[i]) if i != 4]\n",
    "                loc_to_present = random.choice(good_locs)\n",
    "                task.append(loc_to_present)\n",
    "\n",
    "        elif seq_df['task_ind'].iloc[i] == 2: # letter\n",
    "            if seq_df['answer_type'].iloc[i] == 0: # incorrect\n",
    "                pattern = ['-'] * 4\n",
    "                output_string = []\n",
    "                lett_incorr = ast.literal_eval(seq_df['incorr_letters'].iloc[i])\n",
    "\n",
    "                # Randomly pick a letter from the dictionary\n",
    "                picked_letter = random.choice(list(lett_incorr.keys()))\n",
    "                # Randomly select one of the incorrect positions for the picked letter\n",
    "                incorrect_position = random.choice(lett_incorr[picked_letter])\n",
    "                # Update the pattern with the picked letter in the incorrect position\n",
    "                pattern[incorrect_position] = picked_letter\n",
    "                # Convert the pattern list to a single string with incorrect position only\n",
    "                output_string = ' '.join(pattern)\n",
    "                task.append(f\"'{output_string}\")\n",
    "\n",
    "            elif seq_df['answer_type'].iloc[i] == 1: # correct\n",
    "                lett_corr = ast.literal_eval(seq_df['corr_letters'].iloc[i])\n",
    "                pattern = ['-'] * 4\n",
    "                output_string = []\n",
    "\n",
    "                # Randomly pick a letter from the dictionary\n",
    "                picked_letter = random.choice(list(lett_corr.keys()))\n",
    "                # Retrieve the correct position for the picked letter\n",
    "                correct_position = lett_corr[picked_letter]\n",
    "                # Update the pattern with the picked letter in the correct position\n",
    "                pattern[correct_position] = picked_letter\n",
    "                output_string = ' '.join(pattern)\n",
    "                task.append(f\"'{output_string}\")\n",
    "\n",
    "        elif seq_df['task_ind'].iloc[i] == 3: # rotation\n",
    "            if seq_df['answer_type'].iloc[i] == 0: # incorrect\n",
    "                good_locs = [i for i in ast.literal_eval(seq_df['incorr_rot'].iloc[i]) if i != 4]\n",
    "                loc_to_present = random.choice(good_locs)\n",
    "                task.append(loc_to_present)\n",
    "\n",
    "            elif seq_df['answer_type'].iloc[i] == 1: # correct\n",
    "                good_locs = [i for i in ast.literal_eval(seq_df['rotated_inds'].iloc[i]) if i != 4]\n",
    "                loc_to_present = random.choice(good_locs)\n",
    "                task.append(loc_to_present)\n",
    "\n",
    "        elif seq_df['task_ind'].iloc[i] == 4: # alphabet\n",
    "            if seq_df['answer_type'].iloc[i] == 0: # incorrect\n",
    "                pattern = ['-'] * 4\n",
    "                output_string = []\n",
    "                abc_incorr = ast.literal_eval(seq_df['incorr_abc'].iloc[i])\n",
    "\n",
    "                # Randomly pick a letter from the dictionary\n",
    "                picked_letter = random.choice(list(abc_incorr.keys()))\n",
    "                # Randomly select one of the incorrect positions for the picked letter\n",
    "                incorrect_position = random.choice(abc_incorr[picked_letter])\n",
    "                # Update the pattern with the picked letter in the incorrect position\n",
    "                pattern[incorrect_position] = picked_letter\n",
    "                # Convert the pattern list to a single string with incorrect position only\n",
    "                output_string = ' '.join(pattern)\n",
    "                task.append(f\"'{output_string}\")\n",
    "\n",
    "            elif seq_df['answer_type'].iloc[i] == 1: # correct\n",
    "                abc_corr = ast.literal_eval(seq_df['abc_ordered'].iloc[i])\n",
    "                # Define a list of letters to drop\n",
    "                letters_to_drop = ['Г', 'Д', 'Ф', 'Ш']\n",
    "                # Remove the letters to drop from the dictionary\n",
    "                for letter in letters_to_drop:\n",
    "                    if letter in abc_corr:\n",
    "                        if abc_corr[letter] == 0 or abc_corr[letter] == 3:\n",
    "                            del abc_corr[letter]\n",
    "\n",
    "                # Check if there are remaining letters in the dictionary\n",
    "                pattern = ['-'] * 4\n",
    "                output_string = []\n",
    "                # Randomly pick a letter from the updated dictionary\n",
    "                picked_letter = random.choice(list(abc_corr.keys()))\n",
    "                # Retrieve the correct position for the picked letter\n",
    "                correct_position = abc_corr[picked_letter]\n",
    "                # Update the pattern with the picked letter in the correct position\n",
    "                pattern[correct_position] = picked_letter\n",
    "                output_string = ' '.join(pattern)\n",
    "                task.append(f\"'{output_string}\")\n",
    "\n",
    "    return task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_files = os.listdir(seq_path_to_save)[:-1]\n",
    "\n",
    "for file_name in sequence_files:\n",
    "\n",
    "    seq_path = os.path.join(seq_path_to_save, file_name)\n",
    "    seq_df = pd.read_csv(seq_path, delimiter=';', index_col=0)\n",
    "\n",
    "    # Get the number of rows in the DataFrame\n",
    "    num_rows = len(seq_df)\n",
    "    # Define parameters for setting task type\n",
    "    integers = [0, 1, 2, 3, 4]  # Number of integers (0 to 4)\n",
    "    percentage_per_integer = 0.2\n",
    "\n",
    "    # Set task type for each trial in the sequence\n",
    "    seq_df['task_ind'] = set_task_type(seq_df, num_rows, integers, percentage_per_integer)\n",
    "    # Set correctness for each trial (50/50)\n",
    "    seq_df['answer_type'] = set_correctness(num_rows, seq_df)\n",
    "    # Set particular probe for each trial\n",
    "    seq_df['probe_stim'] = set_task(num_rows, seq_df)\n",
    "\n",
    "    # Filter trials by task types\n",
    "    # location trials\n",
    "    locs = seq_df[seq_df.task_ind.isin([1])] # simple\n",
    "    rots = seq_df[seq_df.task_ind.isin([3])] # complex\n",
    "\n",
    "    # letter trials\n",
    "    letts = seq_df[seq_df.task_ind.isin([2])] # simple\n",
    "    abcs = seq_df[seq_df.task_ind.isin([4])] # complex\n",
    "\n",
    "    task_types = [locs, rots, letts]\n",
    "\n",
    "    # Check if any probe appears more than half the time in a trial type\n",
    "    for task_type in task_types:\n",
    "        for probe in task_type['probe_stim'].value_counts():\n",
    "            if probe > len(task_type) // 2:\n",
    "                task_ind = task_type['task_ind'].iloc[0]\n",
    "                task_goodness = f'Value {probe} in trial type {task_ind} is out of range!'\n",
    "                print(file_name[:-4], ':', task_goodness)\n",
    "\n",
    "    # Check if any alphabet probe exceeds half the threshold\n",
    "    lett_list = []\n",
    "    loc_list = []\n",
    "    for string in abcs['probe_stim']:\n",
    "        string = string.replace(' ', '')\n",
    "        match = re.search('[\\u0400-\\u04FF]', string)\n",
    "        char = match.group()\n",
    "        position = string.index(char)\n",
    "        lett_list.append(char)\n",
    "        loc_list.append(position)\n",
    "\n",
    "        lett_counter = Counter(lett_list)\n",
    "        loc_counter = Counter(loc_list)\n",
    "\n",
    "        threshold = len(abcs) // 2\n",
    "        exceeds_threshold = any(value > threshold for value in lett_counter.values())\n",
    "        if exceeds_threshold == True:\n",
    "            abc_goodness = f'Value {string} in trial type 4 is out of range!'\n",
    "            print(file_name[:-4], ':', abc_goodness)\n",
    "\n",
    "    # Save the modified DataFrame back to the CSV file\n",
    "    seq_df.to_csv(seq_path, sep=';', index=True, encoding='utf-8-sig')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LAST CHECK OF RESULTING SEQS: REPETITION OF TRIALS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqs_path = '...\\\\sequences\\\\' #! SET YOUR PATH HERE\n",
    "sequence_files = os.listdir(seqs_path)[:-1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "possible_inds = [1, 2, 3, 4]\n",
    "\n",
    "sequence_files = os.listdir(seqs_path)[:-1]\n",
    "\n",
    "for file_name in sequence_files:\n",
    "    seq_path = os.path.join(seqs_path, file_name)\n",
    "    seq_df = pd.read_csv(seq_path, delimiter=';', index_col=0)\n",
    "    task_indices = seq_df['task_ind'].tolist()\n",
    "\n",
    "    # Initialize counters for the two rules\n",
    "    task_theshold = 0\n",
    "    modality_threshold = 0\n",
    "    last_index = None\n",
    "    last_modality = None\n",
    "\n",
    "    # Define your rules\n",
    "    max_task_theshold = 3\n",
    "    max_modality_threshold = 5\n",
    "    rows_counter = 1\n",
    "\n",
    "    # Set a flag for good sequence\n",
    "    good_seq = True\n",
    "\n",
    "    # Iterate through the list of task indices\n",
    "    for index in task_indices:\n",
    "        rows_counter += 1\n",
    "\n",
    "        # Check rule 1: No more than 3 tasks in a row with the same index\n",
    "        if index == last_index:\n",
    "            task_theshold += 1\n",
    "        else:\n",
    "            task_theshold = 1\n",
    "            last_index = index\n",
    "\n",
    "        if task_theshold > max_task_theshold:\n",
    "            good_seq = False\n",
    "            print(f\"Line {rows_counter}: Rule 1 Violation\")\n",
    "\n",
    "\n",
    "    # Check rule 2: No more than 5 tasks in a row of the same modality\n",
    "        if index in possible_inds:\n",
    "            modality = index % 2  # Assuming 1 and 3 are visual and 2 and 4 are verbal\n",
    "            if modality == last_modality:\n",
    "                modality_threshold += 1\n",
    "            else:\n",
    "                modality_threshold = 1\n",
    "                last_modality = modality\n",
    "\n",
    "            if modality_threshold > max_modality_threshold:\n",
    "                good_seq = False\n",
    "                print(f\"Line {rows_counter}: Rule 2 Violation\")\n",
    "\n",
    "        else:\n",
    "            modality_threshold = 1\n",
    "            last_modality = None\n",
    "\n",
    "    # Check if both rules are satisfied\n",
    "    if good_seq == True:\n",
    "        print(f\"{file_name} is GOOD!\")\n",
    "    else:\n",
    "        print(f'{file_name[:-4]} is a bad sequence! It\\'ll be REMOVED')\n",
    "        os.remove(os.path.join(seqs_path, file_name))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*NOTE:* if you are not satisfied with the resulting number of sequences, repeat the generation and exclusion procedures once again.\n",
    "\n",
    "(!) But do not forget to reset range(0, 15) in the cell that generates sequences to avoid overwriting."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**THE END**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4826bf0c3d2240af24262f67909c256cdace7a9ee93669e372a1248fd44e3f16"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}