[Colab]DBE_Analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ByungjunKim/NationStateChosun/blob/main/%5BColab%5DDBE_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_O1XdUyficeG"
      },
      "source": [
        "# DBE 분석결과 정리"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HSK4fWPCiksp"
      },
      "outputs": [],
      "source": [
        "!pip install git+https://github.com/ByungjunKim/dynamic_bernoulli_embeddings.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M4AFgYxT9Zcu"
      },
      "outputs": [],
      "source": [
        "!pip install -U gdown"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2htx2RVficeJ"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "from dynamic_bernoulli_embeddings.analysis import DynamicEmbeddingAnalysis\n",
        "from dynamic_bernoulli_embeddings.training import train_model\n",
        "from gensim.corpora import Dictionary\n",
        "from tqdm import tqdm\n",
        "from collections import Counter\n",
        "import math\n",
        "import numpy as np\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from itertools import chain\n",
        "import gdown\n",
        "import pickle\n",
        "!mkdir results # results 폴더 생성"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sVbhph0X-CGx"
      },
      "outputs": [],
      "source": [
        "# 학습 완료한 임베딩 모델 다운로드\n",
        "## 조선일보\n",
        "# https://drive.google.com/file/d/1UmsDoOrA6ys0ic-xpEbGFBA-NaqHjdLu/view?usp=sharing\n",
        "gdown.download(id='1UmsDoOrA6ys0ic-xpEbGFBA-NaqHjdLu',output='230222_chosun_emb.pkl')\n",
        "## 동아일보\n",
        "# https://drive.google.com/file/d/1UqBehgGa_Ts9sGGLvVq-zw3f4njNNewt/view?usp=sharing\n",
        "gdown.download(id='1UqBehgGa_Ts9sGGLvVq-zw3f4njNNewt',output='230128_donga_emb.pkl')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dA0ASekO-SuR"
      },
      "outputs": [],
      "source": [
        "# pickle 파일(임베딩 모델) 불러오기\n",
        "with open('./230222_chosun_emb.pkl', 'rb') as f:\n",
        "    chosun_emb = pickle.load(f)\n",
        "with open('./230128_donga_emb.pkl', 'rb') as f:\n",
        "    donga_emb = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6TqT_eG5iceN"
      },
      "outputs": [],
      "source": [
        "# 시작년도(min_year)와 기간(max_time) 설정\n",
        "min_year = 1920\n",
        "max_time = 20"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ynaDZMnRiceN"
      },
      "outputs": [],
      "source": [
        "# 연도별 가장 유사어(고맥락어) 테이블로 추출하기\n",
        "def time_neigh(emb,target, min_year,max_time, top_n):\n",
        "    over_time = {}\n",
        "    for i in range(0, max_time + 1, 1):\n",
        "        col = str(min_year + i)\n",
        "        over_time[col] = emb.neighborhood(target, i, top_n)\n",
        "    return pd.DataFrame(over_time)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vsZWyF9d-dQQ"
      },
      "outputs": [],
      "source": [
        "# test\n",
        "time_neigh(chosun_emb,'민족',1920,20,10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v_xCKRSuiceO"
      },
      "outputs": [],
      "source": [
        "# 타겟단어군(분석대상) 설정\n",
        "target_words = ['민족', '국민', '국가', '독립', '문화', '문명',\\\n",
        "                '인민', '신민' ,'노동자', '사회', '지주', '경제', '계급', '운동',\\\n",
        "                '청년', '생활', '태도', '문학']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "knhdGt3SiceO"
      },
      "source": [
        "### 1. 조선일보"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DywVV6PFiceO"
      },
      "source": [
        "##### a. 단어별 부침(drift)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ysCTwWaDiceO"
      },
      "outputs": [],
      "source": [
        "chosun_drift = pd.DataFrame(chosun_emb.absolute_drift(n=100000),columns=['drift','단어'])\n",
        "chosun_drift.to_excel('./results/chosun_drift.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b8jihbZficeO"
      },
      "outputs": [],
      "source": [
        "len(chosun_drift)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "33UMyb8hiceP"
      },
      "outputs": [],
      "source": [
        "chosun_drift.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hrSset4CiceP"
      },
      "outputs": [],
      "source": [
        "chosun_drift['drift'].hist()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K1BKkNyBiceP"
      },
      "source": [
        "##### b. 주요 단어 맥락 변화"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P7nsiZUsiceP"
      },
      "outputs": [],
      "source": [
        "for idx, target_word in tqdm(enumerate(target_words)):\n",
        "    try:\n",
        "        df = time_neigh(chosun_emb,target_word,1920,20,100)\n",
        "    except KeyError:\n",
        "        print(f'{target_word} 없음')\n",
        "        continue\n",
        "    if idx ==0:\n",
        "        with pd.ExcelWriter(f'./results/chosun_time_neigh.xlsx',mode='w') as writer:\n",
        "            df.to_excel(writer,sheet_name=target_word,index=None)\n",
        "    else:\n",
        "        with pd.ExcelWriter(f'./results/chosun_time_neigh.xlsx',mode='a') as writer:\n",
        "            df.to_excel(writer,sheet_name=target_word,index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tTip2Zs1iceQ"
      },
      "outputs": [],
      "source": [
        "df = time_neigh(chosun_emb,'민족',1920,20,30)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "av65jNaficeQ"
      },
      "outputs": [],
      "source": [
        "# 1923과 1924 서로 다른 단어 (1923에만 존재)\n",
        "print(set(df['1923'].map(lambda x:x[0]).tolist()) - set(df['1924'].map(lambda x:x[0]).tolist()))\n",
        "\n",
        "# 1923과 1924 서로 다른 단어 (1924에만 존재)\n",
        "print(set(df['1924'].map(lambda x:x[0]).tolist()) - set(df['1923'].map(lambda x:x[0]).tolist()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vp-hMiQwiceQ"
      },
      "outputs": [],
      "source": [
        "# 1924과 1925 서로 다른 단어 (1924에만 존재)\n",
        "print(set(df['1924'].map(lambda x:x[0]).tolist()) - set(df['1925'].map(lambda x:x[0]).tolist()))\n",
        "\n",
        "# 1924과 1925 서로 다른 단어 (1925에만 존재)\n",
        "print(set(df['1925'].map(lambda x:x[0]).tolist()) - set(df['1924'].map(lambda x:x[0]).tolist()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B8fn6bXUiceQ"
      },
      "outputs": [],
      "source": [
        "# 1935과 1926 서로 다른 단어 (1935에만 존재)\n",
        "print(set(df['1935'].map(lambda x:x[0]).tolist()) - set(df['1936'].map(lambda x:x[0]).tolist()))\n",
        "\n",
        "# 1935과 1936 서로 다른 단어 (1936에만 존재)\n",
        "print(set(df['1936'].map(lambda x:x[0]).tolist()) - set(df['1935'].map(lambda x:x[0]).tolist()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oc0zGAS_iceQ"
      },
      "source": [
        "##### 두 단어의 시계열에 따른 유사도 추이"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z2SFfxi8iceQ"
      },
      "outputs": [],
      "source": [
        "def time_similarity(main_word, alter_words, max_time):\n",
        "    simil = {}\n",
        "    for word in alter_words:\n",
        "        simil[word] = [t[1] for i in range(0, max_time + 1, 1) for t in chosun_emb.neighborhood(main_word, i, 10000) if t[0]==word]\n",
        "    return simil"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i3-5v3LQiceQ"
      },
      "outputs": [],
      "source": [
        "nation = pd.DataFrame.from_dict(time_similarity('민족',['인종','국가','국민','국어','언어','민중','인민','백성','신민','동포','나라','조선','독립','사회주의','자본주의','문학','문명','문화','근대'],max_time),orient='index').T\n",
        "nation.index = list(range(1920,1941,1))\n",
        "# nation.to_excel('./results/민족_time_similarity.xlsx')\n",
        "nation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yQ4fCa_-iceQ"
      },
      "outputs": [],
      "source": [
        "nation.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ER8KESqNiceQ"
      },
      "outputs": [],
      "source": [
        "nation_국민 = pd.DataFrame.from_dict(time_similarity('국민',['인종','국가','민족','국어','언어','민중','인민','백성','신민','동포','나라','조선','독립','사회주의','자본주의','문학','문명','문화','근대'],max_time),orient='index').T\n",
        "nation_국민.index = list(range(1920,1941,1))\n",
        "nation_국민.to_excel('./results/국민_time_similarity.xlsx')\n",
        "nation_국민"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rdJy9moIiceQ"
      },
      "outputs": [],
      "source": [
        "state = pd.DataFrame.from_dict(time_similarity('국가',['인종','국민','민족','국어','언어','민중','인민','백성','신민','동포','나라','조선','독립','사회주의','자본주의','문학','문명','문화','근대'],max_time),orient='index').T\n",
        "state.index = list(range(1920,1941,1))\n",
        "state.to_excel('./results/국가_time_similarity.xlsx')\n",
        "state"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zZPenPGAiceR"
      },
      "source": [
        "##### c. 단어 변곡점"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4g0QY0-KiceR"
      },
      "outputs": [],
      "source": [
        "chosun_change_point = pd.DataFrame(chosun_emb.change_points(n=1000000),columns=['연도','단어','change'])\n",
        "chosun_change_point['연도'] = chosun_change_point['연도'] + 1920\n",
        "chosun_change_point\n",
        "# chosun_change_point.to_excel('./results/chosun_change_point.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zqZSru34iceR"
      },
      "outputs": [],
      "source": [
        "# 타겟단어 변곡점\n",
        "chosun_change_point_target = chosun_change_point[chosun_change_point['단어'].isin(target_words)]\n",
        "chosun_change_point_target = chosun_change_point_target.sort_values(by=['단어','change'],ascending=False).reset_index(drop=True)\n",
        "chosun_change_point_target.to_excel('./results/chosun_change_point_target.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "44z6nlfGiceR"
      },
      "outputs": [],
      "source": [
        "# @title\n",
        "# 단어별 변곡점 20개씩 있는지 확인\n",
        "chosun_change_point_target.value_counts('단어')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MDMFgT78iceR"
      },
      "outputs": [],
      "source": [
        "chosun_change_point_target[chosun_change_point_target['단어']=='태도'].sort_values(by=['연도'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QEPsBlnniceR"
      },
      "outputs": [],
      "source": [
        "chosun_change_point_target.to_excel('./results/chosun_change_point_target_all.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pNEH3_xoiceR"
      },
      "outputs": [],
      "source": [
        "for idx, target_word in tqdm(enumerate(target_words)):\n",
        "    df = chosun_change_point_target[chosun_change_point_target['단어']==target_word].sort_values(by=['연도'])\n",
        "    if idx ==0:\n",
        "        with pd.ExcelWriter(f'./results/chosun_change_point_target.xlsx',mode='w') as writer:\n",
        "            df.to_excel(writer,sheet_name=target_word,index=None)\n",
        "    else:\n",
        "        with pd.ExcelWriter(f'./results/chosun_change_point_target.xlsx',mode='a') as writer:\n",
        "            df.to_excel(writer,sheet_name=target_word,index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "esJh35zbiceR"
      },
      "outputs": [],
      "source": [
        "# 타겟단어 변곡점(change max 추출)\n",
        "chosun_change_max_idx = chosun_change_point_target.groupby(['단어'])['change'].idxmax().tolist()\n",
        "chosun_change_point_target.iloc[chosun_change_max_idx].sort_values('연도')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bmKMuCcliceR"
      },
      "outputs": [],
      "source": [
        "# 전체 단어 연도별 변동 평균\n",
        "chosun_change_point.groupby(['연도'])['change'].mean().to_excel('./results/chosun_change_point_모든단어 연도별 평균.xlsx')\n",
        "chosun_change_point.groupby(['연도'])['change'].mean()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CkhrXjXdicea"
      },
      "outputs": [],
      "source": [
        "chosun_change_point_des = chosun_change_point.groupby(['단어'])['change'].describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8nz8k1rxicea"
      },
      "outputs": [],
      "source": [
        "# 변동계수\n",
        "chosun_change_point_des['cv'] = chosun_change_point_des['std'] / chosun_change_point_des['mean']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sAG-5hC0icea"
      },
      "outputs": [],
      "source": [
        "# 변화량(표준편차)가 가장 큰 단어들\n",
        "chosun_change_point_des.sort_values(by=['cv'],ascending=False).to_excel('./results/chosun_change_point_des_all.xlsx')\n",
        "chosun_change_point_des.sort_values(by=['cv'],ascending=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SWmvLxg3icea"
      },
      "outputs": [],
      "source": [
        "# chosun_change_point_target.groupby(['단어'])['change'].describe().to_excel('./results/chosun_change_point_des.xlsx')\n",
        "chosun_change_point_target.groupby(['단어'])['change'].describe().sort_values(by=['std'],ascending=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QreNy1kpicea"
      },
      "source": [
        "### 2. 동아일보"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vf0VOCmiicea"
      },
      "source": [
        "##### a. 단어별 부침(drift)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pVskvtLbicea"
      },
      "outputs": [],
      "source": [
        "donga_drift = pd.DataFrame(donga_emb.absolute_drift(n=5000),columns=['drift','단어'])\n",
        "donga_drift.to_excel('./results/donga_drift.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJGQVn6eicea"
      },
      "source": [
        "##### b. 주요 단어 맥락 변화"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R5JXoynNiceb"
      },
      "outputs": [],
      "source": [
        "for idx, target_word in tqdm(enumerate(target_words)):\n",
        "    try:\n",
        "        df = time_neigh(donga_emb,target_word,1920,20,100)\n",
        "    except KeyError:\n",
        "        print(f'{target_word} 없음')\n",
        "        continue\n",
        "    if idx ==0:\n",
        "        with pd.ExcelWriter(f'./results/donga_time_neigh.xlsx',mode='w') as writer:\n",
        "            df.to_excel(writer,sheet_name=target_word,index=None)\n",
        "    else:\n",
        "        with pd.ExcelWriter(f'./results/donga_time_neigh.xlsx',mode='a') as writer:\n",
        "            df.to_excel(writer,sheet_name=target_word,index=None)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2LE9nVbhiceb"
      },
      "source": [
        "##### c. 단어 변곡점"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xsD8dDY6iceb"
      },
      "outputs": [],
      "source": [
        "donga_change_point = pd.DataFrame(donga_emb.change_points(n=150000),columns=['연도','단어','change'])\n",
        "donga_change_point['연도'] = donga_change_point['연도'] + 1920\n",
        "donga_change_point\n",
        "donga_change_point.to_excel('./results/donga_change_point.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ze_upxVhiceb"
      },
      "outputs": [],
      "source": [
        "# 타겟단어 변곡점\n",
        "donga_change_point_target = donga_change_point[donga_change_point['단어'].isin(target_words)]\n",
        "donga_change_point_target = donga_change_point_target.sort_values(by=['단어','change'],ascending=False).reset_index(drop=True)\n",
        "donga_change_point_target.to_excel('./results/donga_change_point_target.xlsx',index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5OsJQBliiceb"
      },
      "outputs": [],
      "source": [
        "# 타겟단어 변곡점(change max 추출)\n",
        "donga_change_max_idx = donga_change_point_target.groupby(['단어'])['change'].idxmax().tolist()\n",
        "donga_change_point_target.iloc[donga_change_max_idx].sort_values('연도')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "DBE",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.16"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "f042807ede7a69bb01d6b24203a6c503adc8fbf7ff9535f4cda3e9ddf44e3a21"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}