From 3e39ebd36fb35df5ec9bd3cf01e3865aca811c42 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 8 Mar 2020 21:12:48 +0900 Subject: [PATCH 01/13] =?UTF-8?q?[update]=20Token=20page=20url=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00c6dc1..0b5384b 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ pip3 install -r requirements.txt ### 2. Slack Token 환경 변수 지정 - 해당 슬랙 workspace의 API를 접근할 수 있는 토큰을 받아와야 합니다. -- [이곳](https://api.slack.com/legacy/custom-integrations/legacy-tokens)에서 슬랙 토큰을 생성받습니다. +- [이곳](https://api.slack.com/apps/AV18VEUMS/oauth?)에서 슬랙 토큰을 생성받습니다. - 생성 받은 토큰을 `.bash_profile`, `.zshrc` 등의 파일에 다음과 같이 환경 변수로 저장해줍니다. ``` export SLACK_TOKEN='xoxo-your-token' From aa93cd3679b567f848707269030c994e146eb65c Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 8 Mar 2020 21:33:58 +0900 Subject: [PATCH 02/13] [chore] reformat code --- common/slack_export.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/common/slack_export.py b/common/slack_export.py index 12acfbc..7662ea2 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -26,8 +26,8 @@ def get_history(pageable_object, channel_id, page_size=100): messages.extend(response['messages']) if response['has_more']: - last_timestamp = messages[-1]['ts'] # -1 means last element in a list - sleep(1) # Respect the Slack API rate limit + last_timestamp = messages[-1]['ts'] # -1 means last element in a list + sleep(1) # Respect the Slack API rate limit else: break return messages @@ -47,7 +47,7 @@ def parse_time_stamp(time_stamp): if len(t_list) != 2: raise ValueError('Invalid time stamp') else: - return datetime.utcfromtimestamp( float(t_list[0])) + return datetime.utcfromtimestamp(float(t_list[0])) def channel_rename(old_room_name, new_room_name): @@ -59,7 +59,7 @@ def channel_rename(old_room_name, new_room_name): return mkdir(new_room_name) for file_name in os.listdir(old_room_name): - shutil.move( os.path.join(old_room_name, file_name), new_room_name) + shutil.move(os.path.join(old_room_name, file_name), new_room_name) os.rmdir(old_room_name) @@ -102,8 +102,8 @@ def parse_messages(room_dir, messages, room_type): channel_rename(old_room_path, new_room_path) current_messages.append(message) - out_file_name = '{room}/{file}.json'.format(room=room_dir, file=current_file_date ) - write_message_file( out_file_name, current_messages) + out_file_name = '{room}/{file}.json'.format(room=room_dir, file=current_file_date) + write_message_file(out_file_name, current_messages) def filter_conversations_by_name(channels_or_groups, channel_or_group_names): @@ -166,7 +166,7 @@ def dump_user_file(users): write to user file, any existing file needs to be overwritten. """ with open(os.path.join('../', 'users.json'), 'w') as userFile: - json.dump( users, userFile, indent=4 ) + json.dump(users, userFile, indent=4) def bootstrap_key_values(slack_obj, _dry_run): From 03801ad0f559cbdba8cc3e8bab48b1194e70c4b6 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 8 Mar 2020 21:37:27 +0900 Subject: [PATCH 03/13] =?UTF-8?q?[add]=20=20{user=5Fname,=20id}=20list=20?= =?UTF-8?q?=EB=A5=BC=20=EC=B6=94=EC=B6=9C=ED=95=98=EB=8A=94=20=EB=A9=94?= =?UTF-8?q?=EC=86=8C=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/slack_export.py b/common/slack_export.py index 7662ea2..65ada36 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -216,3 +216,17 @@ def finalize(zip_name, output_directory): if zip_name: shutil.make_archive(zip_name, 'zip', output_directory, None) shutil.rmtree(output_directory) + + +def get_user_names_with_id(users): + users_with_id = [] + + for user in users: + if is_valid_user(user): + users_with_id.append({'name': user['real_name'], 'id': user['id']}) + + return users_with_id + + +def is_valid_user(user): + return not user['is_bot'] and not user['deleted'] and user['real_name'] != 'Slackbot' From 22f10ab10ab21f7fdd585a55de7816e64d9869a9 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 8 Mar 2020 21:39:05 +0900 Subject: [PATCH 04/13] =?UTF-8?q?[add]=20{user=5Fname,=20channel=5Fname}?= =?UTF-8?q?=20list=20=EB=A5=BC=20=EC=B6=94=EC=B6=9C=ED=95=98=EB=8A=94=20?= =?UTF-8?q?=EB=A9=94=EC=86=8C=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/common/slack_export.py b/common/slack_export.py index 65ada36..cd4741e 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -218,6 +218,21 @@ def finalize(zip_name, output_directory): shutil.rmtree(output_directory) +def get_user_names_with_channel(channels, channel_prefix): + users_with_channel = [] + + for channel in channels: + channel_name = channel['name'] + + if channel_name.startswith(channel_prefix): + member_names = channel['topic']['value'].split() + + for name in member_names: + users_with_channel.append({'name': name, 'channel_name': channel_name}) + + return users_with_channel + + def get_user_names_with_id(users): users_with_id = [] From bc868f9472d799a18d1900b3af8a1e1b12fe72d2 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 8 Mar 2020 22:15:22 +0900 Subject: [PATCH 05/13] =?UTF-8?q?[add]=20User=20table=20=EC=9D=84=20?= =?UTF-8?q?=EB=B0=98=ED=99=98=ED=95=98=EB=8A=94=20=EB=A9=94=EC=86=8C?= =?UTF-8?q?=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/common/slack_export.py b/common/slack_export.py index cd4741e..3258e74 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -4,11 +4,13 @@ from datetime import datetime from pick import pick from time import sleep +from collections import defaultdict user_names_by_id = {} user_ids_by_name = {} dry_run = None +user_name_key = 'user_name' def get_history(pageable_object, channel_id, page_size=100): @@ -218,6 +220,27 @@ def finalize(zip_name, output_directory): shutil.rmtree(output_directory) +# channel_prefix : prefix of the posting channel +def get_user_table(channel_prefix): + users = slack.users.list().body['members'] + channels = slack.channels.list().body['channels'] + + user_names_with_id = get_user_names_with_id(users) + user_names_with_channel = get_user_names_with_channel(channels, channel_prefix) + + if len(user_names_with_id) == len(user_names_with_channel): + user_table = defaultdict(dict) + + for merged_list in (user_names_with_id, user_names_with_channel): + for element in merged_list: + user_table[element[user_name_key]].update(element) + + # row = {'name': '홍길동', 'id': 'UTHXXXXX', 'channel_name': 'prefix_직군_게시글'} + return user_table + else: + print("Fail to get user table. cause, length do not match.") + + def get_user_names_with_channel(channels, channel_prefix): users_with_channel = [] @@ -228,7 +251,7 @@ def get_user_names_with_channel(channels, channel_prefix): member_names = channel['topic']['value'].split() for name in member_names: - users_with_channel.append({'name': name, 'channel_name': channel_name}) + users_with_channel.append({user_name_key: name, 'channel_name': channel_name}) return users_with_channel @@ -238,7 +261,7 @@ def get_user_names_with_id(users): for user in users: if is_valid_user(user): - users_with_id.append({'name': user['real_name'], 'id': user['id']}) + users_with_id.append({user_name_key: user['real_name'], 'id': user['id']}) return users_with_id From 4ab6d18450f486f743b86109ece5deb56988e2d0 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Tue, 10 Mar 2020 21:11:42 +0900 Subject: [PATCH 06/13] =?UTF-8?q?[modify]=20get=5Fuser=5Ftable=20=ED=95=A8?= =?UTF-8?q?=EC=88=98=EA=B0=80=20=EC=A3=BC=EC=9E=85=EB=B0=9B=EC=9D=80=20sla?= =?UTF-8?q?cker=20=EB=A5=BC=20=EC=82=AC=EC=9A=A9=ED=95=98=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/slack_export.py b/common/slack_export.py index 3258e74..0cad41f 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -221,9 +221,9 @@ def finalize(zip_name, output_directory): # channel_prefix : prefix of the posting channel -def get_user_table(channel_prefix): - users = slack.users.list().body['members'] - channels = slack.channels.list().body['channels'] +def get_user_table(slacker, channel_prefix): + users = slacker.users.list().body['members'] + channels = slacker.channels.list().body['channels'] user_names_with_id = get_user_names_with_id(users) user_names_with_channel = get_user_names_with_channel(channels, channel_prefix) From 3ed0719acbc5034eb376817b680d90c46dcfc7e7 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Tue, 10 Mar 2020 21:14:04 +0900 Subject: [PATCH 07/13] =?UTF-8?q?[modify]=20user=20table=20=EC=97=90=20?= =?UTF-8?q?=EB=8C=80=ED=95=9C=20=EC=A3=BC=EC=84=9D=20=EB=82=B4=EC=9A=A9?= =?UTF-8?q?=EC=9D=84=20=EC=A2=80=20=EB=8D=94=20=EB=AA=85=ED=99=95=ED=95=98?= =?UTF-8?q?=EA=B2=8C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/slack_export.py b/common/slack_export.py index 0cad41f..6e3fe3e 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -235,7 +235,7 @@ def get_user_table(slacker, channel_prefix): for element in merged_list: user_table[element[user_name_key]].update(element) - # row = {'name': '홍길동', 'id': 'UTHXXXXX', 'channel_name': 'prefix_직군_게시글'} + # key = '홍길동', value = {'name': '홍길동', 'id': 'UTHXXXXX', 'channel_name': 'prefix_직군_게시글'} 인 dictionary. return user_table else: print("Fail to get user table. cause, length do not match.") From 886db06a9d25ceb6e431afa703f6394694d05dd1 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Thu, 12 Mar 2020 23:48:41 +0900 Subject: [PATCH 08/13] =?UTF-8?q?[modify]=20slack=20api=20=EA=B0=80?= =?UTF-8?q?=EC=9D=B4=EB=93=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 43b2560..9eeb97b 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,21 @@ source env/bin/activate pip3 install -r requirements.txt ``` -### 2. Slack Token 환경 변수 지정 -- 해당 슬랙 workspace의 API를 접근할 수 있는 토큰을 받아와야 합니다. -- [이곳](https://api.slack.com/apps/AV18VEUMS/oauth?)에서 슬랙 토큰을 생성받습니다. -- 생성 받은 토큰을 `.bash_profile`, `.zshrc` 등의 파일에 다음과 같이 환경 변수로 저장해줍니다. +### 2. Slack API +1. workspace admin 권한을 받습니다. +2. 새로운 글또 workspace 에 slack app 을 install 합니다. + + [Slack App 페이지](https://api.slack.com/apps) -> [Create New App] +3. Your apps -> 글또 앱 -> OAuth & Permissions 탭에서 생성된 API 토큰을 확인할 수 있습니다. +4. 해당 토큰을 `.bash_profile`, `.zshrc` 등의 파일에 다음과 같이 환경 변수로 저장해줍니다. ``` export SLACK_TOKEN='xoxo-your-token' ``` +* Collaborator 권한이 있어야 app 관리 대시보드로 접근이 가능하기 때문에 Collaborators + 탭에서 관련 담당자분들을 미리 추가해주시면 좋습니다. + + ### 3. 마감 날짜 데이터, 유저 데이터 저장 - 코드 실행 시 활용되는 유저 데이터와 마감 날짜 데이터를 다음과 같이 `outputs` 디렉토리 아래에 저장해주세요. From 039549752faf9fd73ef734577f978b0395f97578 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sat, 14 Mar 2020 14:48:36 +0900 Subject: [PATCH 09/13] =?UTF-8?q?[modify]=20BigQuery=20=EC=97=85=EB=A1=9C?= =?UTF-8?q?=EB=93=9C=20=ED=8F=AC=EB=A7=B7=EC=97=90=20=EB=A7=9E=EC=B6=B0=20?= =?UTF-8?q?key=20=EB=A5=BC=20=EC=A0=9C=EC=99=B8=ED=95=98=EA=B3=A0=20?= =?UTF-8?q?=EB=A6=AC=ED=84=B4=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/slack_export.py b/common/slack_export.py index 6e3fe3e..535f290 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -235,8 +235,8 @@ def get_user_table(slacker, channel_prefix): for element in merged_list: user_table[element[user_name_key]].update(element) - # key = '홍길동', value = {'name': '홍길동', 'id': 'UTHXXXXX', 'channel_name': 'prefix_직군_게시글'} 인 dictionary. - return user_table + # user_table : key = '홍길동', value = {'name': '홍길동', 'id': 'UTHXXXXX', 'channel_name': 'prefix_직군_게시글'} 인 dict. + return user_table.values() else: print("Fail to get user table. cause, length do not match.") From ecaa7bceabe1ed8ce321839b568273928259d0f9 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sat, 14 Mar 2020 23:29:24 +0900 Subject: [PATCH 10/13] =?UTF-8?q?[modify]=20user=20=ED=85=8C=EC=9D=B4?= =?UTF-8?q?=EB=B8=94=EC=97=90=20channel=5Fid=20=EC=BB=AC=EB=9F=BC=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/slack_export.py b/common/slack_export.py index 535f290..4ce3186 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -245,13 +245,14 @@ def get_user_names_with_channel(channels, channel_prefix): users_with_channel = [] for channel in channels: + channel_id = channel['id'] channel_name = channel['name'] if channel_name.startswith(channel_prefix): member_names = channel['topic']['value'].split() for name in member_names: - users_with_channel.append({user_name_key: name, 'channel_name': channel_name}) + users_with_channel.append({user_name_key: name, 'channel_id': channel_id, 'channel_name': channel_name}) return users_with_channel From 3d1d520a9b80c5c402d3456194d88678a6a22ad4 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 15 Mar 2020 01:46:14 +0900 Subject: [PATCH 11/13] =?UTF-8?q?[modify]=20=EC=BB=AC=EB=9F=BC=20=EC=88=9C?= =?UTF-8?q?=EC=84=9C=20=EB=B3=80=EA=B2=BD=ED=95=98=EC=97=AC=20=EA=B0=80?= =?UTF-8?q?=EB=8F=85=EC=84=B1=20=ED=99=95=EB=B3=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/slack_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/slack_export.py b/common/slack_export.py index 4ce3186..64ac92d 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -262,7 +262,7 @@ def get_user_names_with_id(users): for user in users: if is_valid_user(user): - users_with_id.append({user_name_key: user['real_name'], 'id': user['id']}) + users_with_id.append({'user_id': user['id'], user_name_key: user['real_name']}) return users_with_id From 8e4380b5f451ac4569f635c72a5793e3efc254a8 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 15 Mar 2020 01:56:35 +0900 Subject: [PATCH 12/13] =?UTF-8?q?[add]=20=EB=B9=85=EC=BF=BC=EB=A6=AC?= =?UTF-8?q?=EC=97=90=20user=20=ED=85=8C=EC=9D=B4=EB=B8=94=20read/write=20?= =?UTF-8?q?=ED=95=98=EB=8A=94=20=ED=95=A8=EC=88=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/extract_data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common/extract_data.py b/common/extract_data.py index 2c69d3f..83d9185 100644 --- a/common/extract_data.py +++ b/common/extract_data.py @@ -122,3 +122,13 @@ def send_data_to_gbq(dataz, phase, project_id, log_table_id, status_table_id, pr status_df.to_gbq(status_table_id, project_id=project_id, if_exists='replace') if phase == 'production': status_df.to_gbq(prod_status_table_id, project_id=project_id, if_exists=if_exists_prod) + + +def write_user_table(table_name, user_table, credentials): + user_data_frame = pd.DataFrame(data=user_table) + user_data_frame.to_gbq(table_name, if_exists='replace', credentials=credentials) + + +def read_user_table(table_name, credentials): + query = 'select user_id, user_name, channel_id, channel_name from {}'.format(table_name) + print(pd.read_gbq(query=query, credentials=credentials)) From e577a43b97f3a0bd537be5e3f172b7ed72ec6608 Mon Sep 17 00:00:00 2001 From: KimJiHoon Date: Sun, 15 Mar 2020 02:26:22 +0900 Subject: [PATCH 13/13] =?UTF-8?q?[add]=20=EB=B9=85=EC=BF=BC=EB=A6=AC=20?= =?UTF-8?q?=ED=86=B5=EC=8B=A0=EC=9D=84=20=EC=9C=84=ED=95=9C=20credentials?= =?UTF-8?q?=20=EC=A0=84=EC=97=AD=EB=B3=80=EC=88=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/extract_data.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/extract_data.py b/common/extract_data.py index 83d9185..5e863ad 100644 --- a/common/extract_data.py +++ b/common/extract_data.py @@ -2,11 +2,13 @@ import pandas as pd from datetime import datetime, timedelta from utils import bigquery_config, phase, root_path, cardinal +from google.oauth2 import service_account _project_id = bigquery_config[phase]['project'] _suffix = bigquery_config[phase]['suffix'] _jwt = os.path.join(root_path, 'config', bigquery_config[phase]['jwt']) +_credentials = service_account.Credentials.from_service_account_file(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) def get_deadline_data(abs_output_directory): @@ -124,11 +126,11 @@ def send_data_to_gbq(dataz, phase, project_id, log_table_id, status_table_id, pr status_df.to_gbq(prod_status_table_id, project_id=project_id, if_exists=if_exists_prod) -def write_user_table(table_name, user_table, credentials): +def write_user_table(table_name, user_table): user_data_frame = pd.DataFrame(data=user_table) - user_data_frame.to_gbq(table_name, if_exists='replace', credentials=credentials) + user_data_frame.to_gbq(table_name, if_exists='replace', credentials=_credentials) -def read_user_table(table_name, credentials): +def read_user_table(table_name): query = 'select user_id, user_name, channel_id, channel_name from {}'.format(table_name) - print(pd.read_gbq(query=query, credentials=credentials)) + print(pd.read_gbq(query=query, credentials=_credentials))