diff --git a/README.md b/README.md index 296f697..9eeb97b 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,21 @@ source env/bin/activate pip3 install -r requirements.txt ``` -### 2. Slack Token 환경 변수 지정 -- 해당 슬랙 workspace의 API를 접근할 수 있는 토큰을 받아와야 합니다. -- [이곳](https://api.slack.com/legacy/custom-integrations/legacy-tokens)에서 슬랙 토큰을 생성받습니다. -- 생성 받은 토큰을 `.bash_profile`, `.zshrc` 등의 파일에 다음과 같이 환경 변수로 저장해줍니다. +### 2. Slack API +1. workspace admin 권한을 받습니다. +2. 새로운 글또 workspace 에 slack app 을 install 합니다. + + [Slack App 페이지](https://api.slack.com/apps) -> [Create New App] +3. Your apps -> 글또 앱 -> OAuth & Permissions 탭에서 생성된 API 토큰을 확인할 수 있습니다. +4. 해당 토큰을 `.bash_profile`, `.zshrc` 등의 파일에 다음과 같이 환경 변수로 저장해줍니다. ``` export SLACK_TOKEN='xoxo-your-token' ``` +* Collaborator 권한이 있어야 app 관리 대시보드로 접근이 가능하기 때문에 Collaborators + 탭에서 관련 담당자분들을 미리 추가해주시면 좋습니다. + + ### 3. 마감 날짜 데이터, 유저 데이터 저장 - 코드 실행 시 활용되는 유저 데이터와 마감 날짜 데이터를 다음과 같이 `outputs` 디렉토리 아래에 저장해주세요. diff --git a/common/extract_data.py b/common/extract_data.py index 2c69d3f..5e863ad 100644 --- a/common/extract_data.py +++ b/common/extract_data.py @@ -2,11 +2,13 @@ import pandas as pd from datetime import datetime, timedelta from utils import bigquery_config, phase, root_path, cardinal +from google.oauth2 import service_account _project_id = bigquery_config[phase]['project'] _suffix = bigquery_config[phase]['suffix'] _jwt = os.path.join(root_path, 'config', bigquery_config[phase]['jwt']) +_credentials = service_account.Credentials.from_service_account_file(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) def get_deadline_data(abs_output_directory): @@ -122,3 +124,13 @@ def send_data_to_gbq(dataz, phase, project_id, log_table_id, status_table_id, pr status_df.to_gbq(status_table_id, project_id=project_id, if_exists='replace') if phase == 'production': status_df.to_gbq(prod_status_table_id, project_id=project_id, if_exists=if_exists_prod) + + +def write_user_table(table_name, user_table): + user_data_frame = pd.DataFrame(data=user_table) + user_data_frame.to_gbq(table_name, if_exists='replace', credentials=_credentials) + + +def read_user_table(table_name): + query = 'select user_id, user_name, channel_id, channel_name from {}'.format(table_name) + print(pd.read_gbq(query=query, credentials=_credentials)) diff --git a/common/slack_export.py b/common/slack_export.py index 12acfbc..64ac92d 100644 --- a/common/slack_export.py +++ b/common/slack_export.py @@ -4,11 +4,13 @@ from datetime import datetime from pick import pick from time import sleep +from collections import defaultdict user_names_by_id = {} user_ids_by_name = {} dry_run = None +user_name_key = 'user_name' def get_history(pageable_object, channel_id, page_size=100): @@ -26,8 +28,8 @@ def get_history(pageable_object, channel_id, page_size=100): messages.extend(response['messages']) if response['has_more']: - last_timestamp = messages[-1]['ts'] # -1 means last element in a list - sleep(1) # Respect the Slack API rate limit + last_timestamp = messages[-1]['ts'] # -1 means last element in a list + sleep(1) # Respect the Slack API rate limit else: break return messages @@ -47,7 +49,7 @@ def parse_time_stamp(time_stamp): if len(t_list) != 2: raise ValueError('Invalid time stamp') else: - return datetime.utcfromtimestamp( float(t_list[0])) + return datetime.utcfromtimestamp(float(t_list[0])) def channel_rename(old_room_name, new_room_name): @@ -59,7 +61,7 @@ def channel_rename(old_room_name, new_room_name): return mkdir(new_room_name) for file_name in os.listdir(old_room_name): - shutil.move( os.path.join(old_room_name, file_name), new_room_name) + shutil.move(os.path.join(old_room_name, file_name), new_room_name) os.rmdir(old_room_name) @@ -102,8 +104,8 @@ def parse_messages(room_dir, messages, room_type): channel_rename(old_room_path, new_room_path) current_messages.append(message) - out_file_name = '{room}/{file}.json'.format(room=room_dir, file=current_file_date ) - write_message_file( out_file_name, current_messages) + out_file_name = '{room}/{file}.json'.format(room=room_dir, file=current_file_date) + write_message_file(out_file_name, current_messages) def filter_conversations_by_name(channels_or_groups, channel_or_group_names): @@ -166,7 +168,7 @@ def dump_user_file(users): write to user file, any existing file needs to be overwritten. """ with open(os.path.join('../', 'users.json'), 'w') as userFile: - json.dump( users, userFile, indent=4 ) + json.dump(users, userFile, indent=4) def bootstrap_key_values(slack_obj, _dry_run): @@ -216,3 +218,54 @@ def finalize(zip_name, output_directory): if zip_name: shutil.make_archive(zip_name, 'zip', output_directory, None) shutil.rmtree(output_directory) + + +# channel_prefix : prefix of the posting channel +def get_user_table(slacker, channel_prefix): + users = slacker.users.list().body['members'] + channels = slacker.channels.list().body['channels'] + + user_names_with_id = get_user_names_with_id(users) + user_names_with_channel = get_user_names_with_channel(channels, channel_prefix) + + if len(user_names_with_id) == len(user_names_with_channel): + user_table = defaultdict(dict) + + for merged_list in (user_names_with_id, user_names_with_channel): + for element in merged_list: + user_table[element[user_name_key]].update(element) + + # user_table : key = '홍길동', value = {'name': '홍길동', 'id': 'UTHXXXXX', 'channel_name': 'prefix_직군_게시글'} 인 dict. + return user_table.values() + else: + print("Fail to get user table. cause, length do not match.") + + +def get_user_names_with_channel(channels, channel_prefix): + users_with_channel = [] + + for channel in channels: + channel_id = channel['id'] + channel_name = channel['name'] + + if channel_name.startswith(channel_prefix): + member_names = channel['topic']['value'].split() + + for name in member_names: + users_with_channel.append({user_name_key: name, 'channel_id': channel_id, 'channel_name': channel_name}) + + return users_with_channel + + +def get_user_names_with_id(users): + users_with_id = [] + + for user in users: + if is_valid_user(user): + users_with_id.append({'user_id': user['id'], user_name_key: user['real_name']}) + + return users_with_id + + +def is_valid_user(user): + return not user['is_bot'] and not user['deleted'] and user['real_name'] != 'Slackbot'