diff --git a/.gitignore b/.gitignore index 4f18bd9..e87c0ec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,3 @@ __pycache__/ *.pyd *.swp *.swo -config/ -scripts/ diff --git a/scripts/eod_1_otc_services_dict.py b/scripts/eod_1_otc_services_dict.py index f714cbc..b6a8ae0 100644 --- a/scripts/eod_1_otc_services_dict.py +++ b/scripts/eod_1_otc_services_dict.py @@ -195,9 +195,9 @@ def insert_docs_data(item, conn_csv, cur_csv, table_name): def add_obsolete_services(conn_csv, cur_csv): data_to_insert = [ {"service_uri": "content-delivery-network", "service_title": "Content Delivery Network", "service_category": - "Other", "service_type": "cdn", "squad": "Other"}, + "Other", "service_type": "cdn", "squad": "Other", "environment": "hidden"}, {"service_uri": "data-admin-service", "service_title": "Data Admin Service", "service_category": "Other", - "service_type": "das", "squad": "Other"} + "service_type": "das", "squad": "Other", "environment": "hidden"} ] for item in data_to_insert: diff --git a/scripts/eod_6_last_commit_info.py b/scripts/eod_6_last_commit_info.py index afb65b0..f2636b7 100644 --- a/scripts/eod_6_last_commit_info.py +++ b/scripts/eod_6_last_commit_info.py @@ -9,6 +9,7 @@ import psycopg2 from github import Github +from github.GithubException import GithubException from config import Database, EnvVariables, Timer, setup_logging @@ -50,45 +51,58 @@ def get_last_commit_url(github_repo, path): return None, None -def get_last_commit(org, conn, cur, doctype, string, table_name): +def get_last_commit(org, conn, cur, doctype, string, table_name, rtc): logging.info("Gathering last commit info for %s...", string) - exclude_repos = ["docsportal", "doc-exports", "docs_on_docs", ".github", "presentations", "sandbox", "security", - "template", "content-delivery-network", "data-admin-service", "resource-template-service"] - for repo in org.get_repos(): + try: + cur.execute(f"SELECT DISTINCT \"Repository\" FROM {rtc} WHERE \"Env\" NOT IN ('public');") + exclude_repos = [row[0] for row in cur.fetchall()] + except Exception as e: + logging.error("Fetching public repos: %s", e) + return + + for repo in org.get_repos(): if repo.name in exclude_repos: continue tmp_dir = tempfile.mkdtemp() try: - path = doctype last_commit_url, last_commit_date = get_last_commit_url(repo, path) - if last_commit_url and last_commit_date: - last_commit_url, _ = get_last_commit_url(repo, path) - formatted_commit_date = last_commit_date.strftime('%Y-%m-%d') - now = datetime.utcnow() - duration = now - last_commit_date - duration_days = duration.days - if doctype == "umn/source": - doc_type = "UMN" - else: - doc_type = "API" - service_name = repo.name - cur.execute( - f'INSERT INTO {table_name} ("Service Name", "Doc Type", "Last commit at", "Days passed", ' - f'"Commit URL") VALUES (%s, %s, %s, %s, %s);', - (service_name, doc_type, formatted_commit_date, duration_days, last_commit_url,)) - conn.commit() + if not last_commit_url or not last_commit_date: + logging.info("No commits found for %s, skipping.", repo.name) + continue + + formatted_commit_date = last_commit_date.strftime('%Y-%m-%d') + now = datetime.utcnow() + duration_days = (now - last_commit_date).days + + doc_type = "UMN" if doctype == "umn/source" else "API" + service_name = repo.name + + cur.execute( + f'INSERT INTO {table_name} ("Service Name", "Doc Type", "Last commit at", "Days passed", "Commit URL") ' + f'VALUES (%s, %s, %s, %s, %s);', + (service_name, doc_type, formatted_commit_date, duration_days, last_commit_url) + ) + + conn.commit() + + except GithubException as e: + if e.status == 409: + logging.warning("Empty repo, skipping: %s", repo.name) + else: + logging.error("Last commit: an error occurred while processing repo %s: %s", repo.name, str(e)) except Exception as e: - logging.error("Last commit: an error occurred while processing repo %s: %s", repo.name, str(e)) + logging.error("Unexpected error processing repo %s: %s", repo.name, str(e)) finally: shutil.rmtree(tmp_dir) + def update_squad_and_title(conn, cur, table_name, rtc): logging.info("Updating squads and titles...") try: @@ -118,6 +132,14 @@ def update_squad_and_title(conn, cur, table_name, rtc): conn.rollback() +def delete_non_public_repos(conn, cur, table_name): + cur.execute( + f'DELETE FROM {table_name} WHERE "Squad" IS NULL;' + ) + + conn.commit() + + def main(gorg, table_name, rtc, gh_str, token): g = Github(token) org = g.get_organization(gorg) @@ -126,10 +148,11 @@ def main(gorg, table_name, rtc, gh_str, token): cur_csv.execute(f"DROP TABLE IF EXISTS {table_name}") create_commits_table(conn_csv, cur_csv, table_name) logging.info("Searching for a most recent commit in umn/source...") - get_last_commit(org, conn_csv, cur_csv, "umn/source", gh_str, table_name) + get_last_commit(org, conn_csv, cur_csv, "umn/source", gh_str, table_name, rtc) logging.info("Searching for a most recent commit in api-ref/source...") - get_last_commit(org, conn_csv, cur_csv, "api-ref/source", gh_str, table_name) + get_last_commit(org, conn_csv, cur_csv, "api-ref/source", gh_str, table_name, rtc) update_squad_and_title(conn_csv, cur_csv, table_name, rtc) + delete_non_public_repos(conn_csv, cur_csv, table_name) conn_csv.commit()