CSE is excluded from scope

.gitignore

@@ -5,5 +5,3 @@ __pycache__/
 *.pyd
 *.swp
 *.swo
-config/
-scripts/

scripts/eod_1_otc_services_dict.py

@@ -195,9 +195,9 @@ def insert_docs_data(item, conn_csv, cur_csv, table_name):
 def add_obsolete_services(conn_csv, cur_csv):
     data_to_insert = [
         {"service_uri": "content-delivery-network", "service_title": "Content Delivery Network", "service_category":
-            "Other", "service_type": "cdn", "squad": "Other"},
+            "Other", "service_type": "cdn", "squad": "Other", "environment": "hidden"},
         {"service_uri": "data-admin-service", "service_title": "Data Admin Service", "service_category": "Other",
-         "service_type": "das", "squad": "Other"}
+         "service_type": "das", "squad": "Other", "environment": "hidden"}
     ]
 
     for item in data_to_insert:

scripts/eod_6_last_commit_info.py

@@ -9,6 +9,7 @@
 
 import psycopg2
 from github import Github
+from github.GithubException import GithubException
 
 from config import Database, EnvVariables, Timer, setup_logging
 
@@ -50,45 +51,58 @@ def get_last_commit_url(github_repo, path):
     return None, None
 
 
-def get_last_commit(org, conn, cur, doctype, string, table_name):
+def get_last_commit(org, conn, cur, doctype, string, table_name, rtc):
     logging.info("Gathering last commit info for %s...", string)
-    exclude_repos = ["docsportal", "doc-exports", "docs_on_docs", ".github", "presentations", "sandbox", "security",
-                     "template", "content-delivery-network", "data-admin-service", "resource-template-service"]
-    for repo in org.get_repos():
 
+    try:
+        cur.execute(f"SELECT DISTINCT \"Repository\" FROM {rtc} WHERE \"Env\" NOT IN ('public');")
+        exclude_repos = [row[0] for row in cur.fetchall()]
+    except Exception as e:
+        logging.error("Fetching public repos: %s", e)
+        return
+
+    for repo in org.get_repos():
         if repo.name in exclude_repos:
             continue
 
         tmp_dir = tempfile.mkdtemp()
 
         try:
-
             path = doctype
             last_commit_url, last_commit_date = get_last_commit_url(repo, path)
-            if last_commit_url and last_commit_date:
-                last_commit_url, _ = get_last_commit_url(repo, path)
-                formatted_commit_date = last_commit_date.strftime('%Y-%m-%d')
-                now = datetime.utcnow()
-                duration = now - last_commit_date
-                duration_days = duration.days
-                if doctype == "umn/source":
-                    doc_type = "UMN"
-                else:
-                    doc_type = "API"
-                service_name = repo.name
-                cur.execute(
-                    f'INSERT INTO {table_name} ("Service Name", "Doc Type", "Last commit at", "Days passed", '
-                    f'"Commit URL") VALUES (%s, %s, %s, %s, %s);',
-                    (service_name, doc_type, formatted_commit_date, duration_days, last_commit_url,))
-                conn.commit()
+            if not last_commit_url or not last_commit_date:
+                logging.info("No commits found for %s, skipping.", repo.name)
+                continue
+
+            formatted_commit_date = last_commit_date.strftime('%Y-%m-%d')
+            now = datetime.utcnow()
+            duration_days = (now - last_commit_date).days
+
+            doc_type = "UMN" if doctype == "umn/source" else "API"
+            service_name = repo.name
+
+            cur.execute(
+                f'INSERT INTO {table_name} ("Service Name", "Doc Type", "Last commit at", "Days passed", "Commit URL") '
+                f'VALUES (%s, %s, %s, %s, %s);',
+                (service_name, doc_type, formatted_commit_date, duration_days, last_commit_url)
+            )
+
+            conn.commit()
+
+        except GithubException as e:
+            if e.status == 409:
+                logging.warning("Empty repo, skipping: %s", repo.name)
+            else:
+                logging.error("Last commit: an error occurred while processing repo %s: %s", repo.name, str(e))
 
         except Exception as e:
-            logging.error("Last commit: an error occurred while processing repo %s: %s", repo.name, str(e))
+            logging.error("Unexpected error processing repo %s: %s", repo.name, str(e))
 
         finally:
             shutil.rmtree(tmp_dir)
 
 
+
 def update_squad_and_title(conn, cur, table_name, rtc):
     logging.info("Updating squads and titles...")
     try:
@@ -118,6 +132,14 @@ def update_squad_and_title(conn, cur, table_name, rtc):
         conn.rollback()
 
 
+def delete_non_public_repos(conn, cur, table_name):
+    cur.execute(
+        f'DELETE FROM {table_name} WHERE "Squad" IS NULL;'
+    )
+
+    conn.commit()
+
+
 def main(gorg, table_name, rtc, gh_str, token):
     g = Github(token)
     org = g.get_organization(gorg)
@@ -126,10 +148,11 @@ def main(gorg, table_name, rtc, gh_str, token):
     cur_csv.execute(f"DROP TABLE IF EXISTS {table_name}")
     create_commits_table(conn_csv, cur_csv, table_name)
     logging.info("Searching for a most recent commit in umn/source...")
-    get_last_commit(org, conn_csv, cur_csv, "umn/source", gh_str, table_name)
+    get_last_commit(org, conn_csv, cur_csv, "umn/source", gh_str, table_name, rtc)
     logging.info("Searching for a most recent commit in api-ref/source...")
-    get_last_commit(org, conn_csv, cur_csv, "api-ref/source", gh_str, table_name)
+    get_last_commit(org, conn_csv, cur_csv, "api-ref/source", gh_str, table_name, rtc)
     update_squad_and_title(conn_csv, cur_csv, table_name, rtc)
+    delete_non_public_repos(conn_csv, cur_csv, table_name)
     conn_csv.commit()