From 621cae26ca81e5db34edb54ce83d29652ce4272f Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Sat, 14 Dec 2024 19:38:20 -0500
Subject: [PATCH 01/18] start rework

---
 app/database.py    |  4 ++--
 app/github.py      |  8 ++++----
 app/github_rest.py | 10 +++++-----
 app/main.py        |  4 ++--
 app/slacker.py     |  2 +-
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/app/database.py b/app/database.py
index 4760f73..04890e0 100644
--- a/app/database.py
+++ b/app/database.py
@@ -637,8 +637,8 @@ def change_users_project_status(project_name: str, user_github: str, status: sta
 if __name__ == "__main__":
     # for table in ['user', 'project', 'semester', 'user_project', 'csv']:
     #     dump(table)
-    ingest()
-    # projects()
+    #ingest()
+    print(len(projects()))
     # information()
     # get_users_in_project('Byte')
     # print(process())
diff --git a/app/github.py b/app/github.py
index be61610..61d4cfb 100644
--- a/app/github.py
+++ b/app/github.py
@@ -236,8 +236,7 @@ def reinvite_expired_users_on_repo(self, repo_url: str) -> list[tuple[int, str]]
             return results
         except Exception as e:
             return [(500, str(e))]             
-                                                    
-        
+                                                
     def change_user_permission_on_repo(self, repo_url: str, user: str, permission: perms) -> tuple[int, str]:
         """
         Changes the permission level of a user on a GitHub repository.
@@ -349,7 +348,8 @@ def get_all_repos(self) -> list[str]:
             return []
 
 if __name__ == "__main__":
-    github = Github(GITHUB_PAT, "spark-tests")
+    #github = Github(GITHUB_PAT, "spark-tests")
     #print(github.change_user_permission_on_repo("https://github.com/spark-tests/initial", "mochiakku", "push"))
     #print(github.change_all_user_permission_on_repo("https://github.com/spark-tests/initial", "push"))
-    print(github.get_all_repos())
\ No newline at end of file
+    #print(github.get_all_repos())
+    print()
\ No newline at end of file
diff --git a/app/github_rest.py b/app/github_rest.py
index 3f8efcd..d09e6c0 100644
--- a/app/github_rest.py
+++ b/app/github_rest.py
@@ -671,8 +671,8 @@ def set_all_repos_users_read_only(self):
 
 def test():
     load_dotenv()
-    GITHUB_PAT = os.getenv('GITHUB_PAT')
-    automation = Automation(GITHUB_PAT, 'spark-tests')
+    SPARK_GITHUB_PAT = os.getenv('SPARK_GITHUB_PAT')
+    automation = Automation(SPARK_GITHUB_PAT, 'BU-Spark')
     print(automation.GITHUB_PAT)
     
     #inital_ssh_url = automation.get_repository_ssh_url('initial')
@@ -688,10 +688,10 @@ def test():
     #print("--")
     #print(invited_to_byte)
     #print(automation.remove_or_revoke_user(byte_ssh_url, ''))
-    #invited = automation.reinvite_all_expired_users_to_repos()
-    #print(invited)
+    invited = automation.reinvite_all_expired_users_to_repos()
+    print(invited)
     
-    automation.set_all_repos_users_read_only()
+    #automation.set_all_repos_users_read_only()
     
 if __name__ == "__main__":
     test()
\ No newline at end of file
diff --git a/app/main.py b/app/main.py
index aa01bfb..5b2f596 100644
--- a/app/main.py
+++ b/app/main.py
@@ -81,8 +81,8 @@ async def refresh():
 @app.post("/reinvite_expired_collaborators")
 async def reinvite_expired_collaborators(request: Request):
     try:
-        r = automation.reinvite_all_expired_users_to_repos()
-        print(r)
+        r = [(project["github_url"], github.reinvite_expired_users_on_repo(project["github_url"])) 
+            for project in db.projects()]
         return {"status": r}
     except Exception as e: return {"status": "failed", "error": str(e)}
     
diff --git a/app/slacker.py b/app/slacker.py
index 86af61d..b1f6ad7 100644
--- a/app/slacker.py
+++ b/app/slacker.py
@@ -121,6 +121,6 @@ def create_channels_and_add_users(self, channels_dict: dict, is_private: bool =
 if __name__ == "__main__":
     slacker =   Slacker(token=os.getenv('SLACK_BOT_TOKEN'))
     channels_dict = {
-        'x4': ["x@bu.edu"],
+        'x4': ["x   @bu.edu"],
     }
     created_channels = slacker.create_channels_and_add_users(channels_dict, is_private=False)
\ No newline at end of file

From b2550d032f26617c9f294fa3ae6c3601482a0f97 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Sun, 15 Dec 2024 02:16:32 -0500
Subject: [PATCH 02/18] many backend changes

---
 Pipfile                     |   3 +
 Pipfile.lock                | 618 +++++++++++++++++++++++---------
 app/database.py             |   4 +-
 app/github.py               |   9 +-
 app/github_rest.py          | 697 ------------------------------------
 app/main.py                 |  32 +-
 app/models.py               | 220 +++++++-----
 app/schema.py               | 119 +-----
 app/slacker.py              |  14 +-
 app/spark.py                | 137 +++++++
 scripts/README.md           |  44 ---
 scripts/add-archive-tag.sh  |   7 -
 scripts/readme.md           |  21 --
 scripts/script.py           | 225 ------------
 scripts/update-workflows.sh |  28 --
 15 files changed, 752 insertions(+), 1426 deletions(-)
 delete mode 100644 app/github_rest.py
 create mode 100644 app/spark.py
 delete mode 100644 scripts/README.md
 delete mode 100644 scripts/add-archive-tag.sh
 delete mode 100644 scripts/readme.md
 delete mode 100644 scripts/script.py
 delete mode 100644 scripts/update-workflows.sh

diff --git a/Pipfile b/Pipfile
index a90c84a..f80759c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -14,6 +14,9 @@ python-multipart = "*"
 pandas = "*"
 aiocache = {extras = ["redis", "memcached"], version = "*"}
 slack-sdk = "*"
+sqlmodel = "*"
+pyright = "*"
+mypy = "*"
 
 [dev-packages]
 
diff --git a/Pipfile.lock b/Pipfile.lock
index 9f51c4f..9bca9ee 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "52b401d1f73663754297634b7d28d5d8fc24e63828cfb2f5746e6ad6650953b1"
+            "sha256": "7f7e404f5eda7dd8c781ccc1a8ed37eaaf6cce24cb3caa7c2813e1c3544f2e15"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -44,11 +44,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c",
-                "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"
+                "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48",
+                "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==4.6.2.post1"
+            "version": "==4.7.0"
         },
         "async-timeout": {
             "hashes": [
@@ -60,11 +60,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8",
-                "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"
+                "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56",
+                "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2024.8.30"
+            "version": "==2024.12.14"
         },
         "charset-normalizer": {
             "hashes": [
@@ -195,12 +195,12 @@
         },
         "fastapi": {
             "hashes": [
-                "sha256:0b504a063ffb3cf96a5e27dc1bc32c80ca743a2528574f9cdc77daa2d31b4742",
-                "sha256:db653475586b091cb8b2fec2ac54a680ac6a158e07406e1abae31679e8826349"
+                "sha256:9ec46f7addc14ea472958a96aae5b5de65f39721a46aaf5705c480d9a8b76654",
+                "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.115.4"
+            "version": "==0.115.6"
         },
         "gitdb": {
             "hashes": [
@@ -219,6 +219,85 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.1.43"
         },
+        "greenlet": {
+            "hashes": [
+                "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e",
+                "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7",
+                "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01",
+                "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1",
+                "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159",
+                "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563",
+                "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83",
+                "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9",
+                "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395",
+                "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa",
+                "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942",
+                "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1",
+                "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441",
+                "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22",
+                "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9",
+                "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0",
+                "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba",
+                "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3",
+                "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1",
+                "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6",
+                "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291",
+                "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39",
+                "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d",
+                "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467",
+                "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475",
+                "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef",
+                "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c",
+                "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511",
+                "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c",
+                "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822",
+                "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a",
+                "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8",
+                "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d",
+                "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01",
+                "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145",
+                "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80",
+                "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13",
+                "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e",
+                "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b",
+                "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1",
+                "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef",
+                "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc",
+                "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff",
+                "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120",
+                "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437",
+                "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd",
+                "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981",
+                "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36",
+                "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a",
+                "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798",
+                "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7",
+                "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761",
+                "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0",
+                "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e",
+                "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af",
+                "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa",
+                "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c",
+                "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42",
+                "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e",
+                "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81",
+                "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e",
+                "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617",
+                "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc",
+                "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de",
+                "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111",
+                "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383",
+                "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70",
+                "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6",
+                "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4",
+                "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011",
+                "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803",
+                "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79",
+                "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"
+            ],
+            "markers": "python_version < '3.13' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
+            "version": "==3.1.1"
+        },
         "h11": {
             "hashes": [
                 "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d",
@@ -235,66 +314,121 @@
             "markers": "python_version >= '3.6'",
             "version": "==3.10"
         },
+        "mypy": {
+            "hashes": [
+                "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc",
+                "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e",
+                "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f",
+                "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74",
+                "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a",
+                "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2",
+                "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b",
+                "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73",
+                "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e",
+                "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d",
+                "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d",
+                "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6",
+                "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca",
+                "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d",
+                "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5",
+                "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62",
+                "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a",
+                "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc",
+                "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7",
+                "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb",
+                "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7",
+                "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732",
+                "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80",
+                "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a",
+                "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc",
+                "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2",
+                "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0",
+                "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24",
+                "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7",
+                "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b",
+                "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372",
+                "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==1.13.0"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d",
+                "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.0.0"
+        },
+        "nodeenv": {
+            "hashes": [
+                "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f",
+                "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+            "version": "==1.9.1"
+        },
         "numpy": {
             "hashes": [
-                "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe",
-                "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0",
-                "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48",
-                "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a",
-                "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564",
-                "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958",
-                "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17",
-                "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0",
-                "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee",
-                "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b",
-                "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4",
-                "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4",
-                "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6",
-                "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4",
-                "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d",
-                "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f",
-                "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f",
-                "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f",
-                "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56",
-                "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9",
-                "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd",
-                "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23",
-                "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed",
-                "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a",
-                "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098",
-                "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1",
-                "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512",
-                "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f",
-                "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09",
-                "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f",
-                "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc",
-                "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8",
-                "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0",
-                "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761",
-                "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef",
-                "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5",
-                "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e",
-                "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b",
-                "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d",
-                "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43",
-                "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c",
-                "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41",
-                "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff",
-                "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408",
-                "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2",
-                "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9",
-                "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57",
-                "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb",
-                "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9",
-                "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3",
-                "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a",
-                "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0",
-                "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e",
-                "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598",
-                "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4"
+                "sha256:0557eebc699c1c34cccdd8c3778c9294e8196df27d713706895edc6f57d29608",
+                "sha256:0798b138c291d792f8ea40fe3768610f3c7dd2574389e37c3f26573757c8f7ef",
+                "sha256:0da8495970f6b101ddd0c38ace92edea30e7e12b9a926b57f5fabb1ecc25bb90",
+                "sha256:0f0986e917aca18f7a567b812ef7ca9391288e2acb7a4308aa9d265bd724bdae",
+                "sha256:122fd2fcfafdefc889c64ad99c228d5a1f9692c3a83f56c292618a59aa60ae83",
+                "sha256:140dd80ff8981a583a60980be1a655068f8adebf7a45a06a6858c873fcdcd4a0",
+                "sha256:16757cf28621e43e252c560d25b15f18a2f11da94fea344bf26c599b9cf54b73",
+                "sha256:18142b497d70a34b01642b9feabb70156311b326fdddd875a9981f34a369b671",
+                "sha256:1c92113619f7b272838b8d6702a7f8ebe5edea0df48166c47929611d0b4dea69",
+                "sha256:1e25507d85da11ff5066269d0bd25d06e0a0f2e908415534f3e603d2a78e4ffa",
+                "sha256:30bf971c12e4365153afb31fc73f441d4da157153f3400b82db32d04de1e4066",
+                "sha256:3579eaeb5e07f3ded59298ce22b65f877a86ba8e9fe701f5576c99bb17c283da",
+                "sha256:36b2b43146f646642b425dd2027730f99bac962618ec2052932157e213a040e9",
+                "sha256:3905a5fffcc23e597ee4d9fb3fcd209bd658c352657548db7316e810ca80458e",
+                "sha256:3a4199f519e57d517ebd48cb76b36c82da0360781c6a0353e64c0cac30ecaad3",
+                "sha256:3f2f5cddeaa4424a0a118924b988746db6ffa8565e5829b1841a8a3bd73eb59a",
+                "sha256:40deb10198bbaa531509aad0cd2f9fadb26c8b94070831e2208e7df543562b74",
+                "sha256:440cfb3db4c5029775803794f8638fbdbf71ec702caf32735f53b008e1eaece3",
+                "sha256:4723a50e1523e1de4fccd1b9a6dcea750c2102461e9a02b2ac55ffeae09a4410",
+                "sha256:4bddbaa30d78c86329b26bd6aaaea06b1e47444da99eddac7bf1e2fab717bd72",
+                "sha256:4e58666988605e251d42c2818c7d3d8991555381be26399303053b58a5bbf30d",
+                "sha256:54dc1d6d66f8d37843ed281773c7174f03bf7ad826523f73435deb88ba60d2d4",
+                "sha256:57fcc997ffc0bef234b8875a54d4058afa92b0b0c4223fc1f62f24b3b5e86038",
+                "sha256:58b92a5828bd4d9aa0952492b7de803135038de47343b2aa3cc23f3b71a3dc4e",
+                "sha256:5a145e956b374e72ad1dff82779177d4a3c62bc8248f41b80cb5122e68f22d13",
+                "sha256:6ab153263a7c5ccaf6dfe7e53447b74f77789f28ecb278c3b5d49db7ece10d6d",
+                "sha256:7832f9e8eb00be32f15fdfb9a981d6955ea9adc8574c521d48710171b6c55e95",
+                "sha256:7fe4bb0695fe986a9e4deec3b6857003b4cfe5c5e4aac0b95f6a658c14635e31",
+                "sha256:7fe8f3583e0607ad4e43a954e35c1748b553bfe9fdac8635c02058023277d1b3",
+                "sha256:85ad7d11b309bd132d74397fcf2920933c9d1dc865487128f5c03d580f2c3d03",
+                "sha256:9874bc2ff574c40ab7a5cbb7464bf9b045d617e36754a7bc93f933d52bd9ffc6",
+                "sha256:a184288538e6ad699cbe6b24859206e38ce5fba28f3bcfa51c90d0502c1582b2",
+                "sha256:a222d764352c773aa5ebde02dd84dba3279c81c6db2e482d62a3fa54e5ece69b",
+                "sha256:a50aeff71d0f97b6450d33940c7181b08be1441c6c193e678211bff11aa725e7",
+                "sha256:a55dc7a7f0b6198b07ec0cd445fbb98b05234e8b00c5ac4874a63372ba98d4ab",
+                "sha256:a62eb442011776e4036af5c8b1a00b706c5bc02dc15eb5344b0c750428c94219",
+                "sha256:a7d41d1612c1a82b64697e894b75db6758d4f21c3ec069d841e60ebe54b5b571",
+                "sha256:a98f6f20465e7618c83252c02041517bd2f7ea29be5378f09667a8f654a5918d",
+                "sha256:afe8fb968743d40435c3827632fd36c5fbde633b0423da7692e426529b1759b1",
+                "sha256:b0b227dcff8cdc3efbce66d4e50891f04d0a387cce282fe1e66199146a6a8fca",
+                "sha256:b30042fe92dbd79f1ba7f6898fada10bdaad1847c44f2dff9a16147e00a93661",
+                "sha256:b606b1aaf802e6468c2608c65ff7ece53eae1a6874b3765f69b8ceb20c5fa78e",
+                "sha256:b6207dc8fb3c8cb5668e885cef9ec7f70189bec4e276f0ff70d5aa078d32c88e",
+                "sha256:c2aed8fcf8abc3020d6a9ccb31dbc9e7d7819c56a348cc88fd44be269b37427e",
+                "sha256:cb24cca1968b21355cc6f3da1a20cd1cebd8a023e3c5b09b432444617949085a",
+                "sha256:cff210198bb4cae3f3c100444c5eaa573a823f05c253e7188e1362a5555235b3",
+                "sha256:d35717333b39d1b6bb8433fa758a55f1081543de527171543a2b710551d40881",
+                "sha256:df12a1f99b99f569a7c2ae59aa2d31724e8d835fc7f33e14f4792e3071d11221",
+                "sha256:e09d40edfdb4e260cb1567d8ae770ccf3b8b7e9f0d9b5c2a9992696b30ce2742",
+                "sha256:e12c6c1ce84628c52d6367863773f7c8c8241be554e8b79686e91a43f1733773",
+                "sha256:e2b8cd48a9942ed3f85b95ca4105c45758438c7ed28fff1e4ce3e57c3b589d8e",
+                "sha256:e500aba968a48e9019e42c0c199b7ec0696a97fa69037bea163b55398e390529",
+                "sha256:ebe5e59545401fbb1b24da76f006ab19734ae71e703cdb4a8b347e84a0cece67",
+                "sha256:f0dd071b95bbca244f4cb7f70b77d2ff3aaaba7fa16dc41f58d14854a6204e6c",
+                "sha256:f8c8b141ef9699ae777c6278b52c706b653bf15d135d302754f6b2e90eb30367"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==2.1.3"
+            "version": "==2.2.0"
         },
         "pandas": {
             "hashes": [
@@ -421,106 +555,126 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f",
-                "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"
+                "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d",
+                "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.9.2"
+            "version": "==2.10.3"
         },
         "pydantic-core": {
             "hashes": [
-                "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36",
-                "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05",
-                "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071",
-                "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327",
-                "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c",
-                "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36",
-                "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29",
-                "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744",
-                "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d",
-                "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec",
-                "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e",
-                "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e",
-                "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577",
-                "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232",
-                "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863",
-                "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6",
-                "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368",
-                "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480",
-                "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2",
-                "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2",
-                "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6",
-                "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769",
-                "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d",
-                "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2",
-                "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84",
-                "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166",
-                "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271",
-                "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5",
-                "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb",
-                "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13",
-                "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323",
-                "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556",
-                "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665",
-                "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef",
-                "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb",
-                "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119",
-                "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126",
-                "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510",
-                "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b",
-                "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87",
-                "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f",
-                "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc",
-                "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8",
-                "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21",
-                "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f",
-                "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6",
-                "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658",
-                "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b",
-                "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3",
-                "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb",
-                "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59",
-                "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24",
-                "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9",
-                "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3",
-                "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd",
-                "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753",
-                "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55",
-                "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad",
-                "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a",
-                "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605",
-                "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e",
-                "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b",
-                "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433",
-                "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8",
-                "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07",
-                "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728",
-                "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0",
-                "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327",
-                "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555",
-                "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64",
-                "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6",
-                "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea",
-                "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b",
-                "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df",
-                "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e",
-                "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd",
-                "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068",
-                "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3",
-                "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040",
-                "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12",
-                "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916",
-                "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f",
-                "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f",
-                "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801",
-                "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231",
-                "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5",
-                "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8",
-                "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee",
-                "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"
+                "sha256:00e6424f4b26fe82d44577b4c842d7df97c20be6439e8e685d0d715feceb9fb9",
+                "sha256:029d9757eb621cc6e1848fa0b0310310de7301057f623985698ed7ebb014391b",
+                "sha256:02a3d637bd387c41d46b002f0e49c52642281edacd2740e5a42f7017feea3f2c",
+                "sha256:0325336f348dbee6550d129b1627cb8f5351a9dc91aad141ffb96d4937bd9529",
+                "sha256:062f60e512fc7fff8b8a9d680ff0ddaaef0193dba9fa83e679c0c5f5fbd018bc",
+                "sha256:0b3dfe500de26c52abe0477dde16192ac39c98f05bf2d80e76102d394bd13854",
+                "sha256:0e4216e64d203e39c62df627aa882f02a2438d18a5f21d7f721621f7a5d3611d",
+                "sha256:121ceb0e822f79163dd4699e4c54f5ad38b157084d97b34de8b232bcaad70278",
+                "sha256:159cac0a3d096f79ab6a44d77a961917219707e2a130739c64d4dd46281f5c2a",
+                "sha256:15aae984e46de8d376df515f00450d1522077254ef6b7ce189b38ecee7c9677c",
+                "sha256:15cc53a3179ba0fcefe1e3ae50beb2784dede4003ad2dfd24f81bba4b23a454f",
+                "sha256:161c27ccce13b6b0c8689418da3885d3220ed2eae2ea5e9b2f7f3d48f1d52c27",
+                "sha256:19910754e4cc9c63bc1c7f6d73aa1cfee82f42007e407c0f413695c2f7ed777f",
+                "sha256:1ba5e3963344ff25fc8c40da90f44b0afca8cfd89d12964feb79ac1411a260ac",
+                "sha256:1c00666a3bd2f84920a4e94434f5974d7bbc57e461318d6bb34ce9cdbbc1f6b2",
+                "sha256:1c39b07d90be6b48968ddc8c19e7585052088fd7ec8d568bb31ff64c70ae3c97",
+                "sha256:206b5cf6f0c513baffaeae7bd817717140770c74528f3e4c3e1cec7871ddd61a",
+                "sha256:258c57abf1188926c774a4c94dd29237e77eda19462e5bb901d88adcab6af919",
+                "sha256:2cdf7d86886bc6982354862204ae3b2f7f96f21a3eb0ba5ca0ac42c7b38598b9",
+                "sha256:2d4567c850905d5eaaed2f7a404e61012a51caf288292e016360aa2b96ff38d4",
+                "sha256:35c14ac45fcfdf7167ca76cc80b2001205a8d5d16d80524e13508371fb8cdd9c",
+                "sha256:38de0a70160dd97540335b7ad3a74571b24f1dc3ed33f815f0880682e6880131",
+                "sha256:3af385b0cee8df3746c3f406f38bcbfdc9041b5c2d5ce3e5fc6637256e60bbc5",
+                "sha256:3b748c44bb9f53031c8cbc99a8a061bc181c1000c60a30f55393b6e9c45cc5bd",
+                "sha256:3bbd5d8cc692616d5ef6fbbbd50dbec142c7e6ad9beb66b78a96e9c16729b089",
+                "sha256:3ccaa88b24eebc0f849ce0a4d09e8a408ec5a94afff395eb69baf868f5183107",
+                "sha256:3fa80ac2bd5856580e242dbc202db873c60a01b20309c8319b5c5986fbe53ce6",
+                "sha256:4228b5b646caa73f119b1ae756216b59cc6e2267201c27d3912b592c5e323b60",
+                "sha256:42b0e23f119b2b456d07ca91b307ae167cc3f6c846a7b169fca5326e32fdc6cf",
+                "sha256:45cf8588c066860b623cd11c4ba687f8d7175d5f7ef65f7129df8a394c502de5",
+                "sha256:45d9c5eb9273aa50999ad6adc6be5e0ecea7e09dbd0d31bd0c65a55a2592ca08",
+                "sha256:4603137322c18eaf2e06a4495f426aa8d8388940f3c457e7548145011bb68e05",
+                "sha256:46ccfe3032b3915586e469d4972973f893c0a2bb65669194a5bdea9bacc088c2",
+                "sha256:4fefee876e07a6e9aad7a8c8c9f85b0cdbe7df52b8a9552307b09050f7512c7e",
+                "sha256:5556470f1a2157031e676f776c2bc20acd34c1990ca5f7e56f1ebf938b9ab57c",
+                "sha256:57866a76e0b3823e0b56692d1a0bf722bffb324839bb5b7226a7dbd6c9a40b17",
+                "sha256:5897bec80a09b4084aee23f9b73a9477a46c3304ad1d2d07acca19723fb1de62",
+                "sha256:58ca98a950171f3151c603aeea9303ef6c235f692fe555e883591103da709b23",
+                "sha256:5ca038c7f6a0afd0b2448941b6ef9d5e1949e999f9e5517692eb6da58e9d44be",
+                "sha256:5f6c8a66741c5f5447e047ab0ba7a1c61d1e95580d64bce852e3df1f895c4067",
+                "sha256:5f8c4718cd44ec1580e180cb739713ecda2bdee1341084c1467802a417fe0f02",
+                "sha256:5fde892e6c697ce3e30c61b239330fc5d569a71fefd4eb6512fc6caec9dd9e2f",
+                "sha256:62a763352879b84aa31058fc931884055fd75089cccbd9d58bb6afd01141b235",
+                "sha256:62ba45e21cf6571d7f716d903b5b7b6d2617e2d5d67c0923dc47b9d41369f840",
+                "sha256:64c65f40b4cd8b0e049a8edde07e38b476da7e3aaebe63287c899d2cff253fa5",
+                "sha256:655d7dd86f26cb15ce8a431036f66ce0318648f8853d709b4167786ec2fa4807",
+                "sha256:66ff044fd0bb1768688aecbe28b6190f6e799349221fb0de0e6f4048eca14c16",
+                "sha256:672ebbe820bb37988c4d136eca2652ee114992d5d41c7e4858cdd90ea94ffe5c",
+                "sha256:6b9af86e1d8e4cfc82c2022bfaa6f459381a50b94a29e95dcdda8442d6d83864",
+                "sha256:6e0bd57539da59a3e4671b90a502da9a28c72322a4f17866ba3ac63a82c4498e",
+                "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a",
+                "sha256:7597c07fbd11515f654d6ece3d0e4e5093edc30a436c63142d9a4b8e22f19c35",
+                "sha256:764be71193f87d460a03f1f7385a82e226639732214b402f9aa61f0d025f0737",
+                "sha256:7699b1df36a48169cdebda7ab5a2bac265204003f153b4bd17276153d997670a",
+                "sha256:7ccebf51efc61634f6c2344da73e366c75e735960b5654b63d7e6f69a5885fa3",
+                "sha256:7f7059ca8d64fea7f238994c97d91f75965216bcbe5f695bb44f354893f11d52",
+                "sha256:8065914ff79f7eab1599bd80406681f0ad08f8e47c880f17b416c9f8f7a26d05",
+                "sha256:816f5aa087094099fff7edabb5e01cc370eb21aa1a1d44fe2d2aefdfb5599b31",
+                "sha256:81f2ec23ddc1b476ff96563f2e8d723830b06dceae348ce02914a37cb4e74b89",
+                "sha256:84286494f6c5d05243456e04223d5a9417d7f443c3b76065e75001beb26f88de",
+                "sha256:8bf7b66ce12a2ac52d16f776b31d16d91033150266eb796967a7e4621707e4f6",
+                "sha256:8f1edcea27918d748c7e5e4d917297b2a0ab80cad10f86631e488b7cddf76a36",
+                "sha256:981fb88516bd1ae8b0cbbd2034678a39dedc98752f264ac9bc5839d3923fa04c",
+                "sha256:98476c98b02c8e9b2eec76ac4156fd006628b1b2d0ef27e548ffa978393fd154",
+                "sha256:992cea5f4f3b29d6b4f7f1726ed8ee46c8331c6b4eed6db5b40134c6fe1768bb",
+                "sha256:9a3b0793b1bbfd4146304e23d90045f2a9b5fd5823aa682665fbdaf2a6c28f3e",
+                "sha256:9a42d6a8156ff78981f8aa56eb6394114e0dedb217cf8b729f438f643608cbcd",
+                "sha256:9c10c309e18e443ddb108f0ef64e8729363adbfd92d6d57beec680f6261556f3",
+                "sha256:9cbd94fc661d2bab2bc702cddd2d3370bbdcc4cd0f8f57488a81bcce90c7a54f",
+                "sha256:9fdcf339322a3fae5cbd504edcefddd5a50d9ee00d968696846f089b4432cf78",
+                "sha256:a0697803ed7d4af5e4c1adf1670af078f8fcab7a86350e969f454daf598c4960",
+                "sha256:a28af0695a45f7060e6f9b7092558a928a28553366519f64083c63a44f70e618",
+                "sha256:a2e02889071850bbfd36b56fd6bc98945e23670773bc7a76657e90e6b6603c08",
+                "sha256:a33cd6ad9017bbeaa9ed78a2e0752c5e250eafb9534f308e7a5f7849b0b1bfb4",
+                "sha256:a3cb37038123447cf0f3ea4c74751f6a9d7afef0eb71aa07bf5f652b5e6a132c",
+                "sha256:a57847b090d7892f123726202b7daa20df6694cbd583b67a592e856bff603d6c",
+                "sha256:a5a8e19d7c707c4cadb8c18f5f60c843052ae83c20fa7d44f41594c644a1d330",
+                "sha256:ac3b20653bdbe160febbea8aa6c079d3df19310d50ac314911ed8cc4eb7f8cb8",
+                "sha256:ac6c2c45c847bbf8f91930d88716a0fb924b51e0c6dad329b793d670ec5db792",
+                "sha256:acc07b2cfc5b835444b44a9956846b578d27beeacd4b52e45489e93276241025",
+                "sha256:aee66be87825cdf72ac64cb03ad4c15ffef4143dbf5c113f64a5ff4f81477bf9",
+                "sha256:af52d26579b308921b73b956153066481f064875140ccd1dfd4e77db89dbb12f",
+                "sha256:b94d4ba43739bbe8b0ce4262bcc3b7b9f31459ad120fb595627eaeb7f9b9ca01",
+                "sha256:ba630d5e3db74c79300d9a5bdaaf6200172b107f263c98a0539eeecb857b2337",
+                "sha256:bed0f8a0eeea9fb72937ba118f9db0cb7e90773462af7962d382445f3005e5a4",
+                "sha256:bf99c8404f008750c846cb4ac4667b798a9f7de673ff719d705d9b2d6de49c5f",
+                "sha256:c3027001c28434e7ca5a6e1e527487051136aa81803ac812be51802150d880dd",
+                "sha256:c65af9088ac534313e1963443d0ec360bb2b9cba6c2909478d22c2e363d98a51",
+                "sha256:d0165ab2914379bd56908c02294ed8405c252250668ebcb438a55494c69f44ab",
+                "sha256:d1b26e1dff225c31897696cab7d4f0a315d4c0d9e8666dbffdb28216f3b17fdc",
+                "sha256:d950caa237bb1954f1b8c9227b5065ba6875ac9771bb8ec790d956a699b78676",
+                "sha256:dc61505e73298a84a2f317255fcc72b710b72980f3a1f670447a21efc88f8381",
+                "sha256:e173486019cc283dc9778315fa29a363579372fe67045e971e89b6365cc035ed",
+                "sha256:e1f735dc43da318cad19b4173dd1ffce1d84aafd6c9b782b3abc04a0d5a6f5bb",
+                "sha256:e9386266798d64eeb19dd3677051f5705bf873e98e15897ddb7d76f477131967",
+                "sha256:f216dbce0e60e4d03e0c4353c7023b202d95cbaeff12e5fd2e82ea0a66905073",
+                "sha256:f4e5658dbffe8843a0f12366a4c2d1c316dbe09bb4dfbdc9d2d9cd6031de8aae",
+                "sha256:f5a823165e6d04ccea61a9f0576f345f8ce40ed533013580e087bd4d7442b52c",
+                "sha256:f69ed81ab24d5a3bd93861c8c4436f54afdf8e8cc421562b0c7504cf3be58206",
+                "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.23.4"
+            "version": "==2.27.1"
+        },
+        "pyright": {
+            "hashes": [
+                "sha256:aad7f160c49e0fbf8209507a15e17b781f63a86a1facb69ca877c71ef2e9538d",
+                "sha256:ecebfba5b6b50af7c1a44c2ba144ba2ab542c227eb49bc1f16984ff714e0e110"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==1.1.390"
         },
         "python-dateutil": {
             "hashes": [
@@ -541,12 +695,12 @@
         },
         "python-multipart": {
             "hashes": [
-                "sha256:15dc4f487e0a9476cc1201261188ee0940165cffc94429b6fc565c4d3045cb5d",
-                "sha256:41330d831cae6e2f22902704ead2826ea038d0419530eadff3ea80175aec5538"
+                "sha256:905502ef39050557b7a6af411f454bc19526529ca46ae6831508438890ce12cc",
+                "sha256:f8d5b0b9c618575bf9df01c684ded1d94a338839bdd8223838afacfb4bb2082d"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.0.17"
+            "version": "==0.0.19"
         },
         "pytz": {
             "hashes": [
@@ -557,10 +711,10 @@
         },
         "redis": {
             "hashes": [
-                "sha256:0b1087665a771b1ff2e003aa5bdd354f15a70c9e25d5a7dbf9c722c16528a7b0",
-                "sha256:ae174f2bb3b1bf2b09d54bf3e51fbc1469cf6c10aa03e21141f51969801a7897"
+                "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f",
+                "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"
             ],
-            "version": "==5.2.0"
+            "version": "==5.2.1"
         },
         "requests": {
             "hashes": [
@@ -573,20 +727,20 @@
         },
         "six": {
             "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+                "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
+                "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.16.0"
+            "version": "==1.17.0"
         },
         "slack-sdk": {
             "hashes": [
-                "sha256:0515fb93cd03b18de61f876a8304c4c3cef4dd3c2a3bad62d7394d2eb5a3c8e6",
-                "sha256:4cc44c9ffe4bb28a01fbe3264c2f466c783b893a4eca62026ab845ec7c176ff1"
+                "sha256:a5e74c00c99dc844ad93e501ab764a20d86fa8184bbc9432af217496f632c4ee",
+                "sha256:b8cccadfa3d4005a5e6529f52000d25c583f46173fda8e9136fdd2bc58923ff6"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.6'",
-            "version": "==3.33.3"
+            "version": "==3.33.5"
         },
         "smmap": {
             "hashes": [
@@ -604,13 +758,123 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.3.1"
         },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763",
+                "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436",
+                "sha256:07b441f7d03b9a66299ce7ccf3ef2900abc81c0db434f42a5694a37bd73870f2",
+                "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588",
+                "sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e",
+                "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959",
+                "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d",
+                "sha256:28120ef39c92c2dd60f2721af9328479516844c6b550b077ca450c7d7dc68575",
+                "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908",
+                "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8",
+                "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8",
+                "sha256:3d6718667da04294d7df1670d70eeddd414f313738d20a6f1d1f379e3139a545",
+                "sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7",
+                "sha256:4557e1f11c5f653ebfdd924f3f9d5ebfc718283b0b9beebaa5dd6b77ec290971",
+                "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855",
+                "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c",
+                "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71",
+                "sha256:50aae840ebbd6cdd41af1c14590e5741665e5272d2fee999306673a1bb1fdb4d",
+                "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb",
+                "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72",
+                "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f",
+                "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5",
+                "sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346",
+                "sha256:72c28b84b174ce8af8504ca28ae9347d317f9dba3999e5981a3cd441f3712e24",
+                "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e",
+                "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5",
+                "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08",
+                "sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793",
+                "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88",
+                "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686",
+                "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b",
+                "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2",
+                "sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28",
+                "sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d",
+                "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5",
+                "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a",
+                "sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a",
+                "sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3",
+                "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf",
+                "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5",
+                "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef",
+                "sha256:b817d41d692bf286abc181f8af476c4fbef3fd05e798777492618378448ee689",
+                "sha256:b81ee3d84803fd42d0b154cb6892ae57ea6b7c55d8359a02379965706c7efe6c",
+                "sha256:be9812b766cad94a25bc63bec11f88c4ad3629a0cec1cd5d4ba48dc23860486b",
+                "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07",
+                "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa",
+                "sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06",
+                "sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1",
+                "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff",
+                "sha256:dc022184d3e5cacc9579e41805a681187650e170eb2fd70e28b86192a479dcaa",
+                "sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687",
+                "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4",
+                "sha256:f942a799516184c855e1a32fbc7b29d7e571b52612647866d4ec1c3242578fcb",
+                "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44",
+                "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c",
+                "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e",
+                "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.36"
+        },
+        "sqlmodel": {
+            "hashes": [
+                "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e",
+                "sha256:a1ed13e28a1f4057cbf4ff6cdb4fc09e85702621d3259ba17b3c230bfb2f941b"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==0.0.22"
+        },
         "starlette": {
             "hashes": [
-                "sha256:9834fd799d1a87fd346deb76158668cfa0b0d56f85caefe8268e2d97c3468b62",
-                "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"
+                "sha256:0e4ab3d16522a255be6b28260b938eae2482f98ce5cc934cb08dce8dc3ba5835",
+                "sha256:44cedb2b7c77a9de33a8b74b2b90e9f50d11fcf25d8270ea525ad71a25374ff7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.41.2"
+            "version": "==0.41.3"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6",
+                "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd",
+                "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c",
+                "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b",
+                "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8",
+                "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6",
+                "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77",
+                "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff",
+                "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea",
+                "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192",
+                "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249",
+                "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee",
+                "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4",
+                "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98",
+                "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8",
+                "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4",
+                "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281",
+                "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744",
+                "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69",
+                "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13",
+                "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140",
+                "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e",
+                "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e",
+                "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc",
+                "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff",
+                "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec",
+                "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2",
+                "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222",
+                "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106",
+                "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272",
+                "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a",
+                "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==2.2.1"
         },
         "typing-extensions": {
             "hashes": [
@@ -638,12 +902,12 @@
         },
         "uvicorn": {
             "hashes": [
-                "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82",
-                "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"
+                "sha256:2c30de4aeea83661a520abab179b24084a0019c0c1bbe137e5409f741cbde5f8",
+                "sha256:3577119f82b7091cf4d3d4177bfda0bae4723ed92ab1439e8d779de880c9cc59"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.32.0"
+            "version": "==0.33.0"
         }
     },
     "develop": {}
diff --git a/app/database.py b/app/database.py
index 04890e0..53576d3 100644
--- a/app/database.py
+++ b/app/database.py
@@ -14,8 +14,8 @@
 # env
 load_dotenv()
 POSTGRES_URL = os.getenv('POSTGRES_URL')
-TEST_GITHUB_PAT = os.getenv('TEST_GITHUB_PAT')
-SPARK_GITHUB_PAT = os.getenv('SPARK_GITHUB_PAT')
+TEST_GITHUB_PAT = os.getenv('TEST_GITHUB_PAT') or "-"
+SPARK_GITHUB_PAT = os.getenv('SPARK_GITHUB_PAT') or "-"
 
 # app
 github = git.Github(SPARK_GITHUB_PAT, 'BU-Spark')
diff --git a/app/github.py b/app/github.py
index 61d4cfb..61d216b 100644
--- a/app/github.py
+++ b/app/github.py
@@ -166,8 +166,9 @@ def get_users_invited_on_repo(self, repo_url: str, check_expired: bool = False )
         
         if invitations_response.status_code == 200:
             if check_expired:
-                return {invited_collaborators['invitee']['login'] if invited_collaborators["expired"] else None
-                                        for invited_collaborators in invitations_response.json()} - {None}
+                return {invited_collaborators['invitee']['login']
+                        for invited_collaborators in invitations_response.json()
+                        if invited_collaborators["expired"]}
             else:
                 return {invitation['invitee']['login'] for invitation in invitations_response.json()}
         elif invitations_response.status_code == 404:
@@ -326,11 +327,11 @@ def change_all_user_permission_on_repo(self, repo_url: str, permission: perms) -
         except Exception as e:
             return [(500, str(e))]
 
-    def get_all_repos(self) -> list[str]:
+    def get_all_repos(self) -> list[tuple[str, str]]:
         """
         Retrieves a list of all repositories in the organization.
         
-        Returns: list[str]: A list of all repositories in the organization.
+        Returns: list[tuple[str, str]]: A list of all repositories in the organization.
         """
         
         try:
diff --git a/app/github_rest.py b/app/github_rest.py
deleted file mode 100644
index d09e6c0..0000000
--- a/app/github_rest.py
+++ /dev/null
@@ -1,697 +0,0 @@
-# =========================================== imports =============================================
-import json
-import requests
-import csv
-import os
-
-from typing import Literal, Optional
-from dotenv import load_dotenv
-
-# =========================================== automation ==========================================
-
-class Automation:
-    GITHUB_PAT = None
-    HEADERS = None
-    ORG_NAME = None
-    
-    def __init__(self, GITHUB_PAT: str, ORG_NAME: str):
-        self.GITHUB_PAT = GITHUB_PAT
-        self.ORG_NAME = ORG_NAME
-        self.HEADERS = {
-            'Accept': 'application/vnd.github+json',
-            'Authorization': f'Bearer {GITHUB_PAT}',
-            'X-GitHub-Api-Version': '2022-11-28'
-        }
-        print(f"automation initialized with {GITHUB_PAT} and {ORG_NAME}")
-    
-    def get_organization_repositories(self) -> list[str]:
-        """
-        Retrieves a list of repositories belonging to the organization.
-
-        Returns:
-            list[str]: A list of repository names belonging to the organization.
-        """
-        try:
-            response = requests.get(
-                f'https://api.github.com/orgs/{self.ORG_NAME}/repos', headers=self.HEADERS, timeout=2)
-            
-            if response.status_code == 200:
-                return [repo['name'] for repo in response.json()]
-            
-            elif response.status_code == 404:
-                raise FileNotFoundError(f"Organization '{self.ORG_NAME}' not found.")
-            elif response.status_code == 401:
-                raise PermissionError("Unauthorized: Invalid GitHub PAT.")
-            elif response.status_code == 403 or response.status_code == 429:
-                raise PermissionError("Forbidden: Rate limit exceeded.")
-            else:
-                raise Exception(f"Failed to fetch repositories: {response.json().get('message', 'Unknown error')}")
-                
-        except requests.exceptions.ConnectionError:
-            raise ConnectionError("Failed to establish a connection to the GitHub API.")
-        except requests.exceptions.Timeout:
-            raise TimeoutError("The request to get repositories timed out.")
-        except Exception as e:
-            raise Exception(f"Failed to fetch repositories: {e}") from e
-    
-    def get_repository_ssh_url(self, repo_name: str) -> str:
-        """
-        Retrieves the SSH URL of a repository belonging to the organization.
-
-        Args:
-            repo_name (str): The name of the repository.
-
-        Returns:
-            str: The SSH URL of the repository.
-        """
-        try:
-            url = f'https://api.github.com/repos/{self.ORG_NAME}/{repo_name}'
-            response = requests.get(url, headers=self.HEADERS, timeout=2)
-            if response.status_code == 200:
-                return response.json()['ssh_url']
-            
-            elif response.status_code == 404:
-                raise FileNotFoundError(f"Repository '{repo_name}' not found.")
-            elif response.status_code == 401:
-                raise PermissionError("Unauthorized: Invalid GitHub PAT.")
-            elif response.status_code == 403 or response.status_code == 429:
-                raise PermissionError("Forbidden: Rate limit exceeded.")
-            else:
-                raise Exception(f"Failed to fetch repository URL: {response.json().get('message', 'Unknown error')}")
-            
-        except Exception as e:
-            raise Exception(f"Failed to fetch repository URL: {e}") from e
-    
-    def extract_user_repo_from_ssh(self, ssh_url: str) -> tuple[str, str]:
-        """
-        Extracts the username and repository name from a given SSH URL.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-
-        Returns:
-            Tuple[str, str]: A tuple containing the username and repository name.
-
-        Raises:
-            ValueError: If the SSH URL does not start with "git@github.com:" or is missing required parts.
-        """
-
-        if not ssh_url.startswith("git@github.com:"):
-            raise ValueError("Invalid SSH URL format")
-        try:
-            ssh_url_parts = ssh_url.split(':')[-1].split('/')
-            username = ssh_url_parts[0]
-            repo_name = ssh_url_parts[1].split('.')[0]
-            return username, repo_name
-        except IndexError as e:
-            raise ValueError("SSH URL is missing required parts") from e
-
-    def check_user_exists(self, user: str) -> tuple[int, Optional[str]]:
-        """
-        Checks if a GitHub user exists.
-
-        Args:
-            user (str): The username of the GitHub user to check.
-
-        Returns:
-            Tuple[int, Optional[str]]: A tuple containing the HTTP status code and an optional error message.
-        """
-
-        response = requests.get(
-            f'https://api.github.com/users/{user}', headers=self.HEADERS, timeout=2)
-        if response.status_code == 200:
-            return 200, None
-        elif response.status_code == 404:
-            return 404, f'User does not exist: {response.json()}'
-        elif response.status_code == 401:
-            return 401, f'Unauthorized: Invalid GitHub PAT, {response.json()}'
-        elif response.status_code == 403 or response.status_code == 429:
-            return 403, f'Forbidden: Rate limit exceeded: {response.json()}'
-        else:
-            return response.status_code, f'An error occurred: {response.json()}'
-        
-    def add_user_to_repo(self, ssh_url: str, user: str, permission: Literal['pull', 'triage', 'push', 'maintain', 'admin']) -> tuple[int, Optional[str]]:
-        """
-        Adds a user to a GitHub project with the specified permission.
-
-        This function first extracts the username and repository name from the provided SSH URL. It then checks if the user exists on GitHub. If the user exists, it attempts to add the user to the specified repository with the given permission level. The function handles various HTTP status codes to provide meaningful feedback on the operation's outcome.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            user (str): The username of the GitHub user to add to the project.
-            permission (Literal['pull', 'triage', 'push', 'maintain', 'admin']): The permission level to grant the user.
-
-        Returns:
-            Tuple[int, Optional[str]]: A tuple containing the HTTP status code and an optional error message.
-        Raises:
-            Exception: If an unexpected error occurs.
-
-        """
-        try:
-
-            username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-            status_code, error_message = self.check_user_exists(user)
-            if error_message:
-                return status_code, error_message
-
-            # Add the user to the project with the specified permission
-            response = requests.put(f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}',
-                                    headers=self.HEADERS,
-                                    json={'permission': permission}, timeout=2)
-            if response.status_code == 201:
-                return response.status_code, 'User added to the project with the specified permission'
-            elif response.status_code == 204:
-                return response.status_code, 'User permission updated'
-            else:
-                return response.status_code, response.json()
-        except Exception as e:
-            return -1, str(e)
-    
-    def add_user_to_repos(self, ssh_urls: list[str], user: str, permission: Literal['pull', 'triage', 'push', 'maintain', 'admin']) -> list[tuple[int, str]]:    
-        return [self.add_user_to_repo(ssh_url, user, permission) for ssh_url in ssh_urls]
-    
-    def revoke_user_invitation(self, ssh_url: str, user: str) -> tuple[int, Optional[str]]:
-        """
-        Revokes an invitation to collaborate on a GitHub repository.
-
-        This function first extracts the username and repository name from the provided SSH URL. It then checks if the user exists on GitHub. If the user exists and has been invited to collaborate on the specified repository, it attempts to revoke the user's invitation. The function handles various HTTP status codes to provide meaningful feedback on the operation's outcome.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            user (str): The username of the GitHub user to revoke the invitation for.
-
-        Returns:
-            Tuple[int, Optional[str]]: A tuple containing the HTTP status code and an optional error message.
-        Raises:
-            Exception: If an unexpected error occurs.
-            Timeout: If the request times out.
-
-        """
-        try:
-            username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-            status_code, error_message = self.check_user_exists(user)
-            if error_message:
-                return status_code, error_message
-
-            # Check if the user has been invited to collaborate on the specified repository
-            invited_collaborators_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/invitations',
-                headers=self.HEADERS,
-                timeout=2
-            )
-            if invited_collaborators_response.status_code != 200:
-                return invited_collaborators_response.status_code, 'Failed to fetch invited collaborators'
-            
-            invited_collaborators = invited_collaborators_response.json()
-            for invited_collaborator in invited_collaborators:
-                if invited_collaborator['invitee']['login'] == user:
-                    # Revoke the user's invitation
-                    revoke_response = requests.delete(
-                        f'https://api.github.com/repos/{username}/{repo_name}/invitations/{invited_collaborator["id"]}',
-                        headers=self.HEADERS,
-                        timeout=2
-                    )
-                    if revoke_response.status_code == 204:
-                        return revoke_response.status_code, 'User invitation revoked successfully'
-                    else:
-                        return revoke_response.status_code, revoke_response.json()
-
-            return 404, 'User has not been invited to collaborate on the repository.'
-        except requests.exceptions.Timeout:
-            return -1, "Request timed out"
-        except Exception as e:
-            return -1, str(e)
-    
-    def remove_user_from_repo(self, ssh_url: str, user: str) -> tuple[int, Optional[str]]:
-        """
-        Removes a user from a GitHub repo.
-
-        This function first extracts the username and repository name from the provided SSH URL. It then checks if the user exists on GitHub. If the user exists and has permissions on the specified repository, it attempts to remove the user from the repository. The function handles various HTTP status codes to provide meaningful feedback on the operation's outcome.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            user (str): The username of the GitHub user to remove from the repo.
-
-        Returns:
-            Tuple[int, Optional[str]]: A tuple containing the HTTP status code and an optional error message.
-        Raises:
-            Exception: If an unexpected error occurs.
-            Timeout: If the request times out.
-
-        """
-        try:
-            username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-            status_code, error_message = self.check_user_exists(user)
-            if error_message:
-                return status_code, error_message
-            
-            # Check if the user has permissions on the specified repository
-            permissions_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}/permission', headers=self.HEADERS, timeout=2)
-            if permissions_response.status_code != 200:
-                return permissions_response.status_code, 'Nothing to do - User does not have permissions on the repository.'
-
-            # Remove the user from the repository
-            remove_response = requests.delete(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}', headers=self.HEADERS, timeout=2)
-            if remove_response.status_code == 204:
-                return remove_response.status_code, 'User removed from the repository successfully'
-            else:
-                return remove_response.status_code, remove_response.json()
-        except requests.exceptions.Timeout:
-            return -1, "Request timed out"
-        except Exception as e:
-            return -1, str(e)
-    
-    def remove_all_users_from_repo(self, ssh_url: str) -> list[tuple[int, str]]:
-        """
-        Removes all collaborators from a given repository using the get_users_on_repo function to fetch the list of collaborators.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-
-        Returns:
-            list[tuple[int, str]]: A list of tuples, each containing the HTTP status code and a message indicating the success or failure of the operation.
-        """
-        try:
-            collaborators = self.get_users_on_repo(ssh_url)
-        except Exception as e:
-            return -1, str(e)
-
-        username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-        result = []
-        # Attempt to remove each collaborator
-        for collaborator in collaborators:
-            try:
-                remove_response = requests.delete(
-                    f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{collaborator}',
-                    headers=self.HEADERS,
-                    timeout=2
-                )
-                if remove_response.status_code not in [204, 404]:
-                    result.append((remove_response.status_code,
-                                    f"Failed to remove {collaborator}"))
-                result.append((remove_response.status_code,
-                                f"{collaborator} removed successfully"))
-            except Exception as e:
-                result.append((-1, str(e)))
-
-        return result
-    
-    def remove_or_revoke_user(self, ssh_url: str, user: str) -> tuple[int, Optional[str]]:
-        """
-        Removes a user from a GitHub repository or revokes their invitation if they have not accepted it.
-
-        This function first extracts the username and repository name from the provided SSH URL. It then checks if the user exists on GitHub. If the user exists and has permissions on the specified repository, it attempts to remove the user from the repository. If the user has not accepted an invitation to collaborate on the repository, the function revokes the invitation instead. The function handles various HTTP status codes to provide meaningful feedback on the operation's outcome.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            user (str): The username of the GitHub user to remove or revoke the invitation for.
-
-        Returns:
-            Tuple[int, Optional[str]]: A tuple containing the HTTP status code and an optional error message.
-        Raises:
-            Exception: If an unexpected error occurs.
-            Timeout: If the request times out.
-
-        """
-        try:
-            username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-            status_code, error_message = self.check_user_exists(user)
-            if error_message:
-                return status_code, error_message
-
-            # Check if the user has permissions on the specified repository
-            permissions_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}/permission', headers=self.HEADERS, timeout=2)
-            if permissions_response.status_code == 200:
-                # User has permissions on the repository, remove them
-                return self.remove_user_from_repo(ssh_url, user)
-            elif permissions_response.status_code == 404:
-                # User does not have permissions on the repository, revoke their invitation
-                return self.revoke_user_invitation(ssh_url, user)
-            else:
-                return permissions_response.status_code, 'An error occurred while checking user permissions'
-        except requests.exceptions.Timeout:
-            return -1, "Request timed out"
-        except Exception as e:
-            return -1, str(e)
-    
-    def get_users_on_repo(self, ssh_url: str) -> set[str]:
-        """
-        Retrieves a set of GitHub usernames who are collaborators on a given repository.
-
-        This function extracts the username and repository name from the provided SSH URL. It then makes a request to the GitHub API to fetch the list of collaborators on the specified repository. The function returns a set of GitHub usernames who have access to the repository.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-
-        Returns:
-            set[str]: A set of GitHub usernames who are collaborators on the repository.
-
-        Raises:
-            Exception: If an error occurs during the API request or while processing the response.
-        """
-        username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-        # Get the list of collaborators on the repository
-        try:
-            print(f"Fetching collaborators for {username}/{repo_name}")
-            print(f'Headers: {self.HEADERS}')
-            collaborators_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators',
-                headers=self.HEADERS,
-                timeout=10
-            )
-        except requests.exceptions.Timeout as e:
-            raise TimeoutError(
-                "The request to get collaborators timed out.") from e
-        except requests.exceptions.ConnectionError as e:
-            raise ConnectionError(
-                "Failed to establish a connection to the GitHub API.") from e
-        except requests.exceptions.RequestException as e:
-            raise Exception(
-                f"An error occurred while making the request: {e}") from e
-        except Exception as e:
-            raise Exception(
-                f"Failed to fetch collaborators: {collaborators_response.json().get('message', 'Unknown error')}") from e
-
-        if collaborators_response.status_code == 200:
-            collaborators = {collaborator['login']
-                            for collaborator in collaborators_response.json()}
-            return collaborators
-        elif collaborators_response.status_code == 404:
-            raise FileNotFoundError("The repository was not found.")
-        elif collaborators_response.status_code == 403:
-            raise PermissionError("Access to the repository is forbidden.")
-        else:
-            raise Exception(
-                f"Failed to fetch collaborators: {collaborators_response.json().get('message', 'Unknown error')}")
-        
-    def get_users_invited_repo(self, ssh_url: str) -> set[str]:
-        """
-        Retrieves a set of GitHub usernames who are invited collaborators on a given repository.
-
-        This function extracts the username and repository name from the provided SSH URL. It then makes a request to the GitHub API to fetch the list of invited collaborators on the specified repository. The function returns a set of GitHub usernames who have been invited to collaborate on the repository.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-
-        Returns:
-            set[str]: A set of GitHub usernames who are invited collaborators on the repository.
-
-        Raises:
-            Exception: If an error occurs during the API request or while processing the response.
-        """
-
-        username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-        try:
-            invited_collaborators_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/invitations',
-                headers=self.HEADERS,
-                timeout=10
-            )
-            print(json.dumps(invited_collaborators_response.json(), indent=4))
-            if invited_collaborators_response.status_code == 200:
-                invited_collaborators = {invited_collaborators['invitee']['login']
-                                        for invited_collaborators in invited_collaborators_response.json()}
-                return invited_collaborators
-            elif invited_collaborators_response.status_code == 404:
-                raise FileNotFoundError("The repository was not found.")
-            elif invited_collaborators_response.status_code == 403:
-                raise PermissionError("Access to the repository is forbidden.")
-            else:
-                raise Exception(
-                    f"Failed to fetch invited collaborators: {invited_collaborators_response.json().get('message', 'Unknown error')}")
-                
-        except requests.exceptions.Timeout as e:
-            raise TimeoutError(
-                "The request to get invited collaborators timed out.") from e
-        except requests.exceptions.ConnectionError as e:
-            raise ConnectionError(
-                "Failed to establish a connection to the GitHub API.") from e
-        except requests.exceptions.RequestException as e:
-            raise Exception(
-                f"An error occurred while making the request: {e}") from e
-        except Exception as e:
-            raise Exception(
-                f"Failed to fetch invited collaborators: {invited_collaborators_response.json().get('message', 'Unknown error')}") from e
-    
-    def get_expired_invited_collaborators(self, ssh_url: str) -> set[str]:
-        """same as get_users_invited_repo but only returns expired invites"""
-        username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-        try:
-            invited_collaborators_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/invitations',
-                headers=self.HEADERS,
-                timeout=10
-            )
-            if invited_collaborators_response.status_code == 200:
-                invited_collaborators = {invited_collaborators['invitee']['login'] if invited_collaborators["expired"] else None
-                                        for invited_collaborators in invited_collaborators_response.json()} - {None}
-                return invited_collaborators
-            elif invited_collaborators_response.status_code == 404:
-                raise FileNotFoundError("The repository was not found.")
-            elif invited_collaborators_response.status_code == 403:
-                raise PermissionError("Access to the repository is forbidden.")
-            else:
-                raise Exception(
-                    f"Failed to fetch invited collaborators: {invited_collaborators_response.json().get('message', 'Unknown error')}")
-                
-        except requests.exceptions.Timeout as e:
-            raise TimeoutError(
-                "The request to get invited collaborators timed out.") from e
-        except requests.exceptions.ConnectionError as e:
-            raise ConnectionError(
-                "Failed to establish a connection to the GitHub API.") from e
-        except requests.exceptions.RequestException as e:
-            raise Exception(
-                f"An error occurred while making the request: {e}") from e
-        except Exception as e:
-            raise Exception(
-                f"Failed to fetch invited collaborators: {invited_collaborators_response.json().get('message', 'Unknown error')}") from e
-    
-    def change_user_permission(self, ssh_url: str, user: str, permission: Literal['pull', 'triage', 'push', 'maintain', 'admin']) -> tuple[int, Optional[str]]:
-        """
-        Changes the permission level of a user on a GitHub repository.
-
-        This function first extracts the username and repository name from the provided SSH URL. It then checks if the user exists on GitHub. If the user exists and has permissions on the specified repository, it attempts to change the user's permission level to the specified value. The function handles various HTTP status codes to provide meaningful feedback on the operation's outcome.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            user (str): The username of the GitHub user to change the permission level for.
-            permission (Literal['pull', 'triage', 'push', 'maintain', 'admin']): The new permission level to assign to the user.
-
-        Returns:
-            Tuple[int, Optional[str]]: A tuple containing the HTTP status code and an optional error message.
-        Raises:
-            Exception: If an unexpected error occurs.
-            Timeout: If the request times out.
-
-        """
-        try:
-            username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-            status_code, error_message = self.check_user_exists(user)
-            if error_message:
-                return status_code, error_message
-
-            # Check if the user has permissions on the specified repository
-            permissions_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}/permission', headers=self.HEADERS, timeout=2)
-            if permissions_response.status_code != 200:
-                return permissions_response.status_code, 'User does not have permissions on the repository.'
-
-            # Change the user's permission level
-            change_permission_response = requests.put(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}',
-                headers=self.HEADERS,
-                json={'permission': permission},
-                timeout=2
-            )
-            if change_permission_response.status_code == 200:
-                return change_permission_response.status_code, 'User permission level changed successfully'
-            elif change_permission_response.status_code == 204:
-                return change_permission_response.status_code, 'User permission level updated'
-            else:
-                return change_permission_response.status_code, change_permission_response.json()
-        except requests.exceptions.Timeout:
-            return -1, "Request timed out"
-        except Exception as e:
-            return -1, str(e)
-    
-    def change_all_users_permission(self, ssh_url: str, permission: Literal['pull', 'triage', 'push', 'maintain', 'admin']) -> list[tuple[int, str]]:
-        """
-        Changes the permission level of all collaborators on a given repository.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            permission (Literal['pull', 'triage', 'push', 'maintain', 'admin']): The new permission level to assign to all collaborators.
-
-        Returns:
-            list[tuple[int, str]]: A list of tuples, each containing the HTTP status code and a message indicating the success or failure of the operation.
-        """
-        try:
-            collaborators = self.get_users_on_repo(ssh_url)
-        except Exception as e:
-            return [(-1, str(e))]
-
-        result = []
-        try:
-            for collaborator in collaborators:
-                res = self.change_user_permission(ssh_url, collaborator, permission)
-                result.append(res)
-        except Exception as e:
-            result.append((-1, str(e)))
-    
-    def set_repo_users(self, ssh_url: str, desired_users: set[str]) -> list[tuple[int, str]]:
-        """
-        Sets the repository to only have the specified users. Removes users not in the desired list and adds users missing from the repository.
-
-        Args:
-            ssh_url (str): The SSH URL of the GitHub repository.
-            desired_users (set[str]): A set of usernames that should have access to the repository.
-
-        Returns:
-            list[tuple[int, str]]: A list of tuples, each containing the HTTP status code and a message indicating the success or failure of each operation.
-        """
-        desired_users = set(desired_users)
-        try:
-            current_users = self.get_users_on_repo(ssh_url)
-        except Exception as e:
-            return [(-1, str(e))]
-
-        username, repo_name = self.extract_user_repo_from_ssh(ssh_url)
-
-        result = []
-
-        # Remove users not in the desired list
-        #for user in current_users:
-        #    if user not in desired_users:
-        #        try:
-        #            remove_response = requests.delete(
-        #                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}',
-        #                headers=self.HEADERS,
-        #                timeout=2
-        #            )
-        #            if remove_response.status_code == 204:
-        #                result.append((204, f"{user} removed successfully"))
-        #            else:
-        #                result.append((remove_response.status_code,
-        #                            f"Failed to remove {user}"))
-        #        except Exception as e:
-        #            result.append((-1, str(e)))
-        
-        # Change permission of users not in the desired list to 'pull'
-        
-        print(f"Current users: {current_users}")
-        
-        for user in current_users:
-            if user not in desired_users:
-                res = self.change_user_permission(ssh_url, user, 'pull')
-                result.append(res)
-        
-        print(f"Desired users: {desired_users}")
-        
-        # ensure all current users have write access
-        for user in current_users:
-            if user in desired_users:
-                res = self.change_user_permission(ssh_url, user, 'push')
-                result.append(res)
-        
-        # Add missing users
-        for user in desired_users-current_users:
-            if user not in current_users:
-                try:
-                    add_response = requests.put(
-                        f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}',
-                        headers=self.HEADERS,
-                        timeout=2
-                    )
-                    if add_response.status_code in [201, 204]:
-                        result.append((add_response.status_code,
-                                    f"{user} added successfully"))
-                    else:
-                        result.append((add_response.status_code,
-                                    f"Failed to add {user}"))
-                except Exception as e:
-                    result.append((-1, str(e)))
-
-        return result
-
-    def reinvite_all_expired_users_to_repos(self):
-        """
-        Re-invites all users who have had their invitations expire to all repositories in the organization.
-
-        Returns:
-            list[tuple[str, int, str]]: A list of tuples, each containing the repository name, HTTP status code, and a message indicating the success or failure of the operation.
-        """
-        repositories = self.get_organization_repositories()
-        result = []
-        for repo in repositories:
-            ssh_url = self.get_repository_ssh_url(repo)
-            try:
-                invited_collaborators = self.get_expired_invited_collaborators(ssh_url)
-                print(f"Invited collaborators for {repo}: {invited_collaborators}")
-                for user in invited_collaborators:
-                    remove_res = self.revoke_user_invitation(ssh_url, user)
-                    res = self.add_user_to_repo(ssh_url, user, 'push')
-                    result.append((repo, res[0], res[1]))
-            except Exception as e:
-                result.append((repo, -1, str(e)))
-        return result
-
-    def set_all_repos_users_read_only(self):
-        """
-        Sets all users in all repositories to read-only access.
-
-        Returns:
-            list[tuple[str, int, str]]: A list of tuples, each containing the repository name, HTTP status code, and a message indicating the success or failure of the operation.
-        """
-        repositories = self.get_organization_repositories()
-        result = []
-        for repo in repositories:
-            ssh_url = self.get_repository_ssh_url(repo)
-            try:
-                collaborators = self.get_users_on_repo(ssh_url)
-                for user in collaborators:
-                    res = self.change_user_permission(ssh_url, user, 'pull')
-                    result.append((repo, res[0], res[1]))
-            except Exception as e:
-                result.append((repo, -1, str(e)))
-        return result
-# =========================================== runs =============================================
-
-def test():
-    load_dotenv()
-    SPARK_GITHUB_PAT = os.getenv('SPARK_GITHUB_PAT')
-    automation = Automation(SPARK_GITHUB_PAT, 'BU-Spark')
-    print(automation.GITHUB_PAT)
-    
-    #inital_ssh_url = automation.get_repository_ssh_url('initial')
-    #byte_ssh_url = automation.get_repository_ssh_url('byte')
-    #invited_to_byte = automation.get_users_invited_repo(byte_ssh_url)
-    
-    #set_byte_users = automation.set_repo_users(byte_ssh_url, {'s-alad'})
-    #print(set_byte_users)
-    #print(inital_ssh_url)
-    #set_initial_users = automation.set_repo_users(inital_ssh_url, {'s-alad', 'mochiakku'})
-    #print(set_initial_users)
-    
-    #print("--")
-    #print(invited_to_byte)
-    #print(automation.remove_or_revoke_user(byte_ssh_url, ''))
-    invited = automation.reinvite_all_expired_users_to_repos()
-    print(invited)
-    
-    #automation.set_all_repos_users_read_only()
-    
-if __name__ == "__main__":
-    test()
\ No newline at end of file
diff --git a/app/main.py b/app/main.py
index 5b2f596..7833474 100644
--- a/app/main.py
+++ b/app/main.py
@@ -5,32 +5,29 @@
 from fastapi.middleware.cors import CORSMiddleware
 import uvicorn
 import pandas as pd
-import github_rest as gh
 import github as git
 import database as db
 import middleware as middleware
 import os
 import aiocache
 from dotenv import load_dotenv
-from aiocache import Cache
+from aiocache import Cache, BaseCache
 from aiocache.serializers import JsonSerializer
 from aiocache.decorators import cached
+from typing import Literal, cast
 
 # =========================================== app setup ===========================================
 
 # env
 load_dotenv()
-TEST_GITHUB_PAT = os.getenv('TEST_GITHUB_PAT')
-SPARK_GITHUB_PAT = os.getenv('SPARK_GITHUB_PAT')
+TEST_GITHUB_PAT = os.getenv('TEST_GITHUB_PAT') or "-"
+SPARK_GITHUB_PAT = os.getenv('SPARK_GITHUB_PAT') or "-"
 
 # app
 app = FastAPI()
 
-#automation = gh.Automation(TEST_GITHUB_PAT, 'spark-tests')
-#github = git.Github(TEST_GITHUB_PAT, 'spark-tests')
-
-automation = gh.Automation(SPARK_GITHUB_PAT, 'BU-Spark')
-github = git.Github(SPARK_GITHUB_PAT, 'BU-Spark')
+github = git.Github(TEST_GITHUB_PAT, 'spark-tests')
+# github = git.Github(SPARK_GITHUB_PAT, 'BU-Spark')
 
 aiocache.caches.set_config({
     'default': {
@@ -55,8 +52,8 @@
 # ========================================= functionality =========================================
 
 async def deletecache():
-    cache = aiocache.caches.get('default') 
-    await cache.clear()
+    cache: object = aiocache.caches.get('default')
+    await cast(BaseCache, cache).clear()
 
 # root route
 @app.get("/")
@@ -74,7 +71,7 @@ async def authenticate(): return {"status": "authenticated"}
 @app.post("/refresh")
 async def refresh(): 
     cache = aiocache.caches.get('default') 
-    await cache.clear()
+    await cast(BaseCache, cache).clear()
     return {"status": "cache cleared"}
     
 # route called re-invite expired collaborators that re-invites expired collaborators based on a cron job
@@ -158,12 +155,14 @@ async def set_projects(request: Request):
     
     results: list = []
     projects: list[tuple[str, str]] = data["projects"]
-    action: str = data["action"]
+    action = data["action"]
     
-    if action not in ['push', 'pull']: return {"status": "failed", "error": "action must be 'push' or 'pull'"}
+    if action not in ['push', 'pull']:
+        return {"status": "failed", "error": "action must be 'push' or 'pull'"}
     
     try:
         for project in projects:
+            project_name = ""
             try:
                 project_name = project[0]
                 repo_url = project[1]
@@ -186,17 +185,18 @@ async def set_projects(request: Request):
     except Exception as e: raise HTTPException(status_code=500, detail=str(e))
 
 @app.post("/git/set_projects")
-async def set_projects(request: Request):
+async def set_git_projects(request: Request):
     data = await request.json()
     
     results: list = []
     projects: list[tuple[str, str]] = data["projects"]
-    action: str = data["action"]
+    action: Literal['pull', 'triage', 'push', 'maintain', 'admin'] = data["action"]
     
     if action not in ['push', 'pull']: return {"status": "failed", "error": "action must be 'push' or 'pull'"}
     
     try:
         for project in projects:
+            project_name = ""
             try:
                 project_name = project[0]
                 repo_url = project[1]
diff --git a/app/models.py b/app/models.py
index b8b327a..30cc3f5 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1,3 +1,9 @@
+from __future__ import annotations
+
+from typing import Optional, List
+import enum
+from datetime import datetime
+
 from sqlalchemy import (
     Column,
     Integer,
@@ -5,114 +11,150 @@
     DateTime,
     ForeignKey,
     UniqueConstraint,
-    Enum,
+    Enum as Enum_,
     PrimaryKeyConstraint,
+    TIMESTAMP,
+    func
+)
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    MappedColumn,
+    Mapped,
+    mapped_column,
+    relationship
 )
-from sqlalchemy.orm import relationship
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.sql import func
-import enum
 
-Base = declarative_base()
+class Base(DeclarativeBase):
+    pass
 
-# Enums
-class SemesterEnum(enum.Enum):
-    Spring = 'Spring'
-    Summer = 'Summer'
-    Fall = 'Fall'
-    Winter = 'Winter'
+class Season(enum.Enum):
+    spring = 'spring'
+    summer = 'summer'
+    fall = 'fall'
+    winter = 'winter'
 
-class StatusEnum(enum.Enum):
+class Status(enum.Enum):
     started = 'started'
     invited = 'invited'
     pull = 'pull'
     push = 'push'
 
-class User(Base):
-    __tablename__ = 'user'
-
-    user_id = Column(Integer, primary_key=True)
-    name = Column(Text)
-    email = Column(Text, nullable=False, unique=True)
-    buid = Column(Text, nullable=False, unique=True)
-    github = Column(Text, unique=True)
-
-    # Relationship to UserProject
-    projects = relationship('UserProject', back_populates='user')
-
 class Semester(Base):
     __tablename__ = 'semester'
 
-    semester_id = Column(Integer, primary_key=True)
-    semester_name = Column(Text, nullable=False, unique=True)
-    year = Column(Integer, nullable=False, default=2077)
-    semester = Column(Enum(SemesterEnum))
+    semester_id: Mapped[int] = mapped_column(
+        Integer, primary_key=True, nullable=False,
+        # server_default=func.nextval('semester_semester_id_seq')
+    )
+    semester_name: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    year: Mapped[int] = mapped_column(Integer, nullable=False, default=2077)
+    season: Mapped[Season] = mapped_column(Enum_(Season), nullable=False)
 
-    # Relationship to Project
-    projects = relationship('Project', back_populates='semester')
+    # Bidirectional Relationship to Project
+    projects: Mapped[List[Project]] = \
+        relationship("Project", back_populates="semester", cascade="all, delete-orphan")
 
+    def __repr__(self):
+        return (
+            f"<Semester(semester_id={self.semester_id}, "
+            f"semester_name='{self.semester_name}', "
+            f"year={self.year}, season={self.season.value})>"
+        )
+
+class User(Base):
+    __tablename__ = 'user'
+    
+    user_id: Mapped[int] = mapped_column(
+        Integer, primary_key=True, nullable=False,
+        #server_default=func.nextval('user_user_id_seq')
+    )
+    first_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    last_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    email: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    buid: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    github_username: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    
+    # Bidirectional Relationship to UserProject
+    user_projects: Mapped[List["UserProject"]] = \
+        relationship("UserProject", back_populates="user", cascade="all, delete-orphan")
+
+    def __repr__(self) -> str:
+        return (
+            f"<User(user_id={self.user_id}, first_name='{self.first_name}', "
+            f"last_name='{self.last_name}', email='{self.email}', "
+            f"buid='{self.buid}', github_username='{self.github_username}')>"
+        )
+    
 class Project(Base):
     __tablename__ = 'project'
 
-    project_id = Column(Integer, primary_key=True)
-    project_name = Column(Text, nullable=False, unique=True)
-    semester_id = Column(Integer, ForeignKey('semester.semester_id'))
-    github_url = Column(Text)
-    created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
-
-    # Relationship to Semester
-    semester = relationship('Semester', back_populates='projects')
-
-    # Relationship to UserProject
-    users = relationship('UserProject', back_populates='project')
-
+    project_id: Mapped[int] = mapped_column(
+        Integer, primary_key=True, nullable=False,
+        #server_default=func.nextval('project_project_id_seq')
+    )
+    project_name: Mapped[str] = mapped_column(Text, nullable=False)
+    project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    semester_id: Mapped[int] = mapped_column(ForeignKey('semester.semester_id', ondelete='RESTRICT'), nullable=False)
+    github_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False, server_default=func.now())
+
+    # Bidirectional Relationship to Semester
+    semester: Mapped[Semester] = \
+        relationship("Semester", back_populates="projects")
+    user_projects: Mapped[List["UserProject"]] = \
+        relationship("UserProject", back_populates="project", cascade="all, delete-orphan")
+
+    def __repr__(self):
+        return (
+            f"<Project(project_id={self.project_id}, "
+            f"project_name='{self.project_name}', "
+            f"project_tag='{self.project_tag}', "
+            f"semester_id={self.semester_id}, "
+            f"github_url='{self.github_url}', "
+            f"slack_channel='{self.slack_channel}', "
+            f"created_at={self.created_at.isoformat()})>"
+        )
+        
 class UserProject(Base):
     __tablename__ = 'user_project'
-    __table_args__ = (
-        PrimaryKeyConstraint('project_id', 'user_id'),
-    )
 
-    project_id = Column(
-        Integer,
-        ForeignKey('project.project_id', onupdate='CASCADE', ondelete='CASCADE'),
-        primary_key=True,
-    )
-    user_id = Column(
-        Integer,
-        ForeignKey('user.user_id', onupdate='CASCADE', ondelete='CASCADE'),
-        primary_key=True,
+    project_id: Mapped[int] = \
+        mapped_column(ForeignKey('project.project_id'), primary_key=True, nullable=False)
+    user_id: Mapped[int] = \
+        mapped_column(ForeignKey('user.user_id'), primary_key=True, nullable=False)
+    status: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True),nullable=False, server_default=func.now())
+
+    # Relationships to User and Project
+    user: Mapped["User"] = relationship("User", back_populates="user_projects")
+    project: Mapped["Project"] = relationship("Project", back_populates="user_projects")
+
+    def __repr__(self) -> str:
+        return (
+            f"<UserProject(project_id={self.project_id}, user_id={self.user_id}, "
+            f"status={self.status}, created_at={self.created_at.isoformat()})>"
+        )
+
+class IngestProjectCSV(Base):
+    __tablename__ = "ingest_project_csv"
+    
+    id: Mapped[int] = mapped_column(
+        Integer, primary_key=True, nullable=False,
+        #server_default=func.nextval('ingest_project_csv_id_seq')
     )
-    status = Column(Enum(StatusEnum))
-    created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
-
-    # Relationships
-    user = relationship('User', back_populates='projects')
-    project = relationship('Project', back_populates='users')
-
-class CSV(Base):
-    __tablename__ = 'csv'
-
-    id = Column(Integer, primary_key=True)
-    semester = Column(Text)
-    course = Column(Text)
-    project = Column(Text)
-    organization = Column(Text)
-    team = Column(Text)
-    role = Column(Text)
-    first_name = Column(Text)
-    last_name = Column(Text)
-    full_name = Column(Text)
-    email = Column(Text)
-    buid = Column(Text)
-    github_username = Column(Text)
-    status = Column(Text)
-    project_github_url = Column(Text)
-
-class CSVProjects(Base):
-    __tablename__ = 'csv_projects'
-
-    id = Column(Integer, primary_key=True)
-    semester = Column(Text)
-    project = Column(Text)
-    project_github_url = Column(Text)
-    status = Column(Text)
+    project_name: Mapped[str] = mapped_column(Text, nullable=False)
+    project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    semester: Mapped[str] = mapped_column(Text, nullable=False)
+    github_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    generate_github: Mapped[Optional[bool]] = mapped_column(Integer, nullable=True, default=False)
+    generate_slack: Mapped[Optional[bool]] = mapped_column(Integer, nullable=True, default=False)
+    
+    def __repr__(self) -> str:
+        return (
+            f"<IngestProjectCSV(id={self.id}, project_name='{self.project_name}', "
+            f"project_tag='{self.project_tag}', semester='{self.semester}', "
+            f"github_url='{self.github_url}', slack_channel='{self.slack_channel}', "
+            f"generate_github={self.generate_github}, generate_slack={self.generate_slack})>"
+        )
\ No newline at end of file
diff --git a/app/schema.py b/app/schema.py
index b6870a8..21293d2 100644
--- a/app/schema.py
+++ b/app/schema.py
@@ -1,115 +1,16 @@
-import datetime
-from pydantic import BaseModel, Field, EmailStr
-from typing import Optional, List
-import enum
+from typing import Optional
+from pydantic import BaseModel
 
-# Enums
-class SemesterEnum(str, enum.Enum):
-    Spring = 'Spring'
-    Summer = 'Summer'
-    Fall = 'Fall'
-    Winter = 'Winter'
-
-class StatusEnum(str, enum.Enum):
-    started = 'started'
-    invited = 'invited'
-    pull = 'pull'
-    push = 'push'
-
-class UserBase(BaseModel):
-    name: Optional[str]
-    email: EmailStr
-    buid: str
-    github: Optional[str]
-
-class UserCreate(UserBase):
-    pass
-
-class User(UserBase):
-    user_id: int
-
-    class Config:
-        orm_mode = True
-
-class SemesterBase(BaseModel):
-    semester_name: str
-    year: int = 2077
-    semester: Optional[SemesterEnum]
-
-class SemesterCreate(SemesterBase):
-    pass
-
-class Semester(SemesterBase):
-    semester_id: int
-
-    class Config:
-        orm_mode = True
-
-class ProjectBase(BaseModel):
+class _Project(BaseModel):
     project_name: str
-    semester_id: Optional[int]
+    project_tag: str
+    semester_id: int
     github_url: Optional[str]
-
-class ProjectCreate(ProjectBase):
-    pass
-
-class Project(ProjectBase):
-    project_id: int
-    created_at: datetime.datetime
-
-    class Config:
-        orm_mode = True
-
-class UserProjectBase(BaseModel):
-    project_id: int
-    user_id: int
-    status: Optional[StatusEnum]
-
-class UserProjectCreate(UserProjectBase):
-    pass
-
-class UserProject(UserProjectBase):
-    created_at: datetime.datetime
-
-    class Config:
-        orm_mode = True
-
-class CSVBase(BaseModel):
-    semester: Optional[str]
-    course: Optional[str]
-    project: Optional[str]
-    organization: Optional[str]
-    team: Optional[str]
-    role: Optional[str]
+    slack_channel: Optional[str]
+    
+class _User(BaseModel):
     first_name: Optional[str]
     last_name: Optional[str]
-    full_name: Optional[str]
-    email: Optional[EmailStr]
+    email: str
     buid: Optional[str]
-    github_username: Optional[str]
-    status: Optional[str]
-    project_github_url: Optional[str]
-
-class CSVCreate(CSVBase):
-    pass
-
-class CSV(CSVBase):
-    id: int
-
-    class Config:
-        orm_mode = True
-
-class CSVProjectBase(BaseModel):
-    semester: Optional[str]
-    project: Optional[str]
-    project_github_url: Optional[str]
-    status: Optional[str]
-
-class CSVProjectCreate(CSVProjectBase):
-    pass
-
-class CSVProject(CSVProjectBase):
-    id: int
-
-    class Config:
-        orm_mode = True
+    github_username: Optional[str]
\ No newline at end of file
diff --git a/app/slacker.py b/app/slacker.py
index b1f6ad7..0eb9afd 100644
--- a/app/slacker.py
+++ b/app/slacker.py
@@ -8,7 +8,7 @@
 load_dotenv()
 
 class Slacker:
-    def __init__(self, token: str = None):
+    def __init__(self, token: str = ""):
         self.token = token
         if not self.token: raise ValueError("NO TOKEN PROVIDED")
         self.client = WebClient(token=self.token)
@@ -24,7 +24,7 @@ def get_user_id(self, email: str) -> str:
                 print(f"User with email '{email}' not found.")
             else:
                 print(f"Error fetching user '{email}': {e.response['error']}")
-            return None
+            return ""
 
     def create_channel(self, channel_name: str, is_private: bool = False) -> str:
         try:
@@ -42,7 +42,7 @@ def create_channel(self, channel_name: str, is_private: bool = False) -> str:
                 return existing_channel
             else:
                 print(f"Failed to create channel '{channel_name}': {e.response['error']}")
-                return None
+                return ""
 
     def get_channel_id(self, channel_name: str) -> str:
         try:
@@ -53,10 +53,10 @@ def get_channel_id(self, channel_name: str) -> str:
                     print(f"Found existing channel '{channel_name}' with ID: {channel['id']}")
                     return channel['id']
             print(f"Channel '{channel_name}' not found.")
-            return None
+            return ""
         except SlackApiError as e:
             print(f"Error fetching channels: {e.response['error']}")
-            return None
+            return ""
 
     def invite_users_to_channel(self, channel_id: str, user_ids: list, retry_count: int = 0):
         MAX_RETRIES = 5
@@ -119,8 +119,8 @@ def create_channels_and_add_users(self, channels_dict: dict, is_private: bool =
 
 
 if __name__ == "__main__":
-    slacker =   Slacker(token=os.getenv('SLACK_BOT_TOKEN'))
+    slacker = Slacker(token=os.getenv('SLACK_BOT_TOKEN') or "")
     channels_dict = {
-        'x4': ["x   @bu.edu"],
+        'x4': ["x@bu.edu"],
     }
     created_channels = slacker.create_channels_and_add_users(channels_dict, is_private=False)
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
new file mode 100644
index 0000000..f9cd43d
--- /dev/null
+++ b/app/spark.py
@@ -0,0 +1,137 @@
+import os
+from models import User, Project, Base, IngestProjectCSV, Semester, UserProject
+from schema import _Project
+from slacker import Slacker
+from sqlalchemy import create_engine
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+from sqlalchemy.orm import sessionmaker
+from contextlib import contextmanager
+from pandas import DataFrame
+
+class Spark:
+    
+    # =============================================================================================
+    # SQLAlchemy functionality
+    # =============================================================================================
+    
+    def __init__(self, URL: str, token: str):
+        self.URL = URL
+        self.engine = create_engine(self.URL, echo=False)
+        self.slacker = Slacker(token)
+        
+    def s(self):
+        return sessionmaker(bind=self.engine)()
+        
+    @contextmanager
+    def scope(self):
+        session = self.s()
+        try:
+            yield session
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            raise e
+        finally:
+            session.close()
+            
+    def run(self, func, *args, **kwargs):
+        session = self.s()
+        try:
+            result = func(session, *args, **kwargs)
+            session.commit()
+            return result
+        except IntegrityError as ie:
+            session.rollback()
+            print(f"Integrity error: {ie.orig}")
+        except Exception as e:
+            session.rollback()
+            print(f"Unexpected error: {e}")
+        finally:
+            session.close()
+            
+    # =============================================================================================
+    # auto spark functionality
+    # =============================================================================================        
+    
+    # ---------------------------------------------------------------------------------------------
+    # ingestion of csv files into holding tables
+    # ---------------------------------------------------------------------------------------------
+    
+    def ingest_csv(self, df: DataFrame, colmap: dict[str, str], table: str):
+        """ingest csv"""
+        
+        tomap = {
+            csv_col: \
+                db_col for csv_col, db_col in colmap.items() 
+                if csv_col != db_col and csv_col in df.columns
+        }
+        if tomap: df = df.rename(columns=tomap)
+    
+        if self.engine: df.to_sql(table, self.engine, if_exists='append', index=False)
+    
+    def ingest_project_csv(self, df: DataFrame):
+        """ingest project csv"""
+        
+        colmap: dict[str, str] = {
+            "Project Name": "project_name",
+            "Project Tag": "project_tag",
+            "Semester": "semester",
+            "GitHub Repository": "github_url",
+            "Slack": "slack_channel",
+            "Generate GitHub": "generate_github",
+            "Generate Slack": "generate_slack"
+        }
+
+        self.ingest_csv(df, colmap, "ingest_project_csv")
+        
+    def ingest_user_project_csv(self, df: DataFrame):
+        """ingest user project csv"""
+        
+        colmap: dict[str, str] = {
+            
+        }
+
+        self.ingest_csv(df, colmap, "ingest_user_project_csv")
+        
+    # ---------------------------------------------------------------------------------------------
+    # processing of csv files into holding tables
+    # ---------------------------------------------------------------------------------------------
+    
+    def process_ingest_project_csv(self):
+        """Process project csv"""
+        with self.scope() as session:
+            for row in session.query(IngestProjectCSV).all():
+                
+                semester = session.query(Semester).filter(Semester.semester_name == row.semester.lower()).first()
+                
+                if not semester:
+                    print(f"Semester {row.semester} not found")
+                    continue
+                
+                project_data = _Project(
+                    project_name=row.project_name,
+                    project_tag=row.project_tag,
+                    semester_id=semester.semester_id,
+                    github_url=row.github_url,
+                    slack_channel=row.slack_channel
+                )
+                project = Project(**project_data.model_dump())
+                session.add(project)
+                
+                if row.generate_github:
+                    # generate github
+                    pass
+                if row.generate_slack:
+                    channel_name = row.project_tag
+                    channel_id = self.slacker.create_channel(channel_name=channel_name, is_private=False)
+                    
+            session.commit()
+        
+if __name__ == "__main__":
+    TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
+    TEST_SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
+    spark = Spark(TEST_POSTGRES, TEST_SLACK_TOKEN)
+    #df = pd.read_csv("./ingestprojects.csv")
+    #spark.ingest_project_csv(df)
+    spark.process_ingest_project_csv()
\ No newline at end of file
diff --git a/scripts/README.md b/scripts/README.md
deleted file mode 100644
index 21cc747..0000000
--- a/scripts/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# About
-A collection scripts for automating the management of the BU Spark! GitHub organization.
-
-There are two main scripts in this repository:
-- `script.py` is for automatically adding people to COLLABORATORS file in BU Spark!'s repository.
-- `github_rest.py` is a collection of functions for interacting with the GitHub REST API and is now the preferred method for adding users to repositories.
-This script is for automatically adding people to COLLABORATORS file in BU Spark!'s repository.
-It works in conjunction with BU Spark!s GitHub workflow.
-
-## Github workflow scripts
-The GitHub workflow scripts include two main scripts to automate processes within the BU Spark! GitHub organization:
-
-1. `add-archive-tag.sh`: This script is used for adding an archive tag to a specific project repository. It clones the repository, creates a tag with a custom message indicating the project archive, and then pushes the tag to the repository. Finally, it cleans up by removing the cloned repository from the local machine.
-
-2. `update-workflows.sh`: This script is designed to update GitHub Actions workflows in a project repository by copying workflow files from a template repository. It ensures that the project repository has the latest CI/CD practices as defined in the template. The script also checks for the existence of a `COLLABORATORS` file and creates one if it doesn't exist, ensuring that the repository setup is complete.
-
-
-## `github_rest.py` Directions
-This script is meant to be integrated somewhere else but for now is just a collection of functions that can be used.
-
-Modify the `main` function as needed to read the correct input file and call the correct functions.
-
-There is basic logging that is in place but it needs some updating to increase the verbosity of the logs. 
-
-The script should also probably be refactored to use a class instead of a bunch of functions.
-
-## `script.py` Directions
-
-### To run the script:
-`python3 script.py <ssh_remote> <branchname> <collaborators_seperated_by_space>`
-
-### Current features
-- Checking if the entered username is a valid GitHub username (via GitHub API).
-- Adding username to COLLABORATORS file.
-- Specifying the branch to make the edit.
-
-### Global Variable Setup
-Check the top of the Python script for a serious of global variables that should be set for propper function.
-
-### Todo
-- Take input from CSV files.
-- Sync with each repo's collaborators list.
-- Remove users.
-- Change permission levels.
diff --git a/scripts/add-archive-tag.sh b/scripts/add-archive-tag.sh
deleted file mode 100644
index c9871f4..0000000
--- a/scripts/add-archive-tag.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-echo "Adding archive tag"
-git clone $2 dest
-cd dest
-git tag -a spark-archive-$1 -m "Spark! Project Archive $1"
-git push origin spark-archive-$1
-cd ..
-rm -rf ./dest
\ No newline at end of file
diff --git a/scripts/readme.md b/scripts/readme.md
deleted file mode 100644
index 92c6da6..0000000
--- a/scripts/readme.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Github Scripts Readme
-
-This read me holds all the documentation for this directory.
-
-# Scripts
-
-## `update-workflow.sh` 
-
-This script copies the latest version of the workflow scripts from the template repo to the target repo. 
-
-Usage(on *nix): `sh update-workflows.sh <git repo address>`
-
-It will clone the template repo if you don't have it already, then clone the target repo. Copy the updated workflows and then commit the changes and push. It will finally cleanup files afterwards. Probably only works on *nix systems.
-
-## `add-archive-tag.sh`
-
-Adds an archive tag to the provided repo and pushes tag to remote. 
-
-Provided tag name is pre-pended with `spark-archive-` so all you need is to specify the final part of the tag. For example `fall2021`
-
-Usage: `sh add-archive-tag.sh <tag ending> <github repo url>`
\ No newline at end of file
diff --git a/scripts/script.py b/scripts/script.py
deleted file mode 100644
index 1731f03..0000000
--- a/scripts/script.py
+++ /dev/null
@@ -1,225 +0,0 @@
-import os
-import sys
-import requests
-import traceback
-import csv
-
-from collections import defaultdict
-from time import sleep
-from dotenv import load_dotenv
-
-from git import Repo
-
-# =====================================================================================================================
-
-load_dotenv()
-
-# this is the directory where the repo will be cloned to, and be deleted afterwards
-LOCAL_PATH = os.path.dirname(os.path.realpath(__file__)) + '/temp/'
-
-# ensure that the ssh key path is correct
-SSH_KEY_PATH =  os.getenv('SSH_KEY_PATH')
-
-# your github personal access token
-GITHUB_PAT = os.getenv('GITHUB_PAT')
-
-# You shouldn't need to change these
-ERROR = True
-NO_ERROR = False
-EXCEPTION_LOG = '=============================================EXCEPTION!============================================='
-UPSTREAM = 'origin'
-
-# =====================================================================================================================
-
-
-def check_valid_user(username: str) -> tuple[bool, bool]:
-    """
-    checks if a user exists on github takes a username and returns T/F, ERROR/NO_ERROR
-    :param username: the username to check
-    :return: a tuple of two booleans, the first is if the user exists, the second is if there was an error
-    """
-    try:
-        r = requests.get(
-            f"https://api.github.com/users/{username}", 
-            headers={'Authorization': f"Bearer {GITHUB_PAT}"}
-        )
-
-        if r.status_code == 200:
-            print(f'Verified user {username} exists')
-            return True, NO_ERROR
-        else:
-            print(f'Failed to verify user {username} exists with status code {r.status_code}')
-            return False, NO_ERROR
-    except:
-        return False, ERROR
-
-
-def add_collaborators(path, collaborators):
-    try:
-        # open the file with 'r' to first read the current collaborators
-        with open(path + 'COLLABORATORS', 'r') as f:
-            # filter empty lines and new line characters, should they exist
-            exisiting_collaborators = list(
-                filter(lambda x: x != '' and x != '\n', f.readlines()))
-            exisiting_collaborators = list(
-                map(lambda x: x.replace('\n', ''), exisiting_collaborators))
-            
-            valid, invalid, errors = [], [], []
-
-            # first check if the user is already in the file
-            for user in list(filter(lambda x: x not in exisiting_collaborators,collaborators)):
-                # then check if the user exists through the github api, and filter them based on the result
-                if check_valid_user(user)[0]:
-                    valid.append(user)
-                elif check_valid_user(user)[1] == ERROR:
-                    errors.append(user)
-                else:
-                    invalid.append(user)
-            f.close()
-
-            print(f'Valid: {valid}')
-            print(f'Invalid: {invalid}')
-            print(f'Errors: {errors}')
-
-        # open the file with 'w' to write the existing + new collaborators from scratch
-        with open(path + 'COLLABORATORS', 'w') as f:
-            if '\n' in exisiting_collaborators:
-                exisiting_collaborators.remove('\n')
-            to_add = exisiting_collaborators + valid
-            print(to_add)
-            print('\n'.join(to_add))
-            f.write('\n'.join(to_add))
-            f.close()
-
-            return_message = 'Following users will be added as collaborators: ' + str(
-                valid) + '\nFollowing users were NOT added because they were invalid users: ' + str(invalid)
-            if len(errors) > 0:
-                return valid, (return_message + '\nUsers not added due to errors: ' + str(errors), ERROR)
-            return valid, (return_message, NO_ERROR)
-    except Exception as e:
-        return [], ('add_collaborators: ' + traceback.format_exc(), ERROR)
-
-def git_checkout(local_path, branchname):
-    try:
-        repo = Repo(local_path)
-        # check if branch exists
-        for remote_ref in repo.remotes.origin.refs:
-            upstream, reference = remote_ref.name.split('/')
-            if upstream == UPSTREAM and reference == branchname:
-                repo.git.checkout(branchname)
-                return 'Checked out branch: {}'.format(branchname), NO_ERROR
-
-        # if the branch does not exist, create it
-        repo.git.checkout('-b', branchname)
-        return 'Created and checked out branch: {}'.format(branchname), NO_ERROR
-    except Exception as e:
-        return 'git_checkout: ' + traceback.format_exc(), ERROR
-
-
-def git_push(added_collaborators, branchname):
-    try:
-        if len(added_collaborators) > 0:
-            # initialize the repo and the commit message
-            repo = Repo(LOCAL_PATH)
-            commit_message = 'Added collaborator(s): ' + \
-                ', '.join(added_collaborators)
-
-            # add, commit, and push
-            repo.git.add(update=True)
-            repo.index.commit(commit_message)
-            repo.git.push('--set-upstream', UPSTREAM, branchname)
-            return (
-                'Pushed to remote ({}/{}) with commit message: "{}"'.format(
-                    UPSTREAM, branchname, commit_message),
-                NO_ERROR)
-        else:
-            # no new collaborators to add
-            return 'No collaborators added.', NO_ERROR
-    except Exception as e:
-        return 'git_push: ' + traceback.format_exc(), ERROR
-
-
-def remove_path(path):
-    try:
-        if os.path.exists(path):
-            os.system('rm -rf "{}"'.format(path))
-            return 'Removed path ' + path, NO_ERROR
-        else:
-            return 'No directory to remove.', NO_ERROR
-    except Exception as e:
-        return 'remove_path: ' + traceback.format_exc(), ERROR
-
-
-def git_init(remote, local_path, branchname, collaborators=[]):
-    results = []
-    try:
-        # repo init
-
-        # just to make sure if the previous run was not successful
-        results.append(remove_path(local_path))
-
-        # clones the repo into the local path (ie: /temp)
-        print(f'cloning ')
-        Repo.clone_from(remote, local_path,
-                        env={"GIT_SSH_COMMAND": "ssh -i " + SSH_KEY_PATH})  # change this to your ssh key path
-        
-        # checkout to the branch
-        results.append(git_checkout(local_path, branchname))
-
-        # modifying the collaborators file
-        added, log = add_collaborators(local_path, collaborators)
-        results.append(log)
-
-        # git ops
-        results.append(git_push(added, branchname))
-        results.append(remove_path(local_path))
-    except Exception as e:
-        results.append(('git_init: ' + traceback.format_exc(), ERROR))
-        # in case something goes wrong, remove the directory
-        results.append(remove_path(local_path))
-    finally:
-        for result, status in results:  # debugging purposes
-            if status == ERROR:
-                print('\n{}\n{}{}\n'.format(
-                    EXCEPTION_LOG, result, EXCEPTION_LOG))
-            else:
-                print(result)
-
-
-# OLD: input: python3 test.py <remote (ssh)> <branchname> <collaborators (separated by space)>
-# NEW: input: pipenv run python3 script.py <roaster path>
-def main():
-    EXPECTED_HEADER = ["gh_username", "gh_repo"]
-    if len(sys.argv) < 2:
-        print("Invalid number of arguments expected: script.py <roster path>")
-        exit(-1)
-    roster_path = sys.argv[1]
-    roster_dict = defaultdict(list)
-    with open(roster_path, 'r', encoding='utf-8-sig') as f:
-        csvreader = csv.reader(f)
-        count = 0
-        for row in csvreader:
-            # Verify the header row
-            print(row)
-            if count == 0:
-                for (actual, expected) in zip(row, EXPECTED_HEADER):
-                    if actual != expected:
-                        print(
-                            f'Expected {expected} got {actual} for column name.')
-                        exit(-1)
-                count = count + 1
-                continue
-            try:
-                roster_dict[row[1]].append(row[0])
-            except Exception as e:
-                print(f'Error with row {count}, skipping')
-            count = count + 1
-    print(f'Adding the following users to repos: {roster_dict}')
-    for repo, user_list in roster_dict.items():
-        print(f'Adding users {user_list} to {repo}...')
-        git_init(repo, LOCAL_PATH, 'main', user_list)
-        sleep(10)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/scripts/update-workflows.sh b/scripts/update-workflows.sh
deleted file mode 100644
index 5f01890..0000000
--- a/scripts/update-workflows.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-if [[ ! -d ./template ]];
-then
-    git clone git@github.com:BU-Spark/TEMPLATE-base-repo.git template
-fi
-mkdir working
-cd working
-cp -R ../template .
-git clone $1 dest
-mkdir -p ./dest/.github/workflows
-cp -R ../template/.github/workflows/ ./dest/.github/workflows 
-cd dest
-# Create the collaborators file if it doesn't exist
-if [[ ! -d ./COLLABORATORS ]];
-then
-    touch COLLABORATORS
-fi
-echo "git Status before"
-git status
-git add ./.github/*
-git commit -m "Updated workflows with latest template"
-git push origin
-echo "Git status after"
-git status
-echo "Completed update"
-cd ..
-cd ..
-rm -rf ./working
-echo "Cleanup complete"
\ No newline at end of file

From 443baa56a6eea5640964d01117ac589eede200bc Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Sun, 15 Dec 2024 23:06:17 -0500
Subject: [PATCH 03/18] type err

---
 app/main.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/main.py b/app/main.py
index 7833474..4578dc4 100644
--- a/app/main.py
+++ b/app/main.py
@@ -155,7 +155,7 @@ async def set_projects(request: Request):
     
     results: list = []
     projects: list[tuple[str, str]] = data["projects"]
-    action = data["action"]
+    action: Literal["push", "pull"] = data["action"]
     
     if action not in ['push', 'pull']:
         return {"status": "failed", "error": "action must be 'push' or 'pull'"}
@@ -175,7 +175,10 @@ async def set_projects(request: Request):
                         continue
                     else:
                         db_status, db_msg = db.change_users_project_status(project_name, github_username, action)
-                        results.append(f"PROCESSED: {project_name} - {github_username} -> gh {gh_status} {gh_msg} | db {db_status} {db_msg}")
+                        results.append(
+                            f"PROCESSED: {project_name} - {github_username} 
+                            -> gh {gh_status} {gh_msg} | db {db_status} {db_msg}"
+                        )
                     
             except Exception as e: 
                 print(e)

From f3e42d3aba21cbfeb8bb52c7c8d39f8d06f2222d Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Mon, 16 Dec 2024 02:24:15 -0500
Subject: [PATCH 04/18] clean + create repo api

---
 app/github.py | 81 ++++++++++++++++++++++++++++++++++++---------------
 app/spark.py  | 17 ++++++-----
 2 files changed, 66 insertions(+), 32 deletions(-)

diff --git a/app/github.py b/app/github.py
index 61d216b..08180a8 100644
--- a/app/github.py
+++ b/app/github.py
@@ -1,4 +1,6 @@
-# =========================================== imports =============================================
+# ==========================================================================================================================
+# imports 
+# ==========================================================================================================================
 
 import json
 from typing import Literal, Optional
@@ -6,7 +8,9 @@
 import csv
 import os
 
-# ============================================= Github ============================================
+# ==========================================================================================================================
+# github
+# ==========================================================================================================================
 
 class Github:
     GITHUB_PAT = None
@@ -51,8 +55,7 @@ def check_user_exists(self, user: str) -> bool:
         Returns: bool: True if the user exists, False otherwise.
         """
 
-        response = requests.get(
-            f'https://api.github.com/users/{user}', headers=self.HEADERS, timeout=2)
+        response = requests.get(f'https://api.github.com/users/{user}', headers=self.HEADERS, timeout=2)
         if response.status_code == 200: return True
         else: return False
     
@@ -64,6 +67,7 @@ def check_user_is_collaborator(self, repo_url: str, user: str) -> bool:
             repo_url (str): The URL of the GitHub repository.
             user (str): The username of the GitHub user.
         Returns: bool: True if the user is a collaborator, False otherwise.
+        Raises: Exception: If an error occurs during the API request.
         """
         
         ssh_url = repo_url.replace("https://github.com/", "git@github.com:")
@@ -75,9 +79,7 @@ def check_user_is_collaborator(self, repo_url: str, user: str) -> bool:
                 headers=self.HEADERS,
                 timeout=2
             )
-            
-            if response.status_code == 204: return True
-            else: return False
+            return response.status_code == 204
         except Exception as e:
             raise Exception(f"Failed to check if user is a collaborator: {str(e)}")
     
@@ -90,17 +92,17 @@ def add_user_to_repo(self, repo_url: str, user: str, permission: perms) -> tuple
             user (str): The username of the GitHub user.
             permission ("pull" | "triage" | "push" | "maintain" | "admin"): The permission level for the user.
         Returns: Tuple[int, str]: A tuple containing the status code and message.
+        Raises: Exception: If an error occurs during the API request.
         """
         
         ssh_url = repo_url.replace("https://github.com/", "git@github.com:")
-        username, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
-        exists = self.check_user_exists(user)
+        owner, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
         
-        if not exists: return 404, f"User {user} does not exist"
+        if not self.check_user_exists(user): return 404, f"User {user} does not exist"
         
         try:
             response = requests.put(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators/{user}',
+                f'https://api.github.com/repos/{owner}/{repo_name}/collaborators/{user}',
                 headers=self.HEADERS,
                 json={'permission': permission},
                 timeout=2
@@ -121,11 +123,11 @@ def get_users_on_repo(self, repo_url: str) -> set[str]:
         """
         
         ssh_url = repo_url.replace("https://github.com/", "git@github.com:")
-        username, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
+        owner, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
 
         try:
             collaborators_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/collaborators',
+                f'https://api.github.com/repos/{owner}/{repo_name}/collaborators',
                 headers=self.HEADERS,
                 timeout=10
             )
@@ -139,7 +141,7 @@ def get_users_on_repo(self, repo_url: str) -> set[str]:
         elif collaborators_response.status_code == 403:
             raise Exception("Access to the repository is forbidden.")
         else:
-            raise Exception(f"Failed to fetch collaborators: {collaborators_response.json().get('message', 'Unknown error')}")
+            raise Exception(f"Failed: {collaborators_response.json().get('message', 'Unknown error')}")
     
     def get_users_invited_on_repo(self, repo_url: str, check_expired: bool = False ) -> set[str]:
         """
@@ -153,11 +155,11 @@ def get_users_invited_on_repo(self, repo_url: str, check_expired: bool = False )
         """
         
         ssh_url = repo_url.replace("https://github.com/", "git@github.com:")
-        username, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
+        owner, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
         
         try:
             invitations_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/invitations',
+                f'https://api.github.com/repos/{owner}/{repo_name}/invitations',
                 headers=self.HEADERS,
                 timeout=10
             )
@@ -176,7 +178,7 @@ def get_users_invited_on_repo(self, repo_url: str, check_expired: bool = False )
         elif invitations_response.status_code == 403:
             raise Exception("Access to the repository is forbidden.")
         else:
-            raise Exception(f"Failed to fetch invitations: {invitations_response.json().get('message', 'Unknown error')}")
+            raise Exception(f"Failed: {invitations_response.json().get('message', 'Unknown error')}")
         
     def revoke_user_invitation_on_repo(self, repo_url: str, user: str) -> tuple[int, str]:
         """
@@ -186,14 +188,15 @@ def revoke_user_invitation_on_repo(self, repo_url: str, user: str) -> tuple[int,
             repo_url (str): The URL of the GitHub repository.
             user (str): The username of the GitHub user.
         Returns: Tuple[int, str]: A tuple containing the status code and message.
+        Raises: Exception: If an error occurs during the API request.
         """
         
         ssh_url = repo_url.replace("https://github.com/", "git@github.com:")
-        username, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
+        owner, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
         
         try:
             invitations_response = requests.get(
-                f'https://api.github.com/repos/{username}/{repo_name}/invitations',
+                f'https://api.github.com/repos/{owner}/{repo_name}/invitations',
                 headers=self.HEADERS,
                 timeout=10
             )
@@ -204,7 +207,7 @@ def revoke_user_invitation_on_repo(self, repo_url: str, user: str) -> tuple[int,
             invitation = next((inv for inv in invitations_response.json() if inv['invitee']['login'] == user), None)
             if invitation:
                 response = requests.delete(
-                    f'https://api.github.com/repos/{username}/{repo_name}/invitations/{invitation["id"]}',
+                    f'https://api.github.com/repos/{owner}/{repo_name}/invitations/{invitation["id"]}',
                     headers=self.HEADERS,
                     timeout=2
                 )
@@ -223,6 +226,7 @@ def reinvite_expired_users_on_repo(self, repo_url: str) -> list[tuple[int, str]]
         
         Args: repo_url (str): The HTTPS URL of the GitHub repository.
         Returns: list[tuple[int, str]]: A list of tuples containing the status code and message.
+        Raises: Exception: If an error occurs during the API request.
         """
         
         results = []
@@ -247,6 +251,7 @@ def change_user_permission_on_repo(self, repo_url: str, user: str, permission: p
             user (str): The username of the GitHub user.
             permission ("pull" | "triage" | "push" | "maintain" | "admin"): The new permission level for the user.
         Returns: Tuple[int, str]: A tuple containing the status code and message.
+        Raises: Exception: If an error occurs during the API
         """
         
         try:
@@ -274,7 +279,8 @@ def change_user_permission_on_repo(self, repo_url: str, user: str, permission: p
                 )
                 
                 if change_permission_response.status_code == 200 or change_permission_response.status_code == 204:
-                    return change_permission_response.status_code, f"Successfully changed {user}'s permission to {permission}"
+                    return change_permission_response.status_code, \
+                        f"Successfully changed {user}'s permission to {permission}"
                 else:
                     return change_permission_response.status_code, change_permission_response.json()
                 
@@ -299,7 +305,8 @@ def change_user_permission_on_repo(self, repo_url: str, user: str, permission: p
                     else:
                         return update_invitation_response.status_code, update_invitation_response.json()
                 else:
-                    return collaborator_response.status_code, f'User {user} is not a collaborator or invited on the repository'
+                    return collaborator_response.status_code, \
+                        f'User {user} is not a collaborator or invited on the repository'
 
         except Exception as e:
             return 500, str(e)
@@ -312,6 +319,7 @@ def change_all_user_permission_on_repo(self, repo_url: str, permission: perms) -
             repo_url (str): The HTTPS URL of the GitHub repository.
             permission ("pull" | "triage" | "push" | "maintain" | "admin"): The new permission level for the users.
         Returns: Tuple[int, str]: A tuple containing the status code and message.
+        Raises: Exception: If an error occurs during the API request.
         """
         
         results = []
@@ -332,6 +340,7 @@ def get_all_repos(self) -> list[tuple[str, str]]:
         Retrieves a list of all repositories in the organization.
         
         Returns: list[tuple[str, str]]: A list of all repositories in the organization.
+        Raises: Exception: If an error occurs during the API request.
         """
         
         try:
@@ -340,14 +349,38 @@ def get_all_repos(self) -> list[tuple[str, str]]:
                 headers=self.HEADERS,
                 timeout=10
             )
-            #print(response.json())
             if response.status_code == 200:
                 return [(repo['name'], repo['ssh_url']) for repo in response.json()]
             else:
                 return []
         except Exception as e:
-            return []
+            raise Exception(f"Failed to fetch repositories: {str(e)}")
 
+    def create_repo(self, repo_name: str, private: bool = False) -> tuple[int, str]:
+        """
+        Creates a new repository in the organization.
+        
+        Args: 
+            repo_name (str): The name of the new repository.
+            private (bool): If True, the repository will be private.
+        Returns: Tuple[int, str]: A tuple containing the status code and message.
+        Raises: Exception: If an error occurs during the API request.
+        """
+        
+        try:
+            response = requests.post(
+                f'https://api.github.com/orgs/{self.ORG_NAME}/repos',
+                headers=self.HEADERS,
+                json={'name': repo_name, 'private': private},
+                timeout=10
+            )
+            if response.status_code == 201:
+                return 201, f"Successfully created repository {repo_name}"
+            else:
+                return response.status_code, response.json()
+        except Exception as e:
+            return 500, str(e)
+    
 if __name__ == "__main__":
     #github = Github(GITHUB_PAT, "spark-tests")
     #print(github.change_user_permission_on_repo("https://github.com/spark-tests/initial", "mochiakku", "push"))
diff --git a/app/spark.py b/app/spark.py
index f9cd43d..f94be0d 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -11,9 +11,9 @@
 
 class Spark:
     
-    # =============================================================================================
+    # ======================================================================================================================
     # SQLAlchemy functionality
-    # =============================================================================================
+    # ======================================================================================================================
     
     def __init__(self, URL: str, token: str):
         self.URL = URL
@@ -50,13 +50,13 @@ def run(self, func, *args, **kwargs):
         finally:
             session.close()
             
-    # =============================================================================================
+    # ======================================================================================================================
     # auto spark functionality
-    # =============================================================================================        
+    # ======================================================================================================================        
     
-    # ---------------------------------------------------------------------------------------------
+    # ----------------------------------------------------------------------------------------------------------------------
     # ingestion of csv files into holding tables
-    # ---------------------------------------------------------------------------------------------
+    # ----------------------------------------------------------------------------------------------------------------------
     
     def ingest_csv(self, df: DataFrame, colmap: dict[str, str], table: str):
         """ingest csv"""
@@ -94,12 +94,13 @@ def ingest_user_project_csv(self, df: DataFrame):
 
         self.ingest_csv(df, colmap, "ingest_user_project_csv")
         
-    # ---------------------------------------------------------------------------------------------
+    # ----------------------------------------------------------------------------------------------------------------------
     # processing of csv files into holding tables
-    # ---------------------------------------------------------------------------------------------
+    # ----------------------------------------------------------------------------------------------------------------------
     
     def process_ingest_project_csv(self):
         """Process project csv"""
+        
         with self.scope() as session:
             for row in session.query(IngestProjectCSV).all():
                 

From 434c0b62740f5a4dd15e1011ba08edbfe5197942 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Mon, 16 Dec 2024 03:32:44 -0500
Subject: [PATCH 05/18] rework slacker + add logging

---
 app/github.py  |  12 +++-
 app/log.py     |  71 +++++++++++++++++++++++
 app/slacker.py | 150 +++++++++++++++++++++++++++++--------------------
 app/spark.py   |  13 +++--
 4 files changed, 179 insertions(+), 67 deletions(-)
 create mode 100644 app/log.py

diff --git a/app/github.py b/app/github.py
index 08180a8..c73ce78 100644
--- a/app/github.py
+++ b/app/github.py
@@ -7,6 +7,7 @@
 import requests
 import csv
 import os
+import log
 
 # ==========================================================================================================================
 # github
@@ -26,7 +27,9 @@ def __init__(self, GITHUB_PAT: str, ORG_NAME: str):
             'Authorization': f'Bearer {GITHUB_PAT}',
             'X-GitHub-Api-Version': '2022-11-28'
         }
-        print(f"Github initialized with PAT: {GITHUB_PAT} and {ORG_NAME}")
+        self.log = log.SparkLogger("GITHUB", output=True, persist=True)
+        
+        self.log.warning(f"Github initialized with PAT: {GITHUB_PAT[:20]}... and ORG: {ORG_NAME}")
         
     def extract_user_repo_from_ssh_url(self, ssh_url: str) -> tuple[str, str]:
         """
@@ -375,14 +378,19 @@ def create_repo(self, repo_name: str, private: bool = False) -> tuple[int, str]:
                 timeout=10
             )
             if response.status_code == 201:
+                self.log.info(f"Successfully created repository {repo_name}")
                 return 201, f"Successfully created repository {repo_name}"
             else:
+                self.log.error(f"Failed to create repository {repo_name}: {response.json()}")
                 return response.status_code, response.json()
         except Exception as e:
+            self.log.error(f"Failed to create repository {repo_name}: {str(e)}")
             return 500, str(e)
     
 if __name__ == "__main__":
-    #github = Github(GITHUB_PAT, "spark-tests")
+    TEST_GITHUB_PAT = os.getenv("TEST_GITHUB_PAT") or ""
+    github = Github(TEST_GITHUB_PAT, "auto-spark")
+    github.create_repo("test", private=False)
     #print(github.change_user_permission_on_repo("https://github.com/spark-tests/initial", "mochiakku", "push"))
     #print(github.change_all_user_permission_on_repo("https://github.com/spark-tests/initial", "push"))
     #print(github.get_all_repos())
diff --git a/app/log.py b/app/log.py
new file mode 100644
index 0000000..965413e
--- /dev/null
+++ b/app/log.py
@@ -0,0 +1,71 @@
+import logging
+import sys
+from typing import Optional
+
+class SparkLogger:
+    _loggers = {}
+
+    def __init__(self, name: str, level: int = logging.DEBUG, output: bool = True, persist: bool = False):
+        """
+        Initialize the SparkLogger.
+
+        Args:
+            name (str): The name of the logger.
+            level (int, optional): The minimum log level. Defaults to logging.DEBUG.
+            output (bool, optional): Whether to log to the console. Defaults to True.
+            persist (bool, optional): Whether to log to a file. Defaults to False.
+        """
+        self.name = name
+
+        if name in SparkLogger._loggers:
+            self.logger = SparkLogger._loggers[name]
+            self.logger.setLevel(level)
+        else:
+            self.logger = logging.getLogger(name)
+            self.logger.setLevel(level)
+
+            
+            lfmt = f'[{name.upper()}][%(asctime)s][%(levelname)s] --- %(message)s'
+            dfmt = '%Y-%m-%d %H:%M:%S'
+            formatter = logging.Formatter(fmt=lfmt, datefmt=dfmt)
+
+            if output:
+                console_handler = logging.StreamHandler(sys.stdout)
+                console_handler.setLevel(level)
+                console_handler.setFormatter(formatter)
+                self.logger.addHandler(console_handler)
+
+            if persist:
+                file_handler = logging.FileHandler(f"./logs/{name}.log")
+                file_handler.setLevel(level)
+                file_handler.setFormatter(formatter)
+                self.logger.addHandler(file_handler)
+
+            self.logger.propagate = False
+
+            SparkLogger._loggers[name] = self.logger
+
+    def debug(self, message: str):
+        self.logger.debug(message)
+
+    def info(self, message: str):
+        self.logger.info(message)
+
+    def warning(self, message: str):
+        self.logger.warning(message)
+
+    def error(self, message: str):
+        self.logger.error(message)
+
+    def critical(self, message: str):
+        self.logger.critical(message)
+
+    def change(self, level: int):
+        """
+        Set a new log level for the logger and all its handlers.
+
+        Args:
+            level (int): The new logging level (e.g., logging.INFO).
+        """
+        self.logger.setLevel(level)
+        for handler in self.logger.handlers: handler.setLevel(level)
diff --git a/app/slacker.py b/app/slacker.py
index 0eb9afd..0b58fb0 100644
--- a/app/slacker.py
+++ b/app/slacker.py
@@ -1,83 +1,114 @@
 import os
 import time
 import math
+from typing import List
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 from dotenv import load_dotenv
+import log
 
 load_dotenv()
 
 class Slacker:
-    def __init__(self, token: str = ""):
+    def __init__(self, token: str):
         self.token = token
-        if not self.token: raise ValueError("NO TOKEN PROVIDED")
         self.client = WebClient(token=self.token)
+        self.log = log.SparkLogger(name="Slacker", output=True, persist=True)
 
     def get_user_id(self, email: str) -> str:
+        """
+        Fetches the user ID associated with the given email address.
+        
+        Args: email (str): The email address of the user.
+        Returns: str: The user ID if found, otherwise an empty string.
+        Raises: SlackApiError: If the API call fails.
+        """
+        
         try:
             response = self.client.users_lookupByEmail(email=email)
-            user_id = response['user']['id']
-            print(f"Found user '{email}' with ID: {user_id}")
-            return user_id
+            uid = response['user']['id']
+            self.log.info(f"fetched slack uid {uid} for email '{email}'.")
+            return uid
         except SlackApiError as e:
-            if e.response['error'] == 'users_not_found':
-                print(f"User with email '{email}' not found.")
-            else:
-                print(f"Error fetching user '{email}': {e.response['error']}")
-            return ""
+            self.log.error(f"failed to fetch slack uid for email '{email}': {e.response['error']}")
+            raise e
 
     def create_channel(self, channel_name: str, is_private: bool = False) -> str:
+        """
+        Creates a Slack channel with the given name.
+        
+        Args: 
+            channel_name (str): The name of the channel.
+            is_private (bool, optional): Whether the channel is private. Defaults to False.
+        Returns: str: The ID of the created channel if successful, otherwise an empty string.    
+        Raises: SlackApiError: If the API call fails.
+        """
+        
         try:
             response = self.client.conversations_create(
                 name=channel_name,
                 is_private=is_private
             )
             channel_id = response['channel']['id']
-            print(f"Channel '{channel_name}' created with ID: {channel_id}")
+            self.log.info(f"created channel '{channel_name}' with ID: {channel_id}")
             return channel_id
         except SlackApiError as e:
             if e.response['error'] == 'name_taken':
-                print(f"Channel '{channel_name}' already exists.")
+                self.log.warning(f"channel '{channel_name}' already exists.")
                 existing_channel = self.get_channel_id(channel_name)
                 return existing_channel
             else:
-                print(f"Failed to create channel '{channel_name}': {e.response['error']}")
-                return ""
+                self.log.error(f"failed to create channel '{channel_name}': {e.response['error']}")
+                raise e
 
     def get_channel_id(self, channel_name: str) -> str:
+        """
+        Fetches the ID of an existing channel with the given name.
+        
+        Args: channel_name (str): The name of the channel.
+        Returns: str: The ID of the channel if found, otherwise an empty string.
+        Raises: SlackApiError: If the API call fails.
+        """
+        
         try:
             response = self.client.conversations_list(types="public_channel,private_channel", limit=1000)
-            channels = response['channels']
-            for channel in channels:
-                if channel['name'] == channel_name:
-                    print(f"Found existing channel '{channel_name}' with ID: {channel['id']}")
-                    return channel['id']
-            print(f"Channel '{channel_name}' not found.")
-            return ""
+            channel_id = next((c['id'] for c in response['channels'] if c['name'] == channel_name), "")
+            if channel_id:
+                self.log.info(f"fetched channel ID {channel_id} for '{channel_name}'.")
+                return channel_id
+            else:
+                self.log.warning(f"channel '{channel_name}' not found.")
+                raise SlackApiError(f"Channel '{channel_name}' not found.", response)
         except SlackApiError as e:
-            print(f"Error fetching channels: {e.response['error']}")
-            return ""
+            self.log.error(f"failed to fetch channel ID for '{channel_name}': {e.response['error']}")
+            raise e
 
-    def invite_users_to_channel(self, channel_id: str, user_ids: list, retry_count: int = 0):
-        MAX_RETRIES = 5
+    def invite_users_to_channel(self, channel_id: str, user_ids: List[str] | str, retries: int = 0):
+        """
+        Invites users to a Slack channel.
+        
+        Args:
+            channel_id (str): The ID of the channel.
+            user_ids (List[str] | str): A list of user IDs or a single user ID.
+            retries (int, optional): The number of retries. Defaults to 0.
+        Raises: SlackApiError: If the API call fails.
+        """
+        
         try:
-            response = self.client.conversations_invite(
+            self.client.conversations_invite(
                 channel=channel_id,
-                users=user_ids  # Can be a list or comma-separated string
+                users=user_ids 
             )
+            self.log.info(f"invited users to channel ID {channel_id}.")
         except SlackApiError as e:
-            if e.response['error'] == 'already_in_channel':
-                print(f"Some users are already in the channel ID {channel_id}.")
-            elif e.response['error'] == 'user_not_found':
-                print("One or more users not found.")
-            elif e.response['error'] == 'rate_limited' and retry_count < MAX_RETRIES:
-                retry_after = int(e.response.headers.get('Retry-After', 1))
-                backoff_time = retry_after * math.pow(2, retry_count)
-                print(f"Rate limited. Retrying after {backoff_time} seconds.")
-                time.sleep(backoff_time)
-                self.invite_users_to_channel(channel_id, user_ids, retry_count + 1)
+            if e.response['error'] == 'rate_limited' and retries < 5:
+                backoff = int(e.response.headers.get('Retry-After', 1)) * math.pow(2, retries)
+                self.log.warning(f"inviting users rate limited. retrying in {backoff} seconds.")
+                time.sleep(backoff)
+                self.invite_users_to_channel(channel_id, user_ids, retries + 1)
             else:
-                print(f"Failed to invite users to channel ID {channel_id}: {e.response['error']}")
+                self.log.error(f"failed to invite users to channel ID {channel_id}: {e.response['error']}")
+                raise e
 
     def create_channels_and_add_users(self, channels_dict: dict, is_private: bool = False) -> list:
         """
@@ -89,33 +120,30 @@ def create_channels_and_add_users(self, channels_dict: dict, is_private: bool =
 
         Returns:
             list: A list of dictionaries containing channel names and their corresponding IDs.
+        Raises: SlackApiError: If the API call fails.
         """
-        created_channels = []
-        email_to_user_id = {}
-
-        # Extract unique emails to minimize API calls
-        unique_emails = set(email for users in channels_dict.values() for email in users)
-        print("Mapping emails to user IDs...")
-        for email in unique_emails:
-            user_id = self.get_user_id(email)
-            if user_id:
-                email_to_user_id[email] = user_id
-
-        print("\nCreating channels and inviting users...")
-        # Create channels and invite users
-        for channel_name, user_emails in channels_dict.items():
-            print(f"\nProcessing channel: {channel_name}")
-            channel_id = self.create_channel(channel_name, is_private)
-            if channel_id:
-                # Retrieve user IDs for the current channel
+        try:
+            created_channels = []
+            email_to_user_id = {
+                email: self.get_user_id(email)
+                for email in {email for users in channels_dict.values() for email in users}
+            }
+            self.log.info(f"email to user ID mapping: {email_to_user_id}")
+            
+            for channel_name, user_emails in channels_dict.items():
+                channel_id = self.create_channel(channel_name, is_private)
                 user_ids = [email_to_user_id[email] for email in user_emails if email in email_to_user_id]
-                if user_ids:
-                    self.invite_users_to_channel(channel_id, user_ids)
-                else:
-                    print(f"No valid users to invite for channel '{channel_name}'.")
+                self.invite_users_to_channel(channel_id, user_ids)
                 created_channels.append({'name': channel_name, 'id': channel_id})
 
-        return created_channels
+            return created_channels
+        
+        except SlackApiError as e:
+            self.log.error(f"failed to create channels and add users: {e.response['error']}")
+            raise e
+        except Exception as e:
+            self.log.error(f"failed to create channels and add users: {e}")
+            raise e
 
 
 if __name__ == "__main__":
diff --git a/app/spark.py b/app/spark.py
index f94be0d..964f182 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -8,6 +8,7 @@
 from sqlalchemy.orm import sessionmaker
 from contextlib import contextmanager
 from pandas import DataFrame
+import github as git
 
 class Spark:
     
@@ -15,10 +16,11 @@ class Spark:
     # SQLAlchemy functionality
     # ======================================================================================================================
     
-    def __init__(self, URL: str, token: str):
+    def __init__(self, URL: str, slack_token: str, git: git.Github):
         self.URL = URL
         self.engine = create_engine(self.URL, echo=False)
-        self.slacker = Slacker(token)
+        self.slacker = Slacker(slack_token)
+        self.git = git
         
     def s(self):
         return sessionmaker(bind=self.engine)()
@@ -132,7 +134,10 @@ def process_ingest_project_csv(self):
 if __name__ == "__main__":
     TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
     TEST_SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
-    spark = Spark(TEST_POSTGRES, TEST_SLACK_TOKEN)
+    TEST_GITHUB_ORG = "auto-spark"
+    TEST_GITHUB_TOKEN = os.getenv("TEST_GITHUB_PAT") or ""
+    github = git.Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
+    spark = Spark(TEST_POSTGRES, TEST_SLACK_TOKEN, github)
     #df = pd.read_csv("./ingestprojects.csv")
     #spark.ingest_project_csv(df)
-    spark.process_ingest_project_csv()
\ No newline at end of file
+    #spark.process_ingest_project_csv()
\ No newline at end of file

From dc3d62c0b151c08e3045cc584ad8265b71958de9 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Mon, 16 Dec 2024 03:37:32 -0500
Subject: [PATCH 06/18] generation

---
 app/slacker.py |  9 +++++----
 app/spark.py   | 15 ++++++++-------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/app/slacker.py b/app/slacker.py
index 0b58fb0..5aa6cb7 100644
--- a/app/slacker.py
+++ b/app/slacker.py
@@ -7,8 +7,6 @@
 from dotenv import load_dotenv
 import log
 
-load_dotenv()
-
 class Slacker:
     def __init__(self, token: str):
         self.token = token
@@ -145,10 +143,13 @@ def create_channels_and_add_users(self, channels_dict: dict, is_private: bool =
             self.log.error(f"failed to create channels and add users: {e}")
             raise e
 
-
 if __name__ == "__main__":
+    load_dotenv()
+    
     slacker = Slacker(token=os.getenv('SLACK_BOT_TOKEN') or "")
     channels_dict = {
-        'x4': ["x@bu.edu"],
+        'channel1': ["x@bu.edu"],
+        'channel2': ["y@bu.edu"],
+        'channel3': ["z@bu.edu"]
     }
     created_channels = slacker.create_channels_and_add_users(channels_dict, is_private=False)
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index 964f182..adb8aee 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -8,7 +8,7 @@
 from sqlalchemy.orm import sessionmaker
 from contextlib import contextmanager
 from pandas import DataFrame
-import github as git
+from github import Github
 
 class Spark:
     
@@ -16,10 +16,10 @@ class Spark:
     # SQLAlchemy functionality
     # ======================================================================================================================
     
-    def __init__(self, URL: str, slack_token: str, git: git.Github):
+    def __init__(self, URL: str, slacker: Slacker, git: Github):
         self.URL = URL
         self.engine = create_engine(self.URL, echo=False)
-        self.slacker = Slacker(slack_token)
+        self.slacker = slacker
         self.git = git
         
     def s(self):
@@ -123,8 +123,7 @@ def process_ingest_project_csv(self):
                 session.add(project)
                 
                 if row.generate_github:
-                    # generate github
-                    pass
+                    self.git.create_repo(row.project_tag, private=True)
                 if row.generate_slack:
                     channel_name = row.project_tag
                     channel_id = self.slacker.create_channel(channel_name=channel_name, is_private=False)
@@ -136,8 +135,10 @@ def process_ingest_project_csv(self):
     TEST_SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
     TEST_GITHUB_ORG = "auto-spark"
     TEST_GITHUB_TOKEN = os.getenv("TEST_GITHUB_PAT") or ""
-    github = git.Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
-    spark = Spark(TEST_POSTGRES, TEST_SLACK_TOKEN, github)
+    
+    github = Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
+    slacker = Slacker(TEST_SLACK_TOKEN)
+    spark = Spark(TEST_POSTGRES, slacker, github)
     #df = pd.read_csv("./ingestprojects.csv")
     #spark.ingest_project_csv(df)
     #spark.process_ingest_project_csv()
\ No newline at end of file

From f608b193428105a1fcee2f29fd78e3758c770aba Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Tue, 17 Dec 2024 01:05:48 -0500
Subject: [PATCH 07/18] full ingest project

---
 app/models.py | 15 +++++++-
 app/spark.py  | 94 ++++++++++++++++++++++++++++++++++++++-------------
 2 files changed, 85 insertions(+), 24 deletions(-)

diff --git a/app/models.py b/app/models.py
index 30cc3f5..10b978c 100644
--- a/app/models.py
+++ b/app/models.py
@@ -39,6 +39,12 @@ class Status(enum.Enum):
     pull = 'pull'
     push = 'push'
 
+class Outcome(enum.Enum):
+    success = 'success'
+    failure = 'failure'
+    warning = 'warning'
+    unkown = 'unkown'
+
 class Semester(Base):
     __tablename__ = 'semester'
 
@@ -60,6 +66,10 @@ def __repr__(self):
             f"semester_name='{self.semester_name}', "
             f"year={self.year}, season={self.season.value})>"
         )
+        
+    def short(self):
+        lookup ={'spring': 'sp','summer': 'su','fall': 'fa','winter': 'wi'}
+        return f"{lookup[self.season.value]}-{str(self.year)[-2:]}"
 
 class User(Base):
     __tablename__ = 'user'
@@ -150,11 +160,14 @@ class IngestProjectCSV(Base):
     slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
     generate_github: Mapped[Optional[bool]] = mapped_column(Integer, nullable=True, default=False)
     generate_slack: Mapped[Optional[bool]] = mapped_column(Integer, nullable=True, default=False)
+    outcome: Mapped[Optional[Outcome]] = mapped_column(Enum_(Outcome), nullable=True)
+    result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     
     def __repr__(self) -> str:
         return (
             f"<IngestProjectCSV(id={self.id}, project_name='{self.project_name}', "
             f"project_tag='{self.project_tag}', semester='{self.semester}', "
             f"github_url='{self.github_url}', slack_channel='{self.slack_channel}', "
-            f"generate_github={self.generate_github}, generate_slack={self.generate_slack})>"
+            f"generate_github={self.generate_github}, generate_slack={self.generate_slack}, "
+            f"outcome={self.outcome}, result='{self.result}')>"
         )
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index adb8aee..df9ff72 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -1,14 +1,17 @@
 import os
-from models import User, Project, Base, IngestProjectCSV, Semester, UserProject
+from typing import Any, Generator
 from schema import _Project
+from models import User, Project, Base, IngestProjectCSV, Semester, UserProject, Outcome
 from slacker import Slacker
 from sqlalchemy import create_engine
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 from sqlalchemy.orm import sessionmaker
 from contextlib import contextmanager
+import pandas as pd
 from pandas import DataFrame
 from github import Github
+import log
 
 class Spark:
     
@@ -16,17 +19,19 @@ class Spark:
     # SQLAlchemy functionality
     # ======================================================================================================================
     
-    def __init__(self, URL: str, slacker: Slacker, git: Github):
-        self.URL = URL
-        self.engine = create_engine(self.URL, echo=False)
+    def __init__(self, PGURL: str, org: str, slacker: Slacker, git: Github):
+        self.PGURL = PGURL
+        self.org = org
+        self.engine = create_engine(self.PGURL, echo=False)
         self.slacker = slacker
         self.git = git
+        self.log = log.SparkLogger(name="Spark", output=True, persist=True)
         
-    def s(self):
+    def s(self) -> Session:
         return sessionmaker(bind=self.engine)()
         
     @contextmanager
-    def scope(self):
+    def scope(self) -> Generator[Session, Any, None]:
         session = self.s()
         try:
             yield session
@@ -69,8 +74,14 @@ def ingest_csv(self, df: DataFrame, colmap: dict[str, str], table: str):
                 if csv_col != db_col and csv_col in df.columns
         }
         if tomap: df = df.rename(columns=tomap)
-    
-        if self.engine: df.to_sql(table, self.engine, if_exists='append', index=False)
+        if not self.engine: raise Exception("no engine.")
+        for _, row in df.iterrows():
+            try:
+                row_df = pd.DataFrame([row])
+                row_df.to_sql(table, self.engine, if_exists='append', index=False)
+            except Exception as e:
+                self.log.error(f"failure ingesting row into {table}: {e}")
+                continue
     
     def ingest_project_csv(self, df: DataFrame):
         """ingest project csv"""
@@ -103,15 +114,48 @@ def ingest_user_project_csv(self, df: DataFrame):
     def process_ingest_project_csv(self):
         """Process project csv"""
         
-        with self.scope() as session:
-            for row in session.query(IngestProjectCSV).all():
-                
-                semester = session.query(Semester).filter(Semester.semester_name == row.semester.lower()).first()
-                
+        session = self.s()
+            
+        semesters = {semester.semester_name.lower(): semester for semester in session.query(Semester).all()}
+        
+        for row in session.query(IngestProjectCSV).all():
+            self.log.info(f"processing project csv row {row.project_tag}...")
+            try:
+                results = []
+
+                semester = semesters.get(row.semester.lower())
                 if not semester:
-                    print(f"Semester {row.semester} not found")
+                    row.outcome = Outcome.failure
+                    row.result = "failure: semester discrepancy."
+                    self.log.error(f"failure processing project csv row {row.project_tag}: semester discrepancy.")
                     continue
                 
+                if row.generate_github:
+                    if not row.github_url:
+                        self.git.create_repo(row.project_tag)
+                        row.github_url = f"https://github.com/{self.org}/{row.project_tag}"
+                        self.log.info(f"created github repo {row.github_url} for project {row.project_tag}.")
+                    else:
+                        row.outcome = Outcome.warning
+                        results.append("warning: github repo already exists.")
+                        self.log.warning(f"skipping creating repo for {row.project_tag}: repo already exists.")
+                
+                if row.generate_slack:
+                    if not row.slack_channel:
+                        slack_channel_id = self.slacker.create_channel(row.project_tag)
+                        row.slack_channel = slack_channel_id
+                        self.log.info(f"created slack channel {slack_channel_id} for project {row.project_tag}.")
+                    else:
+                        row.outcome = Outcome.warning
+                        results.append("warning: slack channel already exists.")
+                        self.log.warning(f"skipping creating slack for {row.project_tag}: channel already exists.")
+
+                if results: 
+                    row.result = " <> ".join(results)
+                else: 
+                    row.outcome = Outcome.success
+                    row.result = "all systems operational."
+                
                 project_data = _Project(
                     project_name=row.project_name,
                     project_tag=row.project_tag,
@@ -122,13 +166,17 @@ def process_ingest_project_csv(self):
                 project = Project(**project_data.model_dump())
                 session.add(project)
                 
-                if row.generate_github:
-                    self.git.create_repo(row.project_tag, private=True)
-                if row.generate_slack:
-                    channel_name = row.project_tag
-                    channel_id = self.slacker.create_channel(channel_name=channel_name, is_private=False)
-                    
-            session.commit()
+                session.commit()
+            
+            except (IntegrityError, Exception) as e:
+                session.rollback()
+                msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
+                session.query(IngestProjectCSV).filter(IngestProjectCSV.id == row.id).update({
+                    IngestProjectCSV.outcome: Outcome.failure,
+                    IngestProjectCSV.result: f"failure: {msg}"
+                })
+                session.commit()
+                self.log.error(f"failure processing project csv row {row.project_tag}: {msg}")
         
 if __name__ == "__main__":
     TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
@@ -138,7 +186,7 @@ def process_ingest_project_csv(self):
     
     github = Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
     slacker = Slacker(TEST_SLACK_TOKEN)
-    spark = Spark(TEST_POSTGRES, slacker, github)
+    spark = Spark(TEST_POSTGRES, TEST_GITHUB_ORG, slacker, github)
     #df = pd.read_csv("./ingestprojects.csv")
     #spark.ingest_project_csv(df)
-    #spark.process_ingest_project_csv()
\ No newline at end of file
+    spark.process_ingest_project_csv()
\ No newline at end of file

From c001ab3210c6c6b279e49661a7b8733db57ff3d1 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Tue, 17 Dec 2024 03:20:24 -0500
Subject: [PATCH 08/18] ingest updates

---
 app/models.py | 12 +++++++-----
 app/schema.py |  1 +
 app/spark.py  |  4 +++-
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/app/models.py b/app/models.py
index 10b978c..f38f602 100644
--- a/app/models.py
+++ b/app/models.py
@@ -102,6 +102,7 @@ class Project(Base):
         Integer, primary_key=True, nullable=False,
         #server_default=func.nextval('project_project_id_seq')
     )
+    course: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     project_name: Mapped[str] = mapped_column(Text, nullable=False)
     project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
     semester_id: Mapped[int] = mapped_column(ForeignKey('semester.semester_id', ondelete='RESTRICT'), nullable=False)
@@ -153,6 +154,7 @@ class IngestProjectCSV(Base):
         Integer, primary_key=True, nullable=False,
         #server_default=func.nextval('ingest_project_csv_id_seq')
     )
+    course: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     project_name: Mapped[str] = mapped_column(Text, nullable=False)
     project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
     semester: Mapped[str] = mapped_column(Text, nullable=False)
@@ -165,9 +167,9 @@ class IngestProjectCSV(Base):
     
     def __repr__(self) -> str:
         return (
-            f"<IngestProjectCSV(id={self.id}, project_name='{self.project_name}', "
-            f"project_tag='{self.project_tag}', semester='{self.semester}', "
-            f"github_url='{self.github_url}', slack_channel='{self.slack_channel}', "
-            f"generate_github={self.generate_github}, generate_slack={self.generate_slack}, "
-            f"outcome={self.outcome}, result='{self.result}')>"
+            f"<IngestProjectCSV(id={self.id}, course='{self.course}', "
+            f"project_name='{self.project_name}', project_tag='{self.project_tag}', "
+            f"semester='{self.semester}', github_url='{self.github_url}', "
+            f"slack_channel='{self.slack_channel}', generate_github={self.generate_github}, "
+            f"generate_slack={self.generate_slack}, outcome={self.outcome}, result='{self.result}')>"
         )
\ No newline at end of file
diff --git a/app/schema.py b/app/schema.py
index 21293d2..a27c187 100644
--- a/app/schema.py
+++ b/app/schema.py
@@ -2,6 +2,7 @@
 from pydantic import BaseModel
 
 class _Project(BaseModel):
+    course: Optional[str]
     project_name: str
     project_tag: str
     semester_id: int
diff --git a/app/spark.py b/app/spark.py
index df9ff72..8cc30b6 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -87,8 +87,9 @@ def ingest_project_csv(self, df: DataFrame):
         """ingest project csv"""
         
         colmap: dict[str, str] = {
+            "Course": "course",
             "Project Name": "project_name",
-            "Project Tag": "project_tag",
+            "Project Identifier": "project_tag",
             "Semester": "semester",
             "GitHub Repository": "github_url",
             "Slack": "slack_channel",
@@ -157,6 +158,7 @@ def process_ingest_project_csv(self):
                     row.result = "all systems operational."
                 
                 project_data = _Project(
+                    course=None,
                     project_name=row.project_name,
                     project_tag=row.project_tag,
                     semester_id=semester.semester_id,

From 516114b2413fc3059ac750135683076ccb3f631e Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Tue, 17 Dec 2024 03:24:10 -0500
Subject: [PATCH 09/18] ingest user project model

---
 app/models.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/app/models.py b/app/models.py
index f38f602..1ba223d 100644
--- a/app/models.py
+++ b/app/models.py
@@ -172,4 +172,29 @@ def __repr__(self) -> str:
             f"semester='{self.semester}', github_url='{self.github_url}', "
             f"slack_channel='{self.slack_channel}', generate_github={self.generate_github}, "
             f"generate_slack={self.generate_slack}, outcome={self.outcome}, result='{self.result}')>"
+        )
+        
+class IngestUserProjectCSV(Base):
+    __tablename__ = "ingest_user_project_csv"
+    
+    id: Mapped[int] = mapped_column(
+        Integer, primary_key=True, nullable=False,
+        #server_default=func.nextval('ingest_user_project_csv_id_seq')
+    )
+    project_name: Mapped[str] = mapped_column(Text, nullable=False)
+    project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    first_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    last_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    email: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    buid: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    github_username: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    outcome: Mapped[Optional[Outcome]] = mapped_column(Enum_(Outcome), nullable=True)
+    result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    
+    def __repr__(self) -> str:
+        return (
+            f"<IngestUserProjectCSV(id={self.id}, project_name='{self.project_name}', "
+            f"project_tag='{self.project_tag}', first_name='{self.first_name}', "
+            f"last_name='{self.last_name}', email='{self.email}', buid='{self.buid}', "
+            f"github_username='{self.github_username}', outcome={self.outcome}, result='{self.result}')>"
         )
\ No newline at end of file

From edd5ddcaf57d7b588e5ad36bf69dea35f213748c Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 18 Dec 2024 02:23:29 -0500
Subject: [PATCH 10/18] massive backend rework

---
 .gitignore                                    |   2 +
 Pipfile                                       |   1 +
 Pipfile.lock                                  | 106 ++++++-
 app/alembic/README                            |   1 +
 app/alembic/env.py                            |  77 +++++
 app/alembic/script.py.mako                    |  26 ++
 .../0481276c6c83_fix_unique_constraints.py    |  30 ++
 ...a_make_columns_nullable_and_add_unique_.py |  56 ++++
 ...4fce2a59224_change_null_constraints_on_.py |  46 +++
 .../b0a85b0a1488_add_unique_constraint_to_.py |  31 ++
 app/github.py                                 |  65 +++-
 app/models.py                                 |  70 +++--
 app/schema.py                                 |  15 +-
 app/spark.py                                  | 287 ++++++++++++++++--
 14 files changed, 754 insertions(+), 59 deletions(-)
 create mode 100644 app/alembic/README
 create mode 100644 app/alembic/env.py
 create mode 100644 app/alembic/script.py.mako
 create mode 100644 app/alembic/versions/0481276c6c83_fix_unique_constraints.py
 create mode 100644 app/alembic/versions/1bba57a81eea_make_columns_nullable_and_add_unique_.py
 create mode 100644 app/alembic/versions/94fce2a59224_change_null_constraints_on_.py
 create mode 100644 app/alembic/versions/b0a85b0a1488_add_unique_constraint_to_.py

diff --git a/.gitignore b/.gitignore
index 915e930..96df428 100644
--- a/.gitignore
+++ b/.gitignore
@@ -186,3 +186,5 @@ fabric.properties
 
 # idea folder, uncomment if you don't need it
 .idea
+
+alembic.ini
\ No newline at end of file
diff --git a/Pipfile b/Pipfile
index f80759c..85814c1 100644
--- a/Pipfile
+++ b/Pipfile
@@ -17,6 +17,7 @@ slack-sdk = "*"
 sqlmodel = "*"
 pyright = "*"
 mypy = "*"
+alembic = "*"
 
 [dev-packages]
 
diff --git a/Pipfile.lock b/Pipfile.lock
index 9bca9ee..c207414 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "7f7e404f5eda7dd8c781ccc1a8ed37eaaf6cce24cb3caa7c2813e1c3544f2e15"
+            "sha256": "981a28803eadd385294cc8a31e056fd46c300d240fa84593cedef28e541c18cb"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -34,6 +34,15 @@
             ],
             "version": "==0.8.2"
         },
+        "alembic": {
+            "hashes": [
+                "sha256:99bd884ca390466db5e27ffccff1d179ec5c05c965cfefc0607e69f9e411cb25",
+                "sha256:b00892b53b3642d0b8dbedba234dbf1924b69be83a9a769d5a624b01094e304b"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==1.14.0"
+        },
         "annotated-types": {
             "hashes": [
                 "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53",
@@ -314,6 +323,81 @@
             "markers": "python_version >= '3.6'",
             "version": "==3.10"
         },
+        "mako": {
+            "hashes": [
+                "sha256:42f48953c7eb91332040ff567eb7eea69b22e7a4affbc5ba8e845e8f730f6627",
+                "sha256:577b97e414580d3e088d47c2dbbe9594aa7a5146ed2875d4dfa9075af2dd3cc8"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.3.8"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4",
+                "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30",
+                "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0",
+                "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9",
+                "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396",
+                "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13",
+                "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028",
+                "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca",
+                "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557",
+                "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832",
+                "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0",
+                "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b",
+                "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579",
+                "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a",
+                "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c",
+                "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff",
+                "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c",
+                "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22",
+                "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094",
+                "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb",
+                "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e",
+                "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5",
+                "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a",
+                "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d",
+                "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a",
+                "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b",
+                "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8",
+                "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225",
+                "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c",
+                "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144",
+                "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f",
+                "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87",
+                "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d",
+                "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93",
+                "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf",
+                "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158",
+                "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84",
+                "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb",
+                "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48",
+                "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171",
+                "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c",
+                "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6",
+                "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd",
+                "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d",
+                "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1",
+                "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d",
+                "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca",
+                "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a",
+                "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29",
+                "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe",
+                "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798",
+                "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c",
+                "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8",
+                "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f",
+                "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f",
+                "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a",
+                "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178",
+                "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0",
+                "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79",
+                "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430",
+                "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==3.0.2"
+        },
         "mypy": {
             "hashes": [
                 "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc",
@@ -695,12 +779,12 @@
         },
         "python-multipart": {
             "hashes": [
-                "sha256:905502ef39050557b7a6af411f454bc19526529ca46ae6831508438890ce12cc",
-                "sha256:f8d5b0b9c618575bf9df01c684ded1d94a338839bdd8223838afacfb4bb2082d"
+                "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104",
+                "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.0.19"
+            "version": "==0.0.20"
         },
         "pytz": {
             "hashes": [
@@ -735,12 +819,12 @@
         },
         "slack-sdk": {
             "hashes": [
-                "sha256:a5e74c00c99dc844ad93e501ab764a20d86fa8184bbc9432af217496f632c4ee",
-                "sha256:b8cccadfa3d4005a5e6529f52000d25c583f46173fda8e9136fdd2bc58923ff6"
+                "sha256:c61f57f310d85be83466db5a98ab6ae3bb2e5587437b54fa0daa8fae6a0feffa",
+                "sha256:ff61db7012160eed742285ea91f11c72b7a38a6500a7f6c5335662b4bc6b853d"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.6'",
-            "version": "==3.33.5"
+            "version": "==3.34.0"
         },
         "smmap": {
             "hashes": [
@@ -902,12 +986,12 @@
         },
         "uvicorn": {
             "hashes": [
-                "sha256:2c30de4aeea83661a520abab179b24084a0019c0c1bbe137e5409f741cbde5f8",
-                "sha256:3577119f82b7091cf4d3d4177bfda0bae4723ed92ab1439e8d779de880c9cc59"
+                "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4",
+                "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==0.33.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.34.0"
         }
     },
     "develop": {}
diff --git a/app/alembic/README b/app/alembic/README
new file mode 100644
index 0000000..98e4f9c
--- /dev/null
+++ b/app/alembic/README
@@ -0,0 +1 @@
+Generic single-database configuration.
\ No newline at end of file
diff --git a/app/alembic/env.py b/app/alembic/env.py
new file mode 100644
index 0000000..dddd356
--- /dev/null
+++ b/app/alembic/env.py
@@ -0,0 +1,77 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+from models import Base
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/app/alembic/script.py.mako b/app/alembic/script.py.mako
new file mode 100644
index 0000000..fbc4b07
--- /dev/null
+++ b/app/alembic/script.py.mako
@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}
diff --git a/app/alembic/versions/0481276c6c83_fix_unique_constraints.py b/app/alembic/versions/0481276c6c83_fix_unique_constraints.py
new file mode 100644
index 0000000..4186421
--- /dev/null
+++ b/app/alembic/versions/0481276c6c83_fix_unique_constraints.py
@@ -0,0 +1,30 @@
+"""Fix unique constraints
+
+Revision ID: 0481276c6c83
+Revises: 94fce2a59224
+Create Date: 2024-12-17 21:12:10.001325
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '0481276c6c83'
+down_revision: Union[str, None] = '94fce2a59224'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('unique_project_email', 'ingest_user_project_csv', type_='unique')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_unique_constraint('unique_project_email', 'ingest_user_project_csv', ['project_name', 'project_tag', 'first_name', 'last_name', 'email', 'buid', 'github_username'])
+    # ### end Alembic commands ###
diff --git a/app/alembic/versions/1bba57a81eea_make_columns_nullable_and_add_unique_.py b/app/alembic/versions/1bba57a81eea_make_columns_nullable_and_add_unique_.py
new file mode 100644
index 0000000..1b31f31
--- /dev/null
+++ b/app/alembic/versions/1bba57a81eea_make_columns_nullable_and_add_unique_.py
@@ -0,0 +1,56 @@
+"""Make columns nullable and add unique constraint
+
+Revision ID: 1bba57a81eea
+Revises: 0481276c6c83
+Create Date: 2024-12-17 23:16:51.909305
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '1bba57a81eea'
+down_revision: Union[str, None] = '0481276c6c83'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('ingest_user_project_csv', 'email',
+               existing_type=sa.TEXT(),
+               nullable=True,
+               existing_server_default=sa.text("''::text"))
+    op.alter_column('ingest_user_project_csv', 'buid',
+               existing_type=sa.TEXT(),
+               nullable=True,
+               existing_server_default=sa.text("''::text"))
+    op.alter_column('ingest_user_project_csv', 'github_username',
+               existing_type=sa.TEXT(),
+               nullable=True,
+               existing_server_default=sa.text("''::text"))
+    op.create_index('ix_ingest_user_project_csv_unique', 'ingest_user_project_csv', [sa.text("COALESCE(project_name, '')"), sa.text("COALESCE(project_tag, '')"), sa.text("COALESCE(first_name, '')"), sa.text("COALESCE(last_name, '')"), sa.text("COALESCE(email, '')"), sa.text("COALESCE(buid, '')"), sa.text("COALESCE(github_username, '')")], unique=True)
+    op.create_unique_constraint('uq_ingest_user_project_csv_all_columns', 'ingest_user_project_csv', ['project_name', 'project_tag', 'first_name', 'last_name', 'email', 'buid', 'github_username'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('uq_ingest_user_project_csv_all_columns', 'ingest_user_project_csv', type_='unique')
+    op.drop_index('ix_ingest_user_project_csv_unique', table_name='ingest_user_project_csv')
+    op.alter_column('ingest_user_project_csv', 'github_username',
+               existing_type=sa.TEXT(),
+               nullable=False,
+               existing_server_default=sa.text("''::text"))
+    op.alter_column('ingest_user_project_csv', 'buid',
+               existing_type=sa.TEXT(),
+               nullable=False,
+               existing_server_default=sa.text("''::text"))
+    op.alter_column('ingest_user_project_csv', 'email',
+               existing_type=sa.TEXT(),
+               nullable=False,
+               existing_server_default=sa.text("''::text"))
+    # ### end Alembic commands ###
diff --git a/app/alembic/versions/94fce2a59224_change_null_constraints_on_.py b/app/alembic/versions/94fce2a59224_change_null_constraints_on_.py
new file mode 100644
index 0000000..0164cda
--- /dev/null
+++ b/app/alembic/versions/94fce2a59224_change_null_constraints_on_.py
@@ -0,0 +1,46 @@
+"""Change null constraints on IngestuserProjectCSV
+
+Revision ID: 94fce2a59224
+Revises: b0a85b0a1488
+Create Date: 2024-12-17 21:05:47.505382
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '94fce2a59224'
+down_revision: Union[str, None] = 'b0a85b0a1488'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('ingest_user_project_csv', 'email',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    op.alter_column('ingest_user_project_csv', 'buid',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    op.alter_column('ingest_user_project_csv', 'github_username',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('ingest_user_project_csv', 'github_username',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    op.alter_column('ingest_user_project_csv', 'buid',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    op.alter_column('ingest_user_project_csv', 'email',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    # ### end Alembic commands ###
diff --git a/app/alembic/versions/b0a85b0a1488_add_unique_constraint_to_.py b/app/alembic/versions/b0a85b0a1488_add_unique_constraint_to_.py
new file mode 100644
index 0000000..d327b0e
--- /dev/null
+++ b/app/alembic/versions/b0a85b0a1488_add_unique_constraint_to_.py
@@ -0,0 +1,31 @@
+"""Add unique constraint to IngestUserProjectCSV
+
+Revision ID: b0a85b0a1488
+Revises: 
+Create Date: 2024-12-17 20:53:17.286467
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = 'b0a85b0a1488'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add the unique constraint
+    op.create_unique_constraint(
+        'unique_project_email',
+        'ingest_user_project_csv',
+        ['project_name', 'project_tag', 'first_name', 'last_name', 'email', 'buid', 'github_username']
+    )
+
+def downgrade() -> None:
+    # Remove the unique constraint
+    op.drop_constraint('unique_project_email', 'ingest_user_project_csv', type_='unique')
\ No newline at end of file
diff --git a/app/github.py b/app/github.py
index c73ce78..158199b 100644
--- a/app/github.py
+++ b/app/github.py
@@ -28,8 +28,7 @@ def __init__(self, GITHUB_PAT: str, ORG_NAME: str):
             'X-GitHub-Api-Version': '2022-11-28'
         }
         self.log = log.SparkLogger("GITHUB", output=True, persist=True)
-        
-        self.log.warning(f"Github initialized with PAT: {GITHUB_PAT[:20]}... and ORG: {ORG_NAME}")
+        self.log.info(f"Github initialized with PAT: {GITHUB_PAT[:20]}... and ORG: {ORG_NAME}")
         
     def extract_user_repo_from_ssh_url(self, ssh_url: str) -> tuple[str, str]:
         """
@@ -41,6 +40,7 @@ def extract_user_repo_from_ssh_url(self, ssh_url: str) -> tuple[str, str]:
         """
 
         if not ssh_url.startswith("git@github.com:"):
+            self.log.error(f"Invalid SSH URL format: {ssh_url}")
             raise ValueError("Invalid SSH URL format")
         try:
             ssh_url_parts = ssh_url.split(':')[-1].split('/')
@@ -48,6 +48,7 @@ def extract_user_repo_from_ssh_url(self, ssh_url: str) -> tuple[str, str]:
             repo_name = ssh_url_parts[1].split('.')[0]
             return username, repo_name
         except Exception as e:
+            self.log.error(f"Failed to extract username and repo name from SSH URL: {str(e)}")
             raise ValueError("Invalid SSH URL format")
         
     def check_user_exists(self, user: str) -> bool:
@@ -116,6 +117,60 @@ def add_user_to_repo(self, repo_url: str, user: str, permission: perms) -> tuple
         except Exception as e:
             return 500, str(e)
     
+    def remove_user_from_repo(self, repo_url: str, user: str) -> tuple[int, str]:
+        """
+        Removes a GitHub user from a repository. If the user is a collaborator, it will remove
+        them directly. If they are currently invited (but not yet a collaborator), it will revoke
+        their invitation instead.
+
+        Args: 
+            repo_url (str): The URL of the GitHub repository.
+            user (str): The username of the GitHub user.
+        Returns: Tuple[int, str]: A tuple containing the status code and message.
+        Raises: Exception: If an error occurs during the API request.
+        """
+        
+        ssh_url = repo_url.replace("https://github.com/", "git@github.com:")
+        owner, repo_name = self.extract_user_repo_from_ssh_url(ssh_url)
+        
+        if not self.check_user_exists(user): return 404, f"User {user} does not exist"
+
+        try:
+            response = requests.delete(
+                f'https://api.github.com/repos/{owner}/{repo_name}/collaborators/{user}',
+                headers=self.HEADERS,
+                timeout=2
+            )
+            if response.status_code == 204:
+                return 204, f"Successfully removed {user} from {repo_name} repository as a collaborator."
+
+            invitations_response = requests.get(
+                f'https://api.github.com/repos/{owner}/{repo_name}/invitations',
+                headers=self.HEADERS,
+                timeout=10
+            )
+            if invitations_response.status_code != 200:
+                return response.status_code, response.json()
+
+            invitations = invitations_response.json()
+            invitation = next((inv for inv in invitations if inv['invitee']['login'] == user), None)
+
+            if invitation:
+                revoke_response = requests.delete(
+                    f'https://api.github.com/repos/{owner}/{repo_name}/invitations/{invitation["id"]}',
+                    headers=self.HEADERS,
+                    timeout=2
+                )
+                if revoke_response.status_code == 204:
+                    return 204, f"Successfully revoked invitation for {user}."
+                else:
+                    return revoke_response.status_code, revoke_response.json()
+            else:
+                return response.status_code, response.json()
+
+        except Exception as e:
+            return 500, str(e)
+            
     def get_users_on_repo(self, repo_url: str) -> set[str]:
         """
         Retrieves a set of GitHub usernames who are collaborators on a given repository.
@@ -384,14 +439,10 @@ def create_repo(self, repo_name: str, private: bool = False) -> tuple[int, str]:
                 self.log.error(f"Failed to create repository {repo_name}: {response.json()}")
                 return response.status_code, response.json()
         except Exception as e:
-            self.log.error(f"Failed to create repository {repo_name}: {str(e)}")
+            self.log.error(f"Exception Failed to create repository {repo_name}: {str(e)}")
             return 500, str(e)
     
 if __name__ == "__main__":
     TEST_GITHUB_PAT = os.getenv("TEST_GITHUB_PAT") or ""
     github = Github(TEST_GITHUB_PAT, "auto-spark")
-    github.create_repo("test", private=False)
-    #print(github.change_user_permission_on_repo("https://github.com/spark-tests/initial", "mochiakku", "push"))
-    #print(github.change_all_user_permission_on_repo("https://github.com/spark-tests/initial", "push"))
-    #print(github.get_all_repos())
     print()
\ No newline at end of file
diff --git a/app/models.py b/app/models.py
index 1ba223d..1ea9977 100644
--- a/app/models.py
+++ b/app/models.py
@@ -5,7 +5,9 @@
 from datetime import datetime
 
 from sqlalchemy import (
+    Boolean,
     Column,
+    Index,
     Integer,
     Text,
     DateTime,
@@ -14,7 +16,8 @@
     Enum as Enum_,
     PrimaryKeyConstraint,
     TIMESTAMP,
-    func
+    func,
+    text
 )
 from sqlalchemy.orm import (
     DeclarativeBase,
@@ -35,9 +38,10 @@ class Season(enum.Enum):
 
 class Status(enum.Enum):
     started = 'started'
-    invited = 'invited'
     pull = 'pull'
     push = 'push'
+    removed = 'removed'
+    failed = 'failed'
 
 class Outcome(enum.Enum):
     success = 'success'
@@ -82,7 +86,7 @@ class User(Base):
     last_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     email: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
     buid: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
-    github_username: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    github_username: Mapped[str] = mapped_column(Text, unique=True)
     
     # Bidirectional Relationship to UserProject
     user_projects: Mapped[List["UserProject"]] = \
@@ -106,7 +110,7 @@ class Project(Base):
     project_name: Mapped[str] = mapped_column(Text, nullable=False)
     project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
     semester_id: Mapped[int] = mapped_column(ForeignKey('semester.semester_id', ondelete='RESTRICT'), nullable=False)
-    github_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    github_url: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
     slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
     created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False, server_default=func.now())
 
@@ -118,13 +122,10 @@ class Project(Base):
 
     def __repr__(self):
         return (
-            f"<Project(project_id={self.project_id}, "
-            f"project_name='{self.project_name}', "
-            f"project_tag='{self.project_tag}', "
-            f"semester_id={self.semester_id}, "
-            f"github_url='{self.github_url}', "
-            f"slack_channel='{self.slack_channel}', "
-            f"created_at={self.created_at.isoformat()})>"
+            f"<Project(project_id={self.project_id}, course='{self.course}', "
+            f"project_name='{self.project_name}', project_tag='{self.project_tag}', "
+            f"semester_id={self.semester_id}, github_url='{self.github_url}', "
+            f"slack_channel='{self.slack_channel}', created_at={self.created_at.isoformat()})>"
         )
         
 class UserProject(Base):
@@ -134,8 +135,11 @@ class UserProject(Base):
         mapped_column(ForeignKey('project.project_id'), primary_key=True, nullable=False)
     user_id: Mapped[int] = \
         mapped_column(ForeignKey('user.user_id'), primary_key=True, nullable=False)
-    status: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
-    created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True),nullable=False, server_default=func.now())
+    status_github: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
+    status_slack: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
+    github_result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    slack_result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True),nullable=True, server_default=func.now())
 
     # Relationships to User and Project
     user: Mapped["User"] = relationship("User", back_populates="user_projects")
@@ -143,8 +147,11 @@ class UserProject(Base):
 
     def __repr__(self) -> str:
         return (
-            f"<UserProject(project_id={self.project_id}, user_id={self.user_id}, "
-            f"status={self.status}, created_at={self.created_at.isoformat()})>"
+            f"<UserProject(project_id={self.project_id}, "
+            f"user_id={self.user_id}, "
+            f"status_github={self.status_github}, "
+            f"status_slack={self.status_slack}, "
+            f"created_at={self.created_at.isoformat()})>"
         )
 
 class IngestProjectCSV(Base):
@@ -160,8 +167,8 @@ class IngestProjectCSV(Base):
     semester: Mapped[str] = mapped_column(Text, nullable=False)
     github_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
     slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
-    generate_github: Mapped[Optional[bool]] = mapped_column(Integer, nullable=True, default=False)
-    generate_slack: Mapped[Optional[bool]] = mapped_column(Integer, nullable=True, default=False)
+    generate_github: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True, default=False)
+    generate_slack: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True, default=False)
     outcome: Mapped[Optional[Outcome]] = mapped_column(Enum_(Outcome), nullable=True)
     result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     
@@ -181,16 +188,35 @@ class IngestUserProjectCSV(Base):
         Integer, primary_key=True, nullable=False,
         #server_default=func.nextval('ingest_user_project_csv_id_seq')
     )
-    project_name: Mapped[str] = mapped_column(Text, nullable=False)
-    project_tag: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
+    project_name: Mapped[str] = mapped_column(Text)
+    project_tag: Mapped[str] = mapped_column(Text)
     first_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     last_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
-    email: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
-    buid: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
-    github_username: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    email: Mapped[Optional[str]] = mapped_column(Text, nullable=True, default=None, unique=False)
+    buid: Mapped[Optional[str]] = mapped_column(Text, nullable=True, default=None, unique=False)
+    github_username: Mapped[Optional[str]] = mapped_column(Text, nullable=True, default=None, unique=False)
     outcome: Mapped[Optional[Outcome]] = mapped_column(Enum_(Outcome), nullable=True)
     result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     
+    __table_args__ = (
+        UniqueConstraint(
+            'project_name', 'project_tag', 'first_name', 'last_name', 'email', 'buid', 'github_username',
+            name='uq_ingest_user_project_csv_all_columns'
+        ),
+        Index(
+            'ix_ingest_user_project_csv_unique',
+            # replace NULLs with empty strings to ensure uniqueness on comparison
+            func.coalesce('project_name', ''),
+            func.coalesce('project_tag', ''),
+            func.coalesce('first_name', ''),
+            func.coalesce('last_name', ''),
+            func.coalesce('email', ''),
+            func.coalesce('buid', ''),
+            func.coalesce('github_username', ''),
+            unique=True
+        ),
+    )
+    
     def __repr__(self) -> str:
         return (
             f"<IngestUserProjectCSV(id={self.id}, project_name='{self.project_name}', "
diff --git a/app/schema.py b/app/schema.py
index a27c187..091edf7 100644
--- a/app/schema.py
+++ b/app/schema.py
@@ -1,12 +1,14 @@
+from datetime import datetime
 from typing import Optional
 from pydantic import BaseModel
+from models import Status
 
 class _Project(BaseModel):
     course: Optional[str]
     project_name: str
     project_tag: str
     semester_id: int
-    github_url: Optional[str]
+    github_url: str
     slack_channel: Optional[str]
     
 class _User(BaseModel):
@@ -14,4 +16,13 @@ class _User(BaseModel):
     last_name: Optional[str]
     email: str
     buid: Optional[str]
-    github_username: Optional[str]
\ No newline at end of file
+    github_username: str
+    
+class _UserProject(BaseModel):
+    project_id: int
+    user_id: int
+    status_github: Optional[Status]
+    status_slack: Optional[Status]
+    github_result: Optional[str]
+    slack_result: Optional[str]
+    created_at: Optional[datetime]
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index 8cc30b6..9bb1aae 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -1,7 +1,7 @@
 import os
-from typing import Any, Generator
-from schema import _Project
-from models import User, Project, Base, IngestProjectCSV, Semester, UserProject, Outcome
+from typing import Any, Generator, List, Literal
+from schema import _Project, _User, _UserProject
+from models import User, Project, Base, IngestProjectCSV, IngestUserProjectCSV, Semester, UserProject, Outcome, Status
 from slacker import Slacker
 from sqlalchemy import create_engine
 from sqlalchemy.exc import IntegrityError
@@ -79,8 +79,10 @@ def ingest_csv(self, df: DataFrame, colmap: dict[str, str], table: str):
             try:
                 row_df = pd.DataFrame([row])
                 row_df.to_sql(table, self.engine, if_exists='append', index=False)
-            except Exception as e:
-                self.log.error(f"failure ingesting row into {table}: {e}")
+                self.log.info(f"ingested row into {table}: {row['project_tag']}")
+            except (IntegrityError, Exception) as e:
+                msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
+                self.log.error(f"failure ingesting row into {table}: {msg}")
                 continue
     
     def ingest_project_csv(self, df: DataFrame):
@@ -89,7 +91,7 @@ def ingest_project_csv(self, df: DataFrame):
         colmap: dict[str, str] = {
             "Course": "course",
             "Project Name": "project_name",
-            "Project Identifier": "project_tag",
+            "Project Tag": "project_tag",
             "Semester": "semester",
             "GitHub Repository": "github_url",
             "Slack": "slack_channel",
@@ -103,7 +105,13 @@ def ingest_user_project_csv(self, df: DataFrame):
         """ingest user project csv"""
         
         colmap: dict[str, str] = {
-            
+            "Project Name": "project_name",
+            "Project Tag": "project_tag",
+            "First Name": "first_name",
+            "Last Name": "last_name",
+            "Email": "email",
+            "BUID": "buid",
+            "GitHub Username": "github_username"
         }
 
         self.ingest_csv(df, colmap, "ingest_user_project_csv")
@@ -113,7 +121,10 @@ def ingest_user_project_csv(self, df: DataFrame):
     # ----------------------------------------------------------------------------------------------------------------------
     
     def process_ingest_project_csv(self):
-        """Process project csv"""
+        """
+            Takes the ingest_project_csv and converts the rows to projects. 
+            Creates the necessary slack channels & github repos automatically.
+        """
         
         session = self.s()
             
@@ -125,16 +136,24 @@ def process_ingest_project_csv(self):
                 results = []
 
                 semester = semesters.get(row.semester.lower())
-                if not semester:
+                if not semester or (not row.github_url and not row.generate_github):
+                    row.result = "failure: semester discrepancy." if not semester else \
+                        "failure: missing github url or generate flag."
                     row.outcome = Outcome.failure
-                    row.result = "failure: semester discrepancy."
-                    self.log.error(f"failure processing project csv row {row.project_tag}: semester discrepancy.")
+                    self.log.error(f"failure processing project csv row {row.project_tag}: {row.result}")
+                    session.commit()
                     continue
                 
                 if row.generate_github:
                     if not row.github_url:
-                        self.git.create_repo(row.project_tag)
-                        row.github_url = f"https://github.com/{self.org}/{row.project_tag}"
+                        code, m = self.git.create_repo(row.project_tag)
+                        if code != 201 and "name already exists on this account" not in str(m):
+                            row.outcome = Outcome.failure
+                            results.append(f"failure: {m}")
+                            self.log.error(f"failure creating github repo for {row.project_tag}: {m}")
+                            session.commit()
+                            continue
+                        row.github_url = f"https://github.com/{self.org}/{row.project_tag}.git"
                         self.log.info(f"created github repo {row.github_url} for project {row.project_tag}.")
                     else:
                         row.outcome = Outcome.warning
@@ -150,6 +169,13 @@ def process_ingest_project_csv(self):
                         row.outcome = Outcome.warning
                         results.append("warning: slack channel already exists.")
                         self.log.warning(f"skipping creating slack for {row.project_tag}: channel already exists.")
+                        
+                if not row.github_url:
+                    row.outcome = Outcome.failure
+                    results.append("failure: missing github url.")
+                    self.log.error(f"failure processing project csv row {row.project_tag}: missing github url.")
+                    session.commit()
+                    continue
 
                 if results: 
                     row.result = " <> ".join(results)
@@ -158,7 +184,7 @@ def process_ingest_project_csv(self):
                     row.result = "all systems operational."
                 
                 project_data = _Project(
-                    course=None,
+                    course=row.course,
                     project_name=row.project_name,
                     project_tag=row.project_tag,
                     semester_id=semester.semester_id,
@@ -180,6 +206,219 @@ def process_ingest_project_csv(self):
                 session.commit()
                 self.log.error(f"failure processing project csv row {row.project_tag}: {msg}")
         
+        session.close()
+        
+    def process_ingest_user_project_csv(self):
+        """Takes the ingest_user_project_csv and converts the rows to users and user_projects."""
+        
+        session = self.s()
+        
+        for row in session.query(IngestUserProjectCSV).all():
+            self.log.info(f"processing user {row.email} for project {row.project_tag}...")
+            try:
+                project = session.query(Project).filter(Project.project_tag == row.project_tag).first()
+                
+                if not (project and row.email and row.github_username):
+                    row.outcome = Outcome.failure
+                    row.result = "failure: missing project, email, or github username."
+                    self.log.error(f"failure processing {row.project_tag}: missing project, email, or github.")
+                    session.commit()
+                    continue
+                
+                user = session.query(User).filter(
+                    (User.email == row.email) | (User.github_username == row.github_username)
+                ).first()
+                if not user:
+                    user_data = _User(
+                        first_name=row.first_name,
+                        last_name=row.last_name,
+                        email=row.email,
+                        buid=row.buid,
+                        github_username=row.github_username
+                    )
+                    user = User(**user_data.model_dump())
+                    session.add(user)
+                    session.flush()
+                
+                user_project_data = _UserProject(
+                    project_id=project.project_id,
+                    user_id=user.user_id,
+                    status_github=Status.started,   
+                    status_slack=Status.started,
+                    github_result=None,
+                    slack_result=None,
+                    created_at=None
+                )
+                user_project = UserProject(**user_project_data.model_dump())
+                session.add(user_project)
+                
+                row.outcome = Outcome.success
+                row.result = "all systems operational."
+                
+                session.commit()
+                
+            except (IntegrityError, Exception) as e:
+                session.rollback()
+                msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
+                session.query(IngestUserProjectCSV).filter(IngestUserProjectCSV.id == row.id).update({
+                    IngestUserProjectCSV.outcome: Outcome.failure,
+                    IngestUserProjectCSV.result: f"failure: {msg}"
+                })
+                session.commit()
+                self.log.error(f"failure processing project csv row {row.project_tag}: {msg}")
+                
+        session.close()
+    
+    # ----------------------------------------------------------------------------------------------------------------------
+    # core automation functionality
+    # ----------------------------------------------------------------------------------------------------------------------
+        
+    def automate_github(self, tags: List[str] = [], start_state: Status = Status.started, end_state: Status = Status.push):
+        """
+            Automates the handling of permissioning users for projects.
+        """
+        
+        session = self.s()
+        
+        user_projects = session.query(UserProject).join(Project).filter(
+            Project.project_tag.in_(tags or [up.project.project_tag for up in session.query(UserProject).all()]),
+            UserProject.status_github == start_state
+        ).all()
+        
+        def permission_lookup(status: Status) -> Literal['pull', 'triage', 'push', 'maintain', 'admin']:
+            if status == Status.push:       return 'push'
+            else: return "pull"
+        
+        for user_project in user_projects:
+            try:
+                project = user_project.project
+                user = user_project.user
+                self.log.info(f"automating from {start_state} to {end_state} for {user.email} on {project.project_tag}...")
+                
+                if not self.git.check_user_exists(user.github_username):
+                    user_project.status_github = Status.failed
+                    user_project.github_result = "failure: github user does not exist."
+                    self.log.error(f"failure automating {user.email} on {project.project_tag}: github user does not exist.")
+                    session.commit()
+                    continue
+                else: self.log.info(f"found github user {user.github_username}.")
+                
+                if end_state == Status.removed:
+                    code, m = self.git.remove_user_from_repo(repo_url=project.github_url, user=user.github_username)
+                    if code != 204:
+                        user_project.status_github = Status.failed
+                        user_project.github_result = f"failure: {m}"
+                        self.log.error(f"failure automating {user.email} on {project.project_tag}: {m}")
+                        session.commit()
+                        continue
+                    user_project.status_github = end_state
+                    user_project.github_result = "all systems operational."
+                    self.log.info(f"automated {user.email} on {project.project_tag} to {end_state}.")
+                    session.commit()
+                    continue
+                
+                if self.git.check_user_is_collaborator(repo_url=project.github_url, user=user.github_username) or \
+                    user.github_username in self.git.get_users_invited_on_repo(repo_url=project.github_url):
+                    self.git.change_user_permission_on_repo(
+                        repo_url=project.github_url,
+                        user=user.github_username,
+                        permission=permission_lookup(end_state)
+                    )
+                    user_project.status_github = end_state
+                    user_project.github_result = "all systems operational."
+                    self.log.info(f"automated {user.email} on {project.project_tag} to {end_state}.")
+                    session.commit()
+                    continue
+                
+                if start_state == Status.started:
+                    code, m = self.git.add_user_to_repo(
+                        repo_url=project.github_url,
+                        user=user.github_username,
+                        permission=permission_lookup(end_state)
+                    )
+                    if code != 201:
+                        user_project.status_github = Status.failed
+                        user_project.github_result = f"failure: {m}"
+                        self.log.error(f"failure automating {user.email} on {project.project_tag}: {m}")
+                        session.commit()
+                        continue
+                    user_project.status_github = end_state
+                    user_project.github_result = "all systems operational."
+                    self.log.info(f"automated {user.email} on {project.project_tag} to {end_state}.")
+                    session.commit()
+                    continue
+                
+            except (IntegrityError, Exception) as e:
+                session.rollback()
+                msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
+                user_project.status_github = Status.failed
+                user_project.github_result = f"failure: {msg}"
+                session.commit()
+                self.log.error(f"automation failed {user_project.user.email} on {user_project.project.project_tag}: {msg}")
+                
+        session.close() 
+    
+    def automate_slack(self, tags: List[str] = []):
+        """Automates adding users to slack channels."""
+        
+        session = self.s()
+        
+        user_projects = session.query(UserProject).join(Project).filter(
+            Project.project_tag.in_(tags or [up.project.project_tag for up in session.query(UserProject).all()]),
+            UserProject.status_slack == Status.started
+        ).all()
+        
+        for user_project in user_projects:
+            try:
+                project = user_project.project
+                user = user_project.user
+                self.log.info(f"automating slack for {user.email} on {project.project_tag}...")
+                
+                if not project.slack_channel:
+                    user_project.status_slack = Status.failed
+                    user_project.slack_result = "failure: slack channel does not exist."
+                    self.log.error(f"failed as slack channel does not exist for {project.project_tag}.")
+                    session.commit()
+                    continue
+                    
+                slack_uid = self.slacker.get_user_id(email=user.email)
+                if not slack_uid:
+                    user_project.status_slack = Status.failed
+                    user_project.slack_result = "failure: slack user does not exist."
+                    self.log.error(f"failed as slack user does not exist for {user.email}.")
+                    session.commit()
+                    continue
+                
+                self.slacker.invite_users_to_channel(
+                    channel_id=project.slack_channel,
+                    user_ids=slack_uid,
+                    retries=3
+                )
+                user_project.status_slack = Status.push
+                user_project.slack_result = "all systems operational."
+                self.log.info(f"automated slack for {user.email} on {project.project_tag}.")
+                session.commit()
+                
+            except (IntegrityError, Exception) as e:
+                session.rollback()
+                msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
+                
+                if "already_in_channel" in msg:
+                    user_project.status_slack = Status.push
+                    user_project.slack_result = "user already in channel."
+                    self.log.warning(
+                        f"skipping as user already in channel for \
+                        {user_project.user.email} on {user_project.project.project_tag}."
+                    )
+                    session.commit()
+                else:
+                    user_project.status_slack = Status.failed
+                    user_project.slack_result = f"failure: {msg}"
+                    session.commit()
+                    self.log.error(
+                        f"automation failed {user_project.user.email} on {user_project.project.project_tag}: {msg}"
+                    )
+    
 if __name__ == "__main__":
     TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
     TEST_SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
@@ -189,6 +428,20 @@ def process_ingest_project_csv(self):
     github = Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
     slacker = Slacker(TEST_SLACK_TOKEN)
     spark = Spark(TEST_POSTGRES, TEST_GITHUB_ORG, slacker, github)
-    #df = pd.read_csv("./ingestprojects.csv")
-    #spark.ingest_project_csv(df)
-    spark.process_ingest_project_csv()
\ No newline at end of file
+    
+    # ingestproject = pd.read_csv("./ingestproject.csv")
+    # spark.ingest_project_csv(ingestproject)
+    
+    # ingestuserproject = pd.read_csv("./ingestuserproject.csv")
+    # spark.ingest_user_project_csv(ingestuserproject)
+    
+    # print("---")
+    # print("---")
+    # print("---")
+    
+    # spark.process_ingest_project_csv()
+    # spark.process_ingest_user_project_csv()
+    
+    # spark.automate_github(tags=[], start_state=Status.pull, end_state=Status.push)
+    
+    spark.automate_slack(tags=[])
\ No newline at end of file

From 524db4953646cf0c2f66c321aef9ce1d849ad05e Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 18 Dec 2024 02:47:28 -0500
Subject: [PATCH 11/18] add gdrive

---
 Pipfile       |   2 +
 Pipfile.lock  | 123 +++++++++++++++++++++++++++++++++++++++++++++++++-
 app/google.py |  29 ++++++++++++
 app/spark.py  |   4 +-
 4 files changed, 155 insertions(+), 3 deletions(-)
 create mode 100644 app/google.py

diff --git a/Pipfile b/Pipfile
index 85814c1..2e2ada9 100644
--- a/Pipfile
+++ b/Pipfile
@@ -18,6 +18,8 @@ sqlmodel = "*"
 pyright = "*"
 mypy = "*"
 alembic = "*"
+google-api-python-client = "*"
+install = "*"
 
 [dev-packages]
 
diff --git a/Pipfile.lock b/Pipfile.lock
index c207414..5e1d1a2 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "981a28803eadd385294cc8a31e056fd46c300d240fa84593cedef28e541c18cb"
+            "sha256": "84a7d483641526b6197b6c552d622da0f40a5e5be50febf128f240a1a89136ae"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -67,6 +67,14 @@
             "markers": "python_full_version < '3.11.3'",
             "version": "==5.0.1"
         },
+        "cachetools": {
+            "hashes": [
+                "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292",
+                "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.5.0"
+        },
         "certifi": {
             "hashes": [
                 "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56",
@@ -228,6 +236,46 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.1.43"
         },
+        "google-api-core": {
+            "hashes": [
+                "sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9",
+                "sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.24.0"
+        },
+        "google-api-python-client": {
+            "hashes": [
+                "sha256:25529f89f0d13abcf3c05c089c423fb2858ac16e0b3727543393468d0d7af67c",
+                "sha256:83fe9b5aa4160899079d7c93a37be306546a17e6686e2549bcc9584f1a229747"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==2.155.0"
+        },
+        "google-auth": {
+            "hashes": [
+                "sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00",
+                "sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.37.0"
+        },
+        "google-auth-httplib2": {
+            "hashes": [
+                "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05",
+                "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"
+            ],
+            "version": "==0.2.0"
+        },
+        "googleapis-common-protos": {
+            "hashes": [
+                "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c",
+                "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.66.0"
+        },
         "greenlet": {
             "hashes": [
                 "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e",
@@ -315,6 +363,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.14.0"
         },
+        "httplib2": {
+            "hashes": [
+                "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc",
+                "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.22.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9",
@@ -563,6 +619,31 @@
             "markers": "python_version >= '3.9'",
             "version": "==2.2.3"
         },
+        "proto-plus": {
+            "hashes": [
+                "sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961",
+                "sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.25.0"
+        },
+        "protobuf": {
+            "hashes": [
+                "sha256:012ce28d862ff417fd629285aca5d9772807f15ceb1a0dbd15b88f58c776c98c",
+                "sha256:027fbcc48cea65a6b17028510fdd054147057fa78f4772eb547b9274e5219331",
+                "sha256:1fc55267f086dd4050d18ef839d7bd69300d0d08c2a53ca7df3920cc271a3c34",
+                "sha256:22c1f539024241ee545cbcb00ee160ad1877975690b16656ff87dde107b5f110",
+                "sha256:32600ddb9c2a53dedc25b8581ea0f1fd8ea04956373c0c07577ce58d312522e0",
+                "sha256:50879eb0eb1246e3a5eabbbe566b44b10348939b7cc1b267567e8c3d07213853",
+                "sha256:5a41deccfa5e745cef5c65a560c76ec0ed8e70908a67cc8f4da5fce588b50d57",
+                "sha256:683be02ca21a6ffe80db6dd02c0b5b2892322c59ca57fd6c872d652cb80549cb",
+                "sha256:8ee1461b3af56145aca2800e6a3e2f928108c749ba8feccc6f5dd0062c410c0d",
+                "sha256:b5ba1d0e4c8a40ae0496d0e2ecfdbb82e1776928a205106d14ad6985a09ec155",
+                "sha256:d473655e29c0c4bbf8b69e9a8fb54645bc289dead6d753b952e7aa660254ae18"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==5.29.1"
+        },
         "psycopg2-binary": {
             "hashes": [
                 "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff",
@@ -637,6 +718,22 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.9.10"
         },
+        "pyasn1": {
+            "hashes": [
+                "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629",
+                "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.6.1"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd",
+                "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.4.1"
+        },
         "pydantic": {
             "hashes": [
                 "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d",
@@ -751,6 +848,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.27.1"
         },
+        "pyparsing": {
+            "hashes": [
+                "sha256:93d9577b88da0bbea8cc8334ee8b918ed014968fd2ec383e868fb8afb1ccef84",
+                "sha256:cbf74e27246d595d9a74b186b810f6fbb86726dbf3b9532efb343f6d7294fe9c"
+            ],
+            "markers": "python_version >= '3.1'",
+            "version": "==3.2.0"
+        },
         "pyright": {
             "hashes": [
                 "sha256:aad7f160c49e0fbf8209507a15e17b781f63a86a1facb69ca877c71ef2e9538d",
@@ -809,6 +914,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.32.3"
         },
+        "rsa": {
+            "hashes": [
+                "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7",
+                "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"
+            ],
+            "markers": "python_version >= '3.6' and python_version < '4'",
+            "version": "==4.9"
+        },
         "six": {
             "hashes": [
                 "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
@@ -976,6 +1089,14 @@
             "markers": "python_version >= '2'",
             "version": "==2024.2"
         },
+        "uritemplate": {
+            "hashes": [
+                "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0",
+                "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==4.1.1"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac",
diff --git a/app/google.py b/app/google.py
new file mode 100644
index 0000000..4860388
--- /dev/null
+++ b/app/google.py
@@ -0,0 +1,29 @@
+from googleapiclient.discovery import build
+from google.oauth2 import service_account
+
+SERVICE_ACCOUNT_FILE = 'path/to/your-service-account.json'
+SCOPES = ['https://www.googleapis.com/auth/drive']
+credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
+
+class GoogleDrive:
+    
+    def __init__(self):
+        self.service = build('drive', 'v3', credentials=credentials)
+
+    def share(self, file_id: str, email: str, role: str = 'writer'):
+        new_permission = {
+            'type': 'user',
+            'role': role,
+            'emailAddress': email
+        }
+
+        try:
+            permission = self.service.permissions().create(
+                fileId=file_id,
+                body=new_permission,
+                sendNotificationEmail=True,
+                emailMessage='I have shared a file with you!'
+            ).execute()
+            return permission.get('id')
+        except Exception as e:
+            return str(e)
diff --git a/app/spark.py b/app/spark.py
index 9bb1aae..d92b9f9 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -443,5 +443,5 @@ def automate_slack(self, tags: List[str] = []):
     # spark.process_ingest_user_project_csv()
     
     # spark.automate_github(tags=[], start_state=Status.pull, end_state=Status.push)
-    
-    spark.automate_slack(tags=[])
\ No newline at end of file
+
+    # spark.automate_slack(tags=[])
\ No newline at end of file

From faa12c9c72d72ab22d7d9830a96dd4e6b2e24653 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 18 Dec 2024 16:00:52 -0500
Subject: [PATCH 12/18] add drive functionality

---
 app/models.py  |  4 ++++
 app/schema.py  |  3 +++
 app/slacker.py | 23 +++++++++++++++++++++--
 app/spark.py   | 13 +++++++++++--
 4 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/app/models.py b/app/models.py
index 1ea9977..fe6d02e 100644
--- a/app/models.py
+++ b/app/models.py
@@ -112,6 +112,7 @@ class Project(Base):
     semester_id: Mapped[int] = mapped_column(ForeignKey('semester.semester_id', ondelete='RESTRICT'), nullable=False)
     github_url: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
     slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    drive_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False, server_default=func.now())
 
     # Bidirectional Relationship to Semester
@@ -137,8 +138,10 @@ class UserProject(Base):
         mapped_column(ForeignKey('user.user_id'), primary_key=True, nullable=False)
     status_github: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
     status_slack: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
+    status_drive: Mapped[Optional[Status]] = mapped_column(Enum_(Status), nullable=True)
     github_result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     slack_result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    drive_result: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True),nullable=True, server_default=func.now())
 
     # Relationships to User and Project
@@ -167,6 +170,7 @@ class IngestProjectCSV(Base):
     semester: Mapped[str] = mapped_column(Text, nullable=False)
     github_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
     slack_channel: Mapped[Optional[str]] = mapped_column(Text, nullable=True, unique=True)
+    drive_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     generate_github: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True, default=False)
     generate_slack: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True, default=False)
     outcome: Mapped[Optional[Outcome]] = mapped_column(Enum_(Outcome), nullable=True)
diff --git a/app/schema.py b/app/schema.py
index 091edf7..088a503 100644
--- a/app/schema.py
+++ b/app/schema.py
@@ -10,6 +10,7 @@ class _Project(BaseModel):
     semester_id: int
     github_url: str
     slack_channel: Optional[str]
+    drive_url: Optional[str]
     
 class _User(BaseModel):
     first_name: Optional[str]
@@ -23,6 +24,8 @@ class _UserProject(BaseModel):
     user_id: int
     status_github: Optional[Status]
     status_slack: Optional[Status]
+    status_drive: Optional[Status]
     github_result: Optional[str]
     slack_result: Optional[str]
+    drive_result: Optional[str]
     created_at: Optional[datetime]
\ No newline at end of file
diff --git a/app/slacker.py b/app/slacker.py
index 5aa6cb7..2b04cfc 100644
--- a/app/slacker.py
+++ b/app/slacker.py
@@ -81,6 +81,24 @@ def get_channel_id(self, channel_name: str) -> str:
             self.log.error(f"failed to fetch channel ID for '{channel_name}': {e.response['error']}")
             raise e
 
+    def get_channel_name(self, channel_id: str) -> str:
+        """
+        Fetches the name of an existing channel with the given ID.
+        
+        Args: channel_id (str): The ID of the channel.
+        Returns: str: The name of the channel if found, otherwise an empty string.
+        Raises: SlackApiError: If the API call fails.
+        """
+        
+        try:
+            response = self.client.conversations_info(channel=channel_id)
+            channel_name = response['channel']['name']
+            self.log.info(f"fetched channel name '{channel_name}' for ID: {channel_id}")
+            return channel_name
+        except SlackApiError as e:
+            self.log.error(f"failed to fetch channel name for ID: {channel_id}: {e.response['error']}")
+            raise e
+
     def invite_users_to_channel(self, channel_id: str, user_ids: List[str] | str, retries: int = 0):
         """
         Invites users to a Slack channel.
@@ -147,9 +165,10 @@ def create_channels_and_add_users(self, channels_dict: dict, is_private: bool =
     load_dotenv()
     
     slacker = Slacker(token=os.getenv('SLACK_BOT_TOKEN') or "")
-    channels_dict = {
+    """ channels_dict = {
         'channel1': ["x@bu.edu"],
         'channel2': ["y@bu.edu"],
         'channel3': ["z@bu.edu"]
     }
-    created_channels = slacker.create_channels_and_add_users(channels_dict, is_private=False)
\ No newline at end of file
+    created_channels = slacker.create_channels_and_add_users(channels_dict, is_private=False) """
+    print(slacker.get_channel_name("C085LBA78GJ"))
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index d92b9f9..3afe717 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -95,6 +95,7 @@ def ingest_project_csv(self, df: DataFrame):
             "Semester": "semester",
             "GitHub Repository": "github_url",
             "Slack": "slack_channel",
+            "Drive": "drive_url",
             "Generate GitHub": "generate_github",
             "Generate Slack": "generate_slack"
         }
@@ -162,8 +163,12 @@ def process_ingest_project_csv(self):
                 
                 if row.generate_slack:
                     if not row.slack_channel:
+                        ####################################################################################################
+                        # CUSTOM SLACK CHANNEL NAME LOGIC GOES HERE
+                        ####################################################################################################
                         slack_channel_id = self.slacker.create_channel(row.project_tag)
-                        row.slack_channel = slack_channel_id
+                        slack_channel_name = self.slacker.get_channel_name(slack_channel_id)
+                        row.slack_channel = slack_channel_name
                         self.log.info(f"created slack channel {slack_channel_id} for project {row.project_tag}.")
                     else:
                         row.outcome = Outcome.warning
@@ -189,6 +194,7 @@ def process_ingest_project_csv(self):
                     project_tag=row.project_tag,
                     semester_id=semester.semester_id,
                     github_url=row.github_url,
+                    drive_url=row.drive_url,
                     slack_channel=row.slack_channel
                 )
                 project = Project(**project_data.model_dump())
@@ -245,8 +251,10 @@ def process_ingest_user_project_csv(self):
                     user_id=user.user_id,
                     status_github=Status.started,   
                     status_slack=Status.started,
+                    status_drive=Status.started,
                     github_result=None,
                     slack_result=None,
+                    drive_result=None,
                     created_at=None
                 )
                 user_project = UserProject(**user_project_data.model_dump())
@@ -389,8 +397,9 @@ def automate_slack(self, tags: List[str] = []):
                     session.commit()
                     continue
                 
+                slack_channel_id = self.slacker.get_channel_id(channel_name=project.slack_channel)
                 self.slacker.invite_users_to_channel(
-                    channel_id=project.slack_channel,
+                    channel_id=slack_channel_id,
                     user_ids=slack_uid,
                     retries=3
                 )

From 50d192d371824a89766d69220766246fcf89ffe6 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 18 Dec 2024 16:10:43 -0500
Subject: [PATCH 13/18] add drive

---
 app/{google.py => drive.py} | 19 ++++------
 app/spark.py                | 73 ++++++++++++++++++++++++++++++-------
 2 files changed, 68 insertions(+), 24 deletions(-)
 rename app/{google.py => drive.py} (64%)

diff --git a/app/google.py b/app/drive.py
similarity index 64%
rename from app/google.py
rename to app/drive.py
index 4860388..0898ff2 100644
--- a/app/google.py
+++ b/app/drive.py
@@ -5,24 +5,21 @@
 SCOPES = ['https://www.googleapis.com/auth/drive']
 credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
 
-class GoogleDrive:
+class Drive:
     
     def __init__(self):
         self.service = build('drive', 'v3', credentials=credentials)
 
-    def share(self, file_id: str, email: str, role: str = 'writer'):
-        new_permission = {
-            'type': 'user',
-            'role': role,
-            'emailAddress': email
-        }
-
+    def share(self, file_url: str, email: str, role: str = 'writer'):
         try:
             permission = self.service.permissions().create(
-                fileId=file_id,
-                body=new_permission,
+                fileId=file_url.split('/')[-2],
+                body={
+                    'type': 'user',
+                    'role': role,
+                    'emailAddress': email
+                },
                 sendNotificationEmail=True,
-                emailMessage='I have shared a file with you!'
             ).execute()
             return permission.get('id')
         except Exception as e:
diff --git a/app/spark.py b/app/spark.py
index 3afe717..3a298ae 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -12,6 +12,7 @@
 from pandas import DataFrame
 from github import Github
 import log
+from drive import Drive
 
 class Spark:
     
@@ -19,11 +20,12 @@ class Spark:
     # SQLAlchemy functionality
     # ======================================================================================================================
     
-    def __init__(self, PGURL: str, org: str, slacker: Slacker, git: Github):
+    def __init__(self, PGURL: str, org: str, slacker: Slacker, git: Github, drive: Drive):
         self.PGURL = PGURL
         self.org = org
         self.engine = create_engine(self.PGURL, echo=False)
         self.slacker = slacker
+        self.drive = drive
         self.git = git
         self.log = log.SparkLogger(name="Spark", output=True, persist=True)
         
@@ -428,6 +430,50 @@ def automate_slack(self, tags: List[str] = []):
                         f"automation failed {user_project.user.email} on {user_project.project.project_tag}: {msg}"
                     )
     
+    def automate_drive(self, tags: List[str] = []):
+        """Automates sharing google drive folders."""
+        
+        session = self.s()
+        
+        user_projects = session.query(UserProject).join(Project).filter(
+            Project.project_tag.in_(tags or [up.project.project_tag for up in session.query(UserProject).all()]),
+            UserProject.status_drive == Status.started
+        ).all()
+        
+        for user_project in user_projects:
+            try:
+                project = user_project.project
+                user = user_project.user
+                self.log.info(f"automating drive for {user.email} on {project.project_tag}...")
+                
+                if not project.drive_url:
+                    user_project.status_drive = Status.failed
+                    user_project.drive_result = "failure: drive url does not exist."
+                    self.log.error(f"failed as drive url does not exist for {project.project_tag}.")
+                    session.commit()
+                    continue
+                    
+                ############################################################################################################
+                # CUSTOM GOOGLE DRIVE SHARING LOGIC GOES HERE
+                ############################################################################################################
+                
+                self.drive.share(project.drive_url, user.email)
+                
+                user_project.status_drive = Status.push
+                user_project.drive_result = "all systems operational."
+                self.log.info(f"automated drive for {user.email} on {project.project_tag}.")
+                session.commit()
+                
+            except (IntegrityError, Exception) as e:
+                session.rollback()
+                msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
+                user_project.status_drive = Status.failed
+                user_project.drive_result = f"failure: {msg}"
+                session.commit()
+                self.log.error(
+                    f"automation failed {user_project.user.email} on {user_project.project.project_tag}: {msg}"
+                )
+    
 if __name__ == "__main__":
     TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
     TEST_SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
@@ -436,21 +482,22 @@ def automate_slack(self, tags: List[str] = []):
     
     github = Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
     slacker = Slacker(TEST_SLACK_TOKEN)
-    spark = Spark(TEST_POSTGRES, TEST_GITHUB_ORG, slacker, github)
+    drive = Drive()
+    spark = Spark(TEST_POSTGRES, TEST_GITHUB_ORG, slacker, github, drive)
     
-    # ingestproject = pd.read_csv("./ingestproject.csv")
-    # spark.ingest_project_csv(ingestproject)
+    ingestproject = pd.read_csv("./ingestproject.csv")
+    spark.ingest_project_csv(ingestproject)
     
-    # ingestuserproject = pd.read_csv("./ingestuserproject.csv")
-    # spark.ingest_user_project_csv(ingestuserproject)
+    ingestuserproject = pd.read_csv("./ingestuserproject.csv")
+    spark.ingest_user_project_csv(ingestuserproject)
     
-    # print("---")
-    # print("---")
-    # print("---")
+    print("---")
+    print("---")
+    print("---")
     
-    # spark.process_ingest_project_csv()
-    # spark.process_ingest_user_project_csv()
+    spark.process_ingest_project_csv()
+    spark.process_ingest_user_project_csv()
     
-    # spark.automate_github(tags=[], start_state=Status.pull, end_state=Status.push)
+    spark.automate_github(tags=[])
 
-    # spark.automate_slack(tags=[])
\ No newline at end of file
+    spark.automate_slack(tags=[])
\ No newline at end of file

From cdd3e61de9d3fdf920f33ad11af453a7a8b881cc Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 18 Dec 2024 16:23:44 -0500
Subject: [PATCH 14/18] update delegation

---
 .gitignore   |  1 +
 app/drive.py | 13 +++++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 96df428..b66e726 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+auto-spark-key.json
 *.csv
 !*.example.csv
 *.txt
diff --git a/app/drive.py b/app/drive.py
index 0898ff2..63307be 100644
--- a/app/drive.py
+++ b/app/drive.py
@@ -1,9 +1,12 @@
 from googleapiclient.discovery import build
 from google.oauth2 import service_account
 
-SERVICE_ACCOUNT_FILE = 'path/to/your-service-account.json'
+SERVICE_ACCOUNT_FILE = 'app/auto-spark-key.json'
 SCOPES = ['https://www.googleapis.com/auth/drive']
-credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
+DELEGATED_USER = 'autospark@bu.edu'
+credentials = service_account.Credentials.from_service_account_file(
+    SERVICE_ACCOUNT_FILE, scopes=SCOPES
+).with_subject(DELEGATED_USER)
 
 class Drive:
     
@@ -24,3 +27,9 @@ def share(self, file_url: str, email: str, role: str = 'writer'):
             return permission.get('id')
         except Exception as e:
             return str(e)
+
+if __name__ == '__main__':
+    drive = Drive()
+    file_url = 'https://drive.google.com/drive/u/0/folders/1q2H4VT_tHahAZTK8vsmLwq9hjwbeDTJJ'
+    email = 'raquelgr@bu.edu'
+    print(drive.share(file_url, email))
\ No newline at end of file

From 713e93040bd05a9da00a8badb45570434447259d Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 18 Dec 2024 16:44:02 -0500
Subject: [PATCH 15/18] db crud dicts for frontend

---
 app/db.py         | 82 +++++++++++++++++++++++++++++++++++++++++++++++
 app/middleware.py |  2 +-
 app/schema.py     | 41 ++++++++++++++++++++++--
 3 files changed, 122 insertions(+), 3 deletions(-)
 create mode 100644 app/db.py

diff --git a/app/db.py b/app/db.py
new file mode 100644
index 0000000..12c57ff
--- /dev/null
+++ b/app/db.py
@@ -0,0 +1,82 @@
+import os
+from typing import Any, Generator, List, Literal
+from schema import _Project, _User, _UserProject, _IngestProjectCSV, _IngestUserProjectCSV
+from models import (
+    User, Project, Base, IngestProjectCSV, IngestUserProjectCSV, Semester, UserProject, Outcome, Status
+)
+from slacker import Slacker
+from sqlalchemy import create_engine
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+from sqlalchemy.orm import sessionmaker
+from contextlib import contextmanager
+import pandas as pd
+from pandas import DataFrame
+from github import Github
+import log
+from drive import Drive
+
+class DB:
+    
+    # ======================================================================================================================
+    # CRUD functionality
+    # ======================================================================================================================
+    
+    def __init__(self, PGURL: str):
+        self.PGURL = PGURL
+        self.engine = create_engine(self.PGURL, echo=False)
+        self.log = log.SparkLogger(name="Spark", output=True, persist=True)
+        
+    def s(self) -> Session:
+        return sessionmaker(bind=self.engine)()
+        
+    @contextmanager
+    def scope(self) -> Generator[Session, Any, None]:
+        session = self.s()
+        try:
+            yield session
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            raise e
+        finally:
+            session.close()
+            
+    def run(self, func, *args, **kwargs):
+        session = self.s()
+        try:
+            result = func(session, *args, **kwargs)
+            session.commit()
+            return result
+        except IntegrityError as ie:
+            session.rollback()
+            print(f"Integrity error: {ie.orig}")
+        except Exception as e:
+            session.rollback()
+            print(f"Unexpected error: {e}")
+        finally:
+            session.close()
+            
+    def get_projects(self):
+        with self.scope() as session:
+            return [_Project.model_validate(s).model_dump() for s in session.query(Project).all()]
+    
+    def get_users(self):
+        with self.scope() as session:
+            return [_User.model_validate(s).model_dump() for s in session.query(User).all()]
+    
+    def get_ingest_projects(self):
+        with self.scope() as session:
+            return [_IngestProjectCSV.model_validate(s).model_dump() for s in session.query(IngestProjectCSV).all()]
+    
+    def get_ingest_user_projects(self):
+        with self.scope() as session:
+            return [_IngestUserProjectCSV.model_validate(s).model_dump() for s in session.query(IngestUserProjectCSV).all()]
+    
+if __name__ == "__main__":
+    TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
+    db = DB(TEST_POSTGRES)
+    print(db.get_projects())
+    print(db.get_users())
+    print(db.get_ingest_projects())
+    print(db.get_ingest_user_projects())
\ No newline at end of file
diff --git a/app/middleware.py b/app/middleware.py
index 18299d0..94b89f2 100644
--- a/app/middleware.py
+++ b/app/middleware.py
@@ -22,7 +22,7 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
             if request.method == "OPTIONS" or request.url.path in self.allowed:
                 return await call_next(request)
             
-            authorization: str = request.headers.get("Authorization")
+            authorization: str | None = request.headers.get("Authorization") 
             if not authorization:
                 return JSONResponse({"detail": "Authorization required"}, status_code=401)
 
diff --git a/app/schema.py b/app/schema.py
index 088a503..2e8195a 100644
--- a/app/schema.py
+++ b/app/schema.py
@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional
 from pydantic import BaseModel
-from models import Status
+from models import Outcome, Status
 
 class _Project(BaseModel):
     course: Optional[str]
@@ -12,6 +12,8 @@ class _Project(BaseModel):
     slack_channel: Optional[str]
     drive_url: Optional[str]
     
+    model_config = {'from_attributes': True}
+    
 class _User(BaseModel):
     first_name: Optional[str]
     last_name: Optional[str]
@@ -19,6 +21,8 @@ class _User(BaseModel):
     buid: Optional[str]
     github_username: str
     
+    model_config = {'from_attributes': True}
+    
 class _UserProject(BaseModel):
     project_id: int
     user_id: int
@@ -28,4 +32,37 @@ class _UserProject(BaseModel):
     github_result: Optional[str]
     slack_result: Optional[str]
     drive_result: Optional[str]
-    created_at: Optional[datetime]
\ No newline at end of file
+    created_at: Optional[datetime]
+    
+    model_config = {'from_attributes': True}
+    
+class _IngestProjectCSV(BaseModel):
+    id: int
+    course: Optional[str]
+    project_name: str
+    project_tag: str
+    semester: str
+    github_url: Optional[str]
+    slack_channel: Optional[str]
+    drive_url: Optional[str]
+    generate_github: Optional[bool]
+    generate_slack: Optional[bool]
+    outcome: Optional[Outcome]
+    result: Optional[str]
+
+    model_config = {'from_attributes': True}
+
+
+class _IngestUserProjectCSV(BaseModel):
+    id: int
+    project_name: str
+    project_tag: str
+    first_name: Optional[str]
+    last_name: Optional[str]
+    email: Optional[str]
+    buid: Optional[str]
+    github_username: Optional[str]
+    outcome: Optional[Outcome]
+    result: Optional[str]
+
+    model_config = {'from_attributes': True}
\ No newline at end of file

From 009f06d56ce86c2ed307c02110b8a821498e45b3 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Sat, 25 Jan 2025 14:42:13 -0500
Subject: [PATCH 16/18] add auto tagger

---
 app/github.py       | 11 ++++++++---
 app/spark.py        | 39 +++++++++++++++++++++++----------------
 app/utils/tagger.py | 21 +++++++++++++++++++++
 3 files changed, 52 insertions(+), 19 deletions(-)
 create mode 100644 app/utils/tagger.py

diff --git a/app/github.py b/app/github.py
index 158199b..e8b3cb0 100644
--- a/app/github.py
+++ b/app/github.py
@@ -443,6 +443,11 @@ def create_repo(self, repo_name: str, private: bool = False) -> tuple[int, str]:
             return 500, str(e)
     
 if __name__ == "__main__":
-    TEST_GITHUB_PAT = os.getenv("TEST_GITHUB_PAT") or ""
-    github = Github(TEST_GITHUB_PAT, "auto-spark")
-    print()
\ No newline at end of file
+    #TEST_GITHUB_PAT = os.getenv("TEST_GITHUB_PAT") or ""
+    #test_github = Github(TEST_GITHUB_PAT, "auto-spark")
+    
+    GITHUB_PAT = os.getenv("SPARK_GITHUB_PAT") or ""
+    github = Github(GITHUB_PAT, "BU-Spark")
+    
+    ppl = github.get_users_on_repo("https://github.com/BU-Spark/ds-black-response-shotspotter.git")
+    print(ppl)
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index 3a298ae..157e8cc 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -290,6 +290,7 @@ def automate_github(self, tags: List[str] = [], start_state: Status = Status.sta
         
         session = self.s()
         
+        # get all user projects from the tags if the tags are provided otherwise get all user projects
         user_projects = session.query(UserProject).join(Project).filter(
             Project.project_tag.in_(tags or [up.project.project_tag for up in session.query(UserProject).all()]),
             UserProject.status_github == start_state
@@ -475,29 +476,35 @@ def automate_drive(self, tags: List[str] = []):
                 )
     
 if __name__ == "__main__":
-    TEST_POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
-    TEST_SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
-    TEST_GITHUB_ORG = "auto-spark"
-    TEST_GITHUB_TOKEN = os.getenv("TEST_GITHUB_PAT") or ""
+    # POSTGRES = os.getenv("POSTGRES_URL") or ""
+    # SLACK_TOKEN = os.getenv("SLACK_BOT_TOKEN") or ""
+    GITHUB_ORG = "BU-Spark"
+    GITHUB_TOKEN = os.getenv("SPARK_GITHUB_PAT") or ""
     
-    github = Github(TEST_GITHUB_TOKEN, TEST_GITHUB_ORG)
-    slacker = Slacker(TEST_SLACK_TOKEN)
+    POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
+    SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
+    #GITHUB_ORG = "auto-spark"
+    #GITHUB_TOKEN = os.getenv("TEST_GITHUB_PAT") or ""
+    
+    github = Github(GITHUB_TOKEN, GITHUB_ORG)
+    slacker = Slacker(SLACK_TOKEN)
     drive = Drive()
-    spark = Spark(TEST_POSTGRES, TEST_GITHUB_ORG, slacker, github, drive)
+    spark = Spark(POSTGRES, GITHUB_ORG, slacker, github, drive)
     
-    ingestproject = pd.read_csv("./ingestproject.csv")
-    spark.ingest_project_csv(ingestproject)
+    # ingestproject = pd.read_csv("./ingestproject.csv")
+    # spark.ingest_project_csv(ingestproject)
     
-    ingestuserproject = pd.read_csv("./ingestuserproject.csv")
-    spark.ingest_user_project_csv(ingestuserproject)
+    # ingestuserproject = pd.read_csv("./ingestuserproject.csv")
+    # spark.ingest_user_project_csv(ingestuserproject)
     
     print("---")
     print("---")
     print("---")
     
-    spark.process_ingest_project_csv()
-    spark.process_ingest_user_project_csv()
+    # spark.process_ingest_project_csv()
+    # spark.process_ingest_user_project_csv()
     
-    spark.automate_github(tags=[])
-
-    spark.automate_slack(tags=[])
\ No newline at end of file
+    # spark.automate_github(tags=[], start_state=Status.started, end_state=Status.push)
+    spark.automate_github(tags=[], start_state=Status.push, end_state=Status.removed)
+    
+    # spark.automate_slack(tags=[])
\ No newline at end of file
diff --git a/app/utils/tagger.py b/app/utils/tagger.py
new file mode 100644
index 0000000..4f6b61b
--- /dev/null
+++ b/app/utils/tagger.py
@@ -0,0 +1,21 @@
+import pandas as pd
+from slugify import slugify
+
+#i = 'user_projects_untagged.csv'
+#o = 'user_projects_tagged.csv'
+
+i = 'projects_untagged.csv'
+o = 'projects_tagged.csv'
+
+
+# Read the CSV file into a DataFrame
+df = pd.read_csv(i)
+
+def generate_slug(name): return slugify(name)
+
+df['project_tag'] = df.apply(
+    lambda row: row['project_tag'] if pd.notnull(row['project_tag']) and str(row['project_tag']).strip() else generate_slug(row['project_name']),
+    axis=1
+)
+
+df.to_csv(o, index=False)

From 46eadb0cdfab7b581a44cc73851f468c76f15656 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Thu, 6 Feb 2025 14:52:17 -0500
Subject: [PATCH 17/18] various fixes and tweaks

---
 app/slacker.py      | 144 +++++++++++++++++++++++++++++++++++++-------
 app/spark.py        | 102 +++++++++++++++++++++++++------
 app/utils/tagger.py |  12 ++--
 3 files changed, 214 insertions(+), 44 deletions(-)

diff --git a/app/slacker.py b/app/slacker.py
index 2b04cfc..155be8c 100644
--- a/app/slacker.py
+++ b/app/slacker.py
@@ -12,6 +12,7 @@ def __init__(self, token: str):
         self.token = token
         self.client = WebClient(token=self.token)
         self.log = log.SparkLogger(name="Slacker", output=True, persist=True)
+        self.channel_cache = {}
 
     def get_user_id(self, email: str) -> str:
         """
@@ -49,8 +50,10 @@ def create_channel(self, channel_name: str, is_private: bool = False) -> str:
             )
             channel_id = response['channel']['id']
             self.log.info(f"created channel '{channel_name}' with ID: {channel_id}")
+            self.channel_cache[channel_name] = channel_id
             return channel_id
         except SlackApiError as e:
+            print(e)
             if e.response['error'] == 'name_taken':
                 self.log.warning(f"channel '{channel_name}' already exists.")
                 existing_channel = self.get_channel_id(channel_name)
@@ -61,24 +64,55 @@ def create_channel(self, channel_name: str, is_private: bool = False) -> str:
 
     def get_channel_id(self, channel_name: str) -> str:
         """
-        Fetches the ID of an existing channel with the given name.
+        Fetches the ID of an existing channel with the given name using pagination.
         
-        Args: channel_name (str): The name of the channel.
-        Returns: str: The ID of the channel if found, otherwise an empty string.
-        Raises: SlackApiError: If the API call fails.
+        Args:
+            channel_name (str): The name of the channel.
+        
+        Returns:
+            str: The ID of the channel if found.
+        
+        Raises:
+            SlackApiError: If the API call fails or the channel is not found.
         """
         
+        print(f"Fetching channel ID for '{channel_name}'...")
+        
+        if channel_name in self.channel_cache:
+            self.log.info(f"Found channel ID {self.channel_cache[channel_name]} in cache for '{channel_name}'.")
+            return self.channel_cache[channel_name]
+        else:
+            self.log.info(f"Channel ID for '{channel_name}' not found in cache. Fetching from API...")
+        
         try:
-            response = self.client.conversations_list(types="public_channel,private_channel", limit=1000)
-            channel_id = next((c['id'] for c in response['channels'] if c['name'] == channel_name), "")
-            if channel_id:
-                self.log.info(f"fetched channel ID {channel_id} for '{channel_name}'.")
-                return channel_id
-            else:
-                self.log.warning(f"channel '{channel_name}' not found.")
-                raise SlackApiError(f"Channel '{channel_name}' not found.", response)
+            cursor = None
+            total = 0
+            while True:
+                time.sleep(1)
+                response = self.client.conversations_list(
+                    types="public_channel,private_channel,mpim,im",
+                    limit=1000, 
+                    cursor=cursor,
+                    exclude_archived=True
+                )
+                total += len(response.get("channels", []))
+                for channel in response.get("channels", []):
+                    if channel.get("name") == channel_name:
+                        self.log.info(f"Fetched channel ID {channel['id']} for '{channel_name}'.")
+                        return channel["id"]
+                    else:
+                        potential_cache_name = channel.get("name")
+                        if potential_cache_name not in self.channel_cache:
+                            self.channel_cache[potential_cache_name] = channel["id"]
+
+                cursor = response.get("response_metadata", {}).get("next_cursor")
+                if not cursor: break
+            
+            print(f"Total channels fetched: {total}")
+            self.log.warning(f"Channel '{channel_name}' not found.")
+            raise SlackApiError(f"Channel '{channel_name}' not found.", response={})
         except SlackApiError as e:
-            self.log.error(f"failed to fetch channel ID for '{channel_name}': {e.response['error']}")
+            self.log.error(f"Failed to fetch channel ID for '{channel_name}': {e}")
             raise e
 
     def get_channel_name(self, channel_id: str) -> str:
@@ -161,14 +195,82 @@ def create_channels_and_add_users(self, channels_dict: dict, is_private: bool =
             self.log.error(f"failed to create channels and add users: {e}")
             raise e
 
+    def change_channel_name(self, channel_id: str, new_name: str):
+        """
+        Changes the name of a Slack channel.
+        
+        Args:
+            channel_id (str): The ID of the channel.
+            new_name (str): The new name for the channel.
+        Raises: SlackApiError: If the API call fails.
+        """
+        
+        try:
+            self.client.conversations_rename(
+                channel=channel_id,
+                name=new_name
+            )
+            self.log.info(f"changed channel name for ID {channel_id} to '{new_name}'.")
+        except SlackApiError as e:
+            self.log.error(f"failed to change channel name for ID {channel_id}: {e.response['error']}")
+            raise e
+
+    def convert_channel_to_private(self, channel_id: str):
+        """
+        Converts a Slack channel to a private channel.
+        
+        Args: channel_id (str): The ID of the channel.
+        Raises: SlackApiError: If the API call fails.
+        """
+        
+        try:
+            self.client.admin_conversations_convertToPrivate(
+                channel_id=channel_id,
+            )
+        except SlackApiError as e:
+            self.log.error(f"failed to convert channel ID {channel_id} to private: {e.response['error']}")
+            raise e
+
 if __name__ == "__main__":
     load_dotenv()
-    
     slacker = Slacker(token=os.getenv('SLACK_BOT_TOKEN') or "")
-    """ channels_dict = {
-        'channel1': ["x@bu.edu"],
-        'channel2': ["y@bu.edu"],
-        'channel3': ["z@bu.edu"]
-    }
-    created_channels = slacker.create_channels_and_add_users(channels_dict, is_private=False) """
-    print(slacker.get_channel_name("C085LBA78GJ"))
\ No newline at end of file
+    
+    chann_tags = [
+        "i-sp25-ds519-488-d4-constituent-app",
+        #"i-sp25-ds519-488-social-justice-app",
+        "i-sp25-ds519-488-bva",
+        "i-sp25-ds519-488-community-service-hours",
+        "i-sp25-ds519-488-auto-mech-challenge",
+        "i-sp25-ds519-488-mass-courts-v3",
+        #"i-sp25-ds519-488-cmovf",
+        #"i-sp25-ds519-488-academico-ai",
+        #"i-sp25-ds519-488-mola"
+    ]
+    
+    underscore_chan_tags = [
+        #"i-sp25-ds519_488-d4-constituent-app",
+        "i-sp25-ds519_488-social-justice-app",
+        #"i-sp25-ds519_488-bva",
+        #"i-sp25-ds519_488-community-service-hours",
+        #"i-sp25-ds519_488-auto-mech-challenge",
+        #"i-sp25-ds519_488-mass-courts-v3",
+        "i-sp25-ds519_488-cmovf",
+        "i-sp25-ds519_488-academico-ai",
+        "i-sp25-ds519_488-mola"
+    ]
+    
+    #created slack channel C08C1NZK076 for project social-justice-app.
+    #created slack channel C08BZD34X2P for project social-justice-app.
+    
+    #for tag in underscore_chan_tags:  print(slacker.get_channel_id(tag))
+    
+    #print(slacker.create_channel("i-sp25-ds519_488-social-justice-app", is_private=True))
+    
+    #print(slacker.get_channel_name("C08BZD34X2P"))
+    #print(slacker.get_channel_name("C08C1NZK076"))
+    #print(slacker.get_channel_id("i-sp25-ds519-488-social-justice-app"))
+    #print(slacker.get_channel_id("i-sp25-ds519_488-social-justice-app"))
+    
+    #print(slacker.change_channel_name("C08BZD34X2P", "renamed-social-justice-app"))
+    
+    slacker.convert_channel_to_private(slacker.get_channel_id("i-sp25-ds549-wlfc-archive"))
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index 157e8cc..eee9987 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -1,4 +1,5 @@
 import os
+import time
 from typing import Any, Generator, List, Literal
 from schema import _Project, _User, _UserProject
 from models import User, Project, Base, IngestProjectCSV, IngestUserProjectCSV, Semester, UserProject, Outcome, Status
@@ -27,7 +28,7 @@ def __init__(self, PGURL: str, org: str, slacker: Slacker, git: Github, drive: D
         self.slacker = slacker
         self.drive = drive
         self.git = git
-        self.log = log.SparkLogger(name="Spark", output=True, persist=True)
+        self.log = log.SparkLogger(name="Spark-DS594", output=True, persist=True)
         
     def s(self) -> Session:
         return sessionmaker(bind=self.engine)()
@@ -168,10 +169,28 @@ def process_ingest_project_csv(self):
                         ####################################################################################################
                         # CUSTOM SLACK CHANNEL NAME LOGIC GOES HERE
                         ####################################################################################################
-                        slack_channel_id = self.slacker.create_channel(row.project_tag)
-                        slack_channel_name = self.slacker.get_channel_name(slack_channel_id)
-                        row.slack_channel = slack_channel_name
-                        self.log.info(f"created slack channel {slack_channel_id} for project {row.project_tag}.")
+                        try:
+                            slack_prefix = "i-sp25-"
+                            slack_course = (row.course or "").replace("/", "-").replace(" ", "-")
+                            slack_tag = row.project_tag
+
+                            slack_f_name = slack_prefix + (slack_course + "-" if slack_course else "") + slack_tag
+                            slack_lower_name = slack_f_name.lower()
+                            
+                            slack_channel_id = self.slacker.create_channel(
+                                slack_lower_name,
+                                True
+                            )
+                            slack_channel_name = self.slacker.get_channel_name(slack_channel_id)
+                            if slack_channel_name != "undefined-slack-channel":
+                                row.slack_channel = slack_channel_name
+                                self.log.info(f"created or fetched slack channel {slack_channel_id} for project {row.project_tag}.")
+                            else:
+                                self.log.error(f"slack channel creation error for {row.project_tag}: {slack_channel_name}")
+                        except Exception as e:
+                            row.outcome = Outcome.warning
+                            results.append(f"warning: {e}")
+                            self.log.warning(f"slack channel creation error for {row.project_tag}: {e}")
                     else:
                         row.outcome = Outcome.warning
                         results.append("warning: slack channel already exists.")
@@ -205,6 +224,7 @@ def process_ingest_project_csv(self):
                 session.commit()
             
             except (IntegrityError, Exception) as e:
+                print(e)
                 session.rollback()
                 msg = str(e.orig if isinstance(e, IntegrityError) else e).strip().replace("\n", "")
                 session.query(IngestProjectCSV).filter(IngestProjectCSV.id == row.id).update({
@@ -212,7 +232,7 @@ def process_ingest_project_csv(self):
                     IngestProjectCSV.result: f"failure: {msg}"
                 })
                 session.commit()
-                self.log.error(f"failure processing project csv row {row.project_tag}: {msg}")
+                self.log.error(f"integrity failure processing project csv row {row.project_tag}: {msg}")
         
         session.close()
         
@@ -376,10 +396,11 @@ def automate_slack(self, tags: List[str] = []):
         
         user_projects = session.query(UserProject).join(Project).filter(
             Project.project_tag.in_(tags or [up.project.project_tag for up in session.query(UserProject).all()]),
-            UserProject.status_slack == Status.started
+            UserProject.status_slack.in_([Status.started, Status.failed])
         ).all()
         
         for user_project in user_projects:
+            time.sleep(1)
             try:
                 project = user_project.project
                 user = user_project.user
@@ -477,12 +498,12 @@ def automate_drive(self, tags: List[str] = []):
     
 if __name__ == "__main__":
     # POSTGRES = os.getenv("POSTGRES_URL") or ""
-    # SLACK_TOKEN = os.getenv("SLACK_BOT_TOKEN") or ""
+    SLACK_TOKEN = os.getenv("SLACK_BOT_TOKEN") or ""
     GITHUB_ORG = "BU-Spark"
     GITHUB_TOKEN = os.getenv("SPARK_GITHUB_PAT") or ""
     
     POSTGRES = os.getenv("TEST_POSTGRES_URL") or ""
-    SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
+    #SLACK_TOKEN = os.getenv("TEST_SLACK_BOT_TOKEN") or ""
     #GITHUB_ORG = "auto-spark"
     #GITHUB_TOKEN = os.getenv("TEST_GITHUB_PAT") or ""
     
@@ -491,20 +512,65 @@ def automate_drive(self, tags: List[str] = []):
     drive = Drive()
     spark = Spark(POSTGRES, GITHUB_ORG, slacker, github, drive)
     
-    # ingestproject = pd.read_csv("./ingestproject.csv")
-    # spark.ingest_project_csv(ingestproject)
+    #ingestproject = pd.read_csv("./ingestproject.csv")
+    #spark.ingest_project_csv(ingestproject)
     
-    # ingestuserproject = pd.read_csv("./ingestuserproject.csv")
-    # spark.ingest_user_project_csv(ingestuserproject)
+    #ingestuserproject = pd.read_csv("./ingestuserproject.csv")
+    #spark.ingest_user_project_csv(ingestuserproject)
     
     print("---")
     print("---")
     print("---")
     
-    # spark.process_ingest_project_csv()
-    # spark.process_ingest_user_project_csv()
+    #spark.process_ingest_project_csv()
+    #spark.process_ingest_user_project_csv()
+    
+    #spark.automate_github(tags=[], start_state=Status.started, end_state=Status.push)
+    #spark.automate_github(tags=[], start_state=Status.push, end_state=Status.removed)
+    
+    ds519channels = [
+        #"i-sp25-ds519-488-d4-constituent-app",
+        "i-sp25-ds519_488-social-justice-app",
+        #"i-sp25-ds519-488-bva",
+        #"i-sp25-ds519-488-community-service-hours",
+        #"i-sp25-ds519-488-auto-mech-challenge",
+        #"i-sp25-ds519-488-mass-courts-v3",
+        "i-sp25-ds519_488-cmovf",
+        "i-sp25-ds519_488-academico-ai",
+        "i-sp25-ds519_488-mola"
+    ]
+    ds519tags = [
+        #"constituent-app",
+        "social-justice-app",
+        #"bva",
+        #"community-service-hours",
+        #"auto-mech-challenge",
+        #"mass-courts-v3",
+        "cmovf",
+        "academico-ai",
+        "mola"
+    ]
+    #spark.automate_slack(tags=ds519tags)
+    
+    ds549channels = [
+        "i-sp25-ds549-coffeechat-matchmaker",
+        "i-sp25-ds549-wlfc-archive"
+    ]
+    ds549tags = [
+        "wlfc-archive",
+        "coffeechat-matchmaker"
+    ]
+    
+    #spark.automate_slack(tags=ds549tags)
     
-    # spark.automate_github(tags=[], start_state=Status.started, end_state=Status.push)
-    spark.automate_github(tags=[], start_state=Status.push, end_state=Status.removed)
+    saadid = slacker.get_user_id("saad7@bu.edu")
+    langid = slacker.get_user_id("langd0n@bu.edu")
+    omarid = slacker.get_user_id("oea@bu.edu")
     
-    # spark.automate_slack(tags=[])
\ No newline at end of file
+    for c in ds519channels:
+        print(c)
+        slacker.invite_users_to_channel(
+            slacker.get_channel_id(c),
+            [saadid, langid, omarid]
+        )
+        
\ No newline at end of file
diff --git a/app/utils/tagger.py b/app/utils/tagger.py
index 4f6b61b..6696bc3 100644
--- a/app/utils/tagger.py
+++ b/app/utils/tagger.py
@@ -1,17 +1,19 @@
 import pandas as pd
 from slugify import slugify
 
-#i = 'user_projects_untagged.csv'
-#o = 'user_projects_tagged.csv'
+i = 'user_projects_untagged.csv'
+o = 'user_projects_tagged.csv'
 
-i = 'projects_untagged.csv'
-o = 'projects_tagged.csv'
+#i = 'projects_untagged.csv'
+#o = 'projects_tagged.csv'
+
+s = 'sp25-'
 
 
 # Read the CSV file into a DataFrame
 df = pd.read_csv(i)
 
-def generate_slug(name): return slugify(name)
+def generate_slug(name): return s+slugify(name)
 
 df['project_tag'] = df.apply(
     lambda row: row['project_tag'] if pd.notnull(row['project_tag']) and str(row['project_tag']).strip() else generate_slug(row['project_name']),

From c9d5a2680c1511ef68913eb2012fedb534739e51 Mon Sep 17 00:00:00 2001
From: s-alad <n7.saad@gmail.com>
Date: Wed, 12 Feb 2025 17:17:08 -0500
Subject: [PATCH 18/18] tweaks

---
 .env.example | 12 ++++++++----
 app/spark.py | 41 ++++++++++++++++++++++++++---------------
 2 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/.env.example b/.env.example
index 1f83e36..512a4b7 100644
--- a/.env.example
+++ b/.env.example
@@ -1,4 +1,8 @@
-GITHUB_PAT=your-org-github-pat
-POSTGRES_URL=the-retool-postgres-url
-USERNAME=your-username-for-frontend-security
-PASSWORD=your-password-for-frontend-security
\ No newline at end of file
+POSTGRES_URL=
+TEST_POSTGRES_URL=
+SPARK_GITHUB_PAT=
+TEST_GITHUB_PAT=
+SLACK_BOT_TOKEN=
+TEST_SLACK_BOT_TOKEN=
+USERNAME="spark"
+PASSWORD="automations"
\ No newline at end of file
diff --git a/app/spark.py b/app/spark.py
index eee9987..11b2ac8 100644
--- a/app/spark.py
+++ b/app/spark.py
@@ -28,7 +28,7 @@ def __init__(self, PGURL: str, org: str, slacker: Slacker, git: Github, drive: D
         self.slacker = slacker
         self.drive = drive
         self.git = git
-        self.log = log.SparkLogger(name="Spark-DS594", output=True, persist=True)
+        self.log = log.SparkLogger(name="Spark-s", output=True, persist=True)
         
     def s(self) -> Session:
         return sessionmaker(bind=self.engine)()
@@ -68,6 +68,15 @@ def run(self, func, *args, **kwargs):
     # ingestion of csv files into holding tables
     # ----------------------------------------------------------------------------------------------------------------------
     
+    def clear_ingestion_tables(self):
+        """clear ingestion tables"""
+        
+        session = self.s()
+        session.query(IngestProjectCSV).delete()
+        session.query(IngestUserProjectCSV).delete()
+        session.commit()
+        session.close()
+    
     def ingest_csv(self, df: DataFrame, colmap: dict[str, str], table: str):
         """ingest csv"""
         
@@ -356,12 +365,14 @@ def permission_lookup(status: Status) -> Literal['pull', 'triage', 'push', 'main
                         permission=permission_lookup(end_state)
                     )
                     user_project.status_github = end_state
-                    user_project.github_result = "all systems operational."
-                    self.log.info(f"automated {user.email} on {project.project_tag} to {end_state}.")
+                    user_project.github_result = "already a collaborator - all systems operational."
+                    self.log.info(f"user {user.email} already a collaborator on {project.project_tag} with {end_state}.")
                     session.commit()
                     continue
                 
-                if start_state == Status.started:
+                if start_state == Status.started or \
+                    (start_state == Status.removed and end_state == Status.push) or \
+                    (start_state == Status.failed and end_state == Status.push):
                     code, m = self.git.add_user_to_repo(
                         repo_url=project.github_url,
                         user=user.github_username,
@@ -373,6 +384,7 @@ def permission_lookup(status: Status) -> Literal['pull', 'triage', 'push', 'main
                         self.log.error(f"failure automating {user.email} on {project.project_tag}: {m}")
                         session.commit()
                         continue
+                    else: self.log.info(f"added {user.email} to {project.project_tag}.")
                     user_project.status_github = end_state
                     user_project.github_result = "all systems operational."
                     self.log.info(f"automated {user.email} on {project.project_tag} to {end_state}.")
@@ -525,8 +537,7 @@ def automate_drive(self, tags: List[str] = []):
     #spark.process_ingest_project_csv()
     #spark.process_ingest_user_project_csv()
     
-    #spark.automate_github(tags=[], start_state=Status.started, end_state=Status.push)
-    #spark.automate_github(tags=[], start_state=Status.push, end_state=Status.removed)
+    spark.automate_github(tags=["commonwealth-climate"], start_state=Status.failed, end_state=Status.push)
     
     ds519channels = [
         #"i-sp25-ds519-488-d4-constituent-app",
@@ -563,14 +574,14 @@ def automate_drive(self, tags: List[str] = []):
     
     #spark.automate_slack(tags=ds549tags)
     
-    saadid = slacker.get_user_id("saad7@bu.edu")
-    langid = slacker.get_user_id("langd0n@bu.edu")
-    omarid = slacker.get_user_id("oea@bu.edu")
+    #saadid = slacker.get_user_id("saad7@bu.edu")
+    #langid = slacker.get_user_id("langd0n@bu.edu")
+    #omarid = slacker.get_user_id("oea@bu.edu")
     
-    for c in ds519channels:
-        print(c)
-        slacker.invite_users_to_channel(
-            slacker.get_channel_id(c),
-            [saadid, langid, omarid]
-        )
+    #for c in ds519channels:
+    #    print(c)
+    #    slacker.invite_users_to_channel(
+    #        slacker.get_channel_id(c),
+    #        [saadid, langid, omarid]
+    #    )
         
\ No newline at end of file