Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Fix race conditions and GH API endpoints #100

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix race conditions and GH API endpoints #100

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 52 additions & 18 deletions github_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ async def query(self, generated_query: str) -> Dict:
:param generated_query: string query to be sent to the API
:return: decoded GraphQL JSON output
"""

headers = {
"Authorization": f"Bearer {self.access_token}",
}
Expand All @@ -63,6 +64,7 @@ async def query(self, generated_query: str) -> Dict:
result = r_requests.json()
if result is not None:
return result

return dict()

async def query_rest(self, path: str, params: Optional[Dict] = None) -> Dict:
Expand All @@ -76,6 +78,8 @@ async def query_rest(self, path: str, params: Optional[Dict] = None) -> Dict:
for _ in range(60):
headers = {
"Authorization": f"token {self.access_token}",
"X-GitHub-Api-Version": "2022-11-28",
"Accept": "application/vnd.github+json",
}
if params is None:
params = dict()
Expand Down Expand Up @@ -136,6 +140,7 @@ def repos_overview(
isFork: false,
after: {"null" if owned_cursor is None else '"'+ owned_cursor +'"'}
) {{
totalCount
pageInfo {{
hasNextPage
endCursor
Expand Down Expand Up @@ -172,6 +177,7 @@ def repos_overview(
]
after: {"null" if contrib_cursor is None else '"'+ contrib_cursor +'"'}
) {{
totalCount
pageInfo {{
hasNextPage
endCursor
Expand Down Expand Up @@ -273,6 +279,7 @@ def __init__(
self._repos: Optional[Set[str]] = None
self._lines_changed: Optional[Tuple[int, int]] = None
self._views: Optional[int] = None
self._get_stats_completed: Optional[asyncio.Event] = None

async def to_str(self) -> str:
"""
Expand All @@ -296,13 +303,24 @@ async def to_str(self) -> str:
- {formatted_languages}"""

async def get_stats(self) -> None:
if self._get_stats_completed is not None:
await self._get_stats_completed.wait()
return

self._get_stats_completed = asyncio.Event()
await self._get_stats()
self._get_stats_completed.set()

async def _get_stats(self) -> None:
"""
Get lots of summary statistics using one big query. Sets many attributes
"""
self._stargazers = 0
self._forks = 0
self._languages = dict()
self._repos = set()

stargazers = 0
forks = 0
languages = dict()
repos = set()
name = None

exclude_langs_lower = {x.lower() for x in self._exclude_langs}

Expand All @@ -316,9 +334,12 @@ async def get_stats(self) -> None:
)
raw_results = raw_results if raw_results is not None else {}

self._name = raw_results.get("data", {}).get("viewer", {}).get("name", None)
if self._name is None:
self._name = (
repo_count = raw_results.get("data", {}).get("viewer", {}).get("repositories", {}).get("totalCount", 0) \
+ raw_results.get("data", {}).get("viewer", {}).get("repositoriesContributedTo", {}).get("totalCount", 0)

name = raw_results.get("data", {}).get("viewer", {}).get("name", None)
if name is None:
name = (
raw_results.get("data", {})
.get("viewer", {})
.get("login", "No Name")
Expand All @@ -333,23 +354,24 @@ async def get_stats(self) -> None:
raw_results.get("data", {}).get("viewer", {}).get("repositories", {})
)

repos = owned_repos.get("nodes", [])
fetched_repos = owned_repos.get("nodes", [])
if not self._ignore_forked_repos:
repos += contrib_repos.get("nodes", [])
fetched_repos += contrib_repos.get("nodes", [])

for i, repo in enumerate(fetched_repos):
print(f"Fetching stats for repo ({i+1}/{repo_count})")

for repo in repos:
if repo is None:
continue
name = repo.get("nameWithOwner")
if name in self._repos or name in self._exclude_repos:
if name in repos or name in self._exclude_repos:
continue
self._repos.add(name)
self._stargazers += repo.get("stargazers").get("totalCount", 0)
self._forks += repo.get("forkCount", 0)
repos.add(name)
stargazers += repo.get("stargazers").get("totalCount", 0)
forks += repo.get("forkCount", 0)

for lang in repo.get("languages", {}).get("edges", []):
name = lang.get("node", {}).get("name", "Other")
languages = await self.languages
if name.lower() in exclude_langs_lower:
continue
if name in languages:
Expand All @@ -376,10 +398,16 @@ async def get_stats(self) -> None:

# TODO: Improve languages to scale by number of contributions to
# specific filetypes
langs_total = sum([v.get("size", 0) for v in self._languages.values()])
for k, v in self._languages.items():
langs_total = sum([v.get("size", 0) for v in languages.values()])
for k, v in languages.items():
v["prop"] = 100 * (v.get("size", 0) / langs_total)

self._stargazers = stargazers
self._forks = forks
self._languages = languages
self._repos = repos
self._name = name

@property
async def name(self) -> str:
"""
Expand Down Expand Up @@ -532,12 +560,18 @@ async def main() -> None:
"""
access_token = os.getenv("ACCESS_TOKEN")
user = os.getenv("GITHUB_ACTOR")
excluded = os.getenv("EXCLUDED")
ignore_forked_repos = os.getenv("EXCLUDE_FORKED_REPOS") == "true"
if excluded is not None:
exclude_repos = set(excluded.split(","))
if access_token is None or user is None:
raise RuntimeError(
"ACCESS_TOKEN and GITHUB_ACTOR environment variables cannot be None!"
)
async with aiohttp.ClientSession() as session:
s = Stats(user, access_token, session)
s = Stats(user, access_token, session,
exclude_repos=exclude_repos,
ignore_forked_repos=ignore_forked_repos)
print(await s.to_str())


Expand Down