From f08c626675303f1a54d68493a27683c2530a8749 Mon Sep 17 00:00:00 2001 From: MoonJeWoong Date: Sun, 29 Sep 2024 23:48:23 +0900 Subject: [PATCH 1/6] add cli_download_and_extract method test Signed-off-by: MoonJeWoong --- tests/test_download.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index ec20978..67befcb 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -6,14 +6,33 @@ from fosslight_util.download import cli_download_and_extract from tests import constants +from git import Repo def test_download_from_github(): + # given + git_url = "https://github.com/LGE-OSS/example" + target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") + log_dir = "test_result/download_log/example" + # when + success, _, _, _ = cli_download_and_extract(git_url, target_dir, log_dir) + + # then + assert success is True + assert len(os.listdir(target_dir)) > 0 + + +@pytest.mark.parametrize("git_url", + ["git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;branch=ci-test", + "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;tag=v32"]) +def test_download_from_github_with_branch_or_tag(git_url): + # given target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") - success, _, _, _ = cli_download_and_extract("https://github.com/LGE-OSS/example", - target_dir, - "test_result/download_log/example") + log_dir = "test_result/download_log/example" + + # when + success, _, _, _ = cli_download_and_extract(git_url, target_dir, log_dir) # then assert success is True From 464021d8d5ce1515538b11c5bcc269788e3f80d2 Mon Sep 17 00:00:00 2001 From: MoonJeWoong Date: Mon, 30 Sep 2024 22:52:25 +0900 Subject: [PATCH 2/6] refactor download logic with gitPython Signed-off-by: MoonJeWoong --- src/fosslight_util/download.py | 40 ++++++++++++++-------------------- tests/test_download.py | 35 +++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index a4eff90..b97b8ee 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -10,7 +10,7 @@ import logging import argparse import shutil -import pygit2 as git +from git import Repo import bz2 import contextlib from datetime import datetime @@ -230,14 +230,9 @@ def get_github_token(git_url): def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""): - ref_to_checkout = decide_checkout(checkout_to, tag, branch) - msg = "" oss_name = get_github_ossname(git_url) - oss_version = "" - github_token = get_github_token(git_url) - callbacks = None - if github_token != "": - callbacks = git.RemoteCallbacks(credentials=git.UserPass("foo", github_token)) # username is not used, so set to dummy + refs_to_checkout = decide_checkout(checkout_to, tag, branch) + msg = "" try: if platform.system() != "Windows": @@ -248,9 +243,18 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""): alarm.start() Path(target_dir).mkdir(parents=True, exist_ok=True) - repo = git.clone_repository(git_url, target_dir, - bare=False, repository=None, - remote=None, callbacks=callbacks) + if refs_to_checkout != "": + # gitPython uses the branch argument the same whether you check out to a branch or a tag. + repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout) + logger.info(f"git checkout: {refs_to_checkout}") + else: + repo = Repo.clone_from(git_url, target_dir) + + if refs_to_checkout != tag: + oss_version = repo.active_branch.name + else: + oss_version = repo.git.describe('--tags') + if platform.system() != "Windows": signal.alarm(0) else: @@ -258,20 +262,8 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""): except Exception as error: logger.warning(f"git clone - failed: {error}") msg = str(error) - return False, msg, oss_name, oss_version - try: - if ref_to_checkout != "": - ref_list = [x for x in repo.references] - ref_to_checkout = get_ref_to_checkout(ref_to_checkout, ref_list) - logger.info(f"git checkout: {ref_to_checkout}") - repo.checkout(ref_to_checkout) + return False, msg, oss_name, refs_to_checkout - for prefix_ref in prefix_refs: - if ref_to_checkout.startswith(prefix_ref): - oss_version = ref_to_checkout[len(prefix_ref):] - - except Exception as error: - logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}") return True, msg, oss_name, oss_version diff --git a/tests/test_download.py b/tests/test_download.py index 67befcb..b4aa9ae 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -4,9 +4,8 @@ import os import pytest -from fosslight_util.download import cli_download_and_extract +from fosslight_util.download import cli_download_and_extract, download_git_clone from tests import constants -from git import Repo def test_download_from_github(): @@ -57,3 +56,35 @@ def test_download_from_wget(project_name, project_url): # then assert success is True assert len(os.listdir(target_dir)) > 0 + + +def test_download_git_clone_with_branch(): + # given + git_url = "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git" + target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") + branch_name = "ci-test" + + # when + success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, branch=branch_name) + + # then + assert success is True + assert len(os.listdir(target_dir)) > 0 + assert oss_name == '' + assert oss_version == branch_name + + +def test_download_git_clone_with_tag(): + # given + git_url = "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git" + target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") + tag_name = "v32" + + # when + success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, tag=tag_name) + + # then + assert success is True + assert len(os.listdir(target_dir)) > 0 + assert oss_name == '' + assert oss_version == tag_name From 8acc1ebaebdbd731107b7b8d0aff65ac7e689666 Mon Sep 17 00:00:00 2001 From: MoonJeWoong Date: Mon, 30 Sep 2024 22:52:48 +0900 Subject: [PATCH 3/6] refactor tox and requirements Signed-off-by: MoonJeWoong --- requirements-dev.txt | 3 ++- requirements.txt | 3 ++- tox.ini | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index bacb9b2..4211798 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,5 @@ pytest pytest-cov pytest-flake8 flake8==3.9.2 -fosslight-source \ No newline at end of file +fosslight-source +GitPython diff --git a/requirements.txt b/requirements.txt index be836ab..79fd6d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerabil numpy; python_version < '3.8' numpy>=1.22.2; python_version >= '3.8' npm -requests \ No newline at end of file +requests +GitPython diff --git a/tox.ini b/tox.ini index 9786910..481768a 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ exclude = .tox/* filterwarnings = ignore::DeprecationWarning norecursedirs = test_result/* tests/legacy + [testenv:test_run] deps = -r{toxinidir}/requirements-dev.txt From dc090771e1f01739a7f4630c18fa7e076fa8687a Mon Sep 17 00:00:00 2001 From: MoonJeWoong Date: Sun, 6 Oct 2024 19:45:49 +0900 Subject: [PATCH 4/6] refactor for downloading default branch Signed-off-by: MoonJeWoong --- src/fosslight_util/download.py | 19 +++++++++---- tests/test_download.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index b97b8ee..e732ea8 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -10,7 +10,7 @@ import logging import argparse import shutil -from git import Repo +from git import Repo, GitCommandError import bz2 import contextlib from datetime import datetime @@ -232,6 +232,7 @@ def get_github_token(git_url): def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""): oss_name = get_github_ossname(git_url) refs_to_checkout = decide_checkout(checkout_to, tag, branch) + clone_default_branch_flag = False msg = "" try: @@ -244,16 +245,24 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""): Path(target_dir).mkdir(parents=True, exist_ok=True) if refs_to_checkout != "": - # gitPython uses the branch argument the same whether you check out to a branch or a tag. - repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout) - logger.info(f"git checkout: {refs_to_checkout}") + try: + # gitPython uses the branch argument the same whether you check out to a branch or a tag. + repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout) + except GitCommandError as error: + error_msg = error.args[2].decode("utf-8") + if "Remote branch " + refs_to_checkout + " not found in upstream origin" in error_msg: + # clone default branch, when non-existent branch or tag entered + repo = Repo.clone_from(git_url, target_dir) + clone_default_branch_flag = True else: repo = Repo.clone_from(git_url, target_dir) + clone_default_branch_flag = True - if refs_to_checkout != tag: + if refs_to_checkout != tag or clone_default_branch_flag: oss_version = repo.active_branch.name else: oss_version = repo.git.describe('--tags') + logger.info(f"git checkout: {oss_version}") if platform.system() != "Windows": signal.alarm(0) diff --git a/tests/test_download.py b/tests/test_download.py index b4aa9ae..e976632 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -88,3 +88,53 @@ def test_download_git_clone_with_tag(): assert len(os.listdir(target_dir)) > 0 assert oss_name == '' assert oss_version == tag_name + + +def test_download_main_branch_when_any_branch_or_tag_not_entered(): + # given + git_url = "https://github.com/LGE-OSS/example" + target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") + expected_oss_name = "main" + + # when + success, _, oss_name, oss_version = download_git_clone(git_url, target_dir) + + # then + assert success is True + assert len(os.listdir(target_dir)) > 0 + assert oss_name == 'LGE-OSS-example' + assert oss_version == expected_oss_name + + +def test_download_main_branch_when_non_existent_branch_entered(): + # given + git_url = "https://github.com/LGE-OSS/example" + target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") + branch_name = "non-existent-branch" + expected_oss_name = "main" + + # when + success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, branch=branch_name) + + # then + assert success is True + assert len(os.listdir(target_dir)) > 0 + assert oss_name == 'LGE-OSS-example' + assert oss_version == expected_oss_name + + +def test_download_main_branch_when_non_existent_tag_entered(): + # given + git_url = "https://github.com/LGE-OSS/example" + target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example") + tag_name = "non-existent-tag" + expected_oss_name = "main" + + # when + success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, tag=tag_name) + + # then + assert success is True + assert len(os.listdir(target_dir)) > 0 + assert oss_name == 'LGE-OSS-example' + assert oss_version == expected_oss_name From db8ab8103ddf4a6fc550e715b596f049cde5a22a Mon Sep 17 00:00:00 2001 From: MoonJeWoong Date: Sun, 6 Oct 2024 19:50:06 +0900 Subject: [PATCH 5/6] remove dependency requirements-dev Signed-off-by: MoonJeWoong --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4211798..54813f0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,3 @@ pytest-cov pytest-flake8 flake8==3.9.2 fosslight-source -GitPython From 7a67b1d7b1cef34a11e48a496cef610cd3acf4d7 Mon Sep 17 00:00:00 2001 From: Soim Date: Sun, 6 Oct 2024 20:17:13 +0900 Subject: [PATCH 6/6] Update requirements-dev.txt --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 54813f0..6a2e9df 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ pytest-cov pytest-flake8 flake8==3.9.2 fosslight-source +spdx-tools==0.8.2