Skip to content

Commit

Permalink
Use pytest parametrize on test_get_robots_txt_url
Browse files Browse the repository at this point in the history
  • Loading branch information
Mews committed Jun 19, 2024
1 parent 040770c commit c566fb8
Showing 1 changed file with 15 additions and 36 deletions.
51 changes: 15 additions & 36 deletions tests/networking/test_robots_txt.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,23 @@
from io import BytesIO
import urllib.robotparser

from tiny_web_crawler.networking.robots_txt import get_robots_txt_url, is_robots_txt_allowed, setup_robots_txt_parser
import pytest

def test_get_robots_txt_url() -> None:
assert (
get_robots_txt_url("http://example")
== "http://example/robots.txt"
)

assert (
get_robots_txt_url("http://example/path")
== "http://example/robots.txt"
)

assert (
get_robots_txt_url("https://example/")
== "https://example/robots.txt"
)

assert (
get_robots_txt_url("http://example/path1/path2/path3/path4")
== "http://example/robots.txt"
)

assert (
get_robots_txt_url("http://example/path1/path2/path3/path4")
== "http://example/robots.txt"
)

assert (
get_robots_txt_url("http://example/path#fragment")
== "http://example/robots.txt"
)

assert (
get_robots_txt_url("http://example/path?query=test")
== "http://example/robots.txt"
)
from tiny_web_crawler.networking.robots_txt import get_robots_txt_url, is_robots_txt_allowed, setup_robots_txt_parser

@pytest.mark.parametrize(
"url, expected",
[
("http://example", "http://example/robots.txt"),
("http://example/path", "http://example/robots.txt"),
("https://example/", "https://example/robots.txt"),
("http://example/path1/path2/path3/path4", "http://example/robots.txt"),
("http://example/path#fragment", "http://example/robots.txt"),
("http://example/path?query=test", "http://example/robots.txt"),
]
)
def test_get_robots_txt_url(url: str, expected: str) -> None:
assert get_robots_txt_url(url) == expected


@patch('urllib.request.urlopen')
Expand Down

0 comments on commit c566fb8

Please # to comment.