Package Details: python-courlan 1.4.0-1

Git Clone URL: https://aur.archlinux.org/python-courlan.git (read-only, click to copy)
Package Base: python-courlan
Description: Clean, filter and sample URLs to optimize data collection
Upstream URL: https://github.com/adbar/courlan
Licenses: Apache-2.0
Submitter: lfleischer
Maintainer: lfleischer
Last Packager: lfleischer
Votes: 0
Popularity: 0.000000
First Submitted: 2025-10-06 00:55 (UTC)
Last Updated: 2026-07-16 16:54 (UTC)

Latest Comments

Tyrin.price commented on 2026-06-08 23:41 (UTC)

Fails to install:

tests/urlstore_tests.py ...F. [100%]

========================================================================================== FAILURES =========================================================================================== ____________ testfrom_html ______________

def test_from_html():
    "Test link extraction procedures."
    url_store = UrlStore()
    url_store.trailing_slash = False
    base_url = "https://example.org"
    htmlstring = '<html><body><a href="https://example.com/page1"/><a href="https://example.org/page1/"/><a href="https://test.org/page1"/></body></html>'
    # 1 internal link in total
    url_store.add_from_html(htmlstring, base_url)
    rules = pickle.loads(
        b"\x80\x03curllib.robotparser\nRobotFileParser\nq\x00)\x81q\x01}q\x02(X\x07\x00\x00\x00entriesq\x03]q\x04X\r\x00\x00\x00default_entryq\x05NX\x0c\x00\x00\x00disallow_allq\x06\x89X\t\x00\x00\x00allow_allq\x07\x89X\x03\x00\x00\x00urlq\x08X\x1f\x00\x00\x00https://sitemaps.org/robots.txtq\tX\x04\x00\x00\x00hostq\nX\x0c\x00\x00\x00sitemaps.orgq\x0bX\x04\x00\x00\x00pathq\x0cX\x0b\x00\x00\x00/robots.txtq\rX\x0c\x00\x00\x00last_checkedq\x0eGA\xd8\x87\xf5\xdc\xab\xd5\x00ub."
    )
    url_store.store_rules("https://example.org", rules)
    assert len(url_store.find_known_urls(base_url)) == 1
    assert len(url_store.find_unvisited_urls(base_url)) == 1
    # same with content already seen

  url_store.add_from_html(htmlstring, base_url)

tests/urlstore_tests.py:414:

courlan/urlstore.py:313: in add_from_html links, links_priority = filter_links( courlan/core.py:259: in filter_links rules is not None and not rules.can_fetch("*", link) ^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib/python3.14/urllib/robotparser.py:189: in can_fetch entry = self._find_entry(useragent) ^^^^^^^^^^^^^^^^^^^^^^^^^^^

self = <urllib.robotparser.RobotFileParser object at 0x7f706dc8a710>, useragent = '*'

def _find_entry(self, useragent):

  entry = self.groups.get(useragent.lower())
            ^^^^^^^^^^^

E AttributeError: 'RobotFileParser' object has no attribute 'groups'

/usr/lib/python3.14/urllib/robotparser.py:159: AttributeError ====================================================================================== warnings summary ======================================================================================= tests/unit_tests.py::test_urlutils /usr/lib/python3.14/site-packages/tld/utils.py:209: DeprecationWarning: codecs.open() is deprecated. Use open() instead. local_file = codecs_open(local_path, "r", encoding="utf8")

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html =================================================================================== short test summary info =================================================================================== FAILED tests/urlstore_tests.py::test_from_html - AttributeError: 'RobotFileParser' object has no attribute 'groups' ==================================================================== 1 failed, 25 passed, 1 deselected, 1 warning in 1.69s ==================================================================== ==> ERROR: A failure occurred in check(). Aborting... -> error making: python-courlan-exit status 4 -> Failed to install the following packages. Manual intervention is required: python-courlan - exit status 4