{ "about": "Each url serves exact bytes with an exact Content-Type. Fetch it, apply selector (parsel semantics: E::text is the direct child text nodes of E), and compare with browser.values, which is what Chrome extracted from the same response.", "stacks": [ { "id": "frostwork", "language": "Python", "label": "Frostwork main at 6669bbe (unreleased)", "how": "frostwork.extract(), one streaming pass over the bytes", "kind": "http" }, { "id": "scrapy", "language": "Python", "label": "Scrapy 2.19.0", "how": "HtmlResponse(...).css(), with w3lib 2.5.0", "kind": "http" }, { "id": "scrapy(w3lib+parsel)", "language": "Python", "label": "w3lib 2.5.0 + parsel 1.12.1", "how": "html_to_unicode with no fallback detector, then Selector(text=)", "kind": "http" }, { "id": "parsel(body=)", "language": "Python", "label": "parsel 1.12.1", "how": "Selector(body=, encoding=)", "kind": "http" }, { "id": "requests.text+lxml", "language": "Python", "label": "requests 2.34.2 + lxml 6.1.3", "how": "response.text into lxml.html", "kind": "http" }, { "id": "requests.text+parsel", "language": "Python", "label": "requests 2.34.2 + parsel 1.12.1", "how": "response.text into Selector(text=)", "kind": "http" }, { "id": "requests.text+bs4", "language": "Python", "label": "requests 2.34.2 + Beautiful Soup 4.15.0", "how": "response.text into BeautifulSoup(..., \"html.parser\")", "kind": "http" }, { "id": "requests+apparent", "language": "Python", "label": "requests 2.34.2 + charset-normalizer 3.5.2", "how": "response.apparent_encoding forced on", "kind": "http" }, { "id": "crawlee.py(bs4)", "language": "Python", "label": "Crawlee for Python 1.11.0, BeautifulSoupCrawler", "how": "raw bytes to BeautifulSoup(features=\"lxml\")", "kind": "http" }, { "id": "crawlee.py(parsel)", "language": "Python", "label": "Crawlee for Python 1.11.0, ParselCrawler", "how": "Selector(body=) with no encoding", "kind": "http" }, { "id": "lxml(bytes)", "language": "Python", "label": "lxml 6.1.3 (libxml2 2.14.6)", "how": "lxml.html on the raw bytes", "kind": "http" }, { "id": "html5lib", "language": "Python", "label": "html5lib 1.1", "how": "the WHATWG prescan over the raw bytes", "kind": "http" }, { "id": "bs4+dammit", "language": "Python", "label": "Beautiful Soup 4.15.0 + UnicodeDammit", "how": "BeautifulSoup(bytes, \"html.parser\", from_encoding=), UnicodeDammit sniffing the rest", "kind": "http" }, { "id": "selectolax(lexbor)", "language": "Python", "label": "selectolax 1.0.2 (lexbor)", "how": "the lexbor HTML5 parser on the raw bytes", "kind": "http" }, { "id": "fetch.text+cheerio", "language": "JavaScript", "label": "fetch + cheerio 1.2.0 (Node 26.8.2)", "how": "await res.text() into cheerio.load", "kind": "http" }, { "id": "crawlee(cheerio)", "language": "JavaScript", "label": "Crawlee 3.18.2, CheerioCrawler", "how": "Crawlee's own header-only charset rule", "kind": "http" }, { "id": "crawlee(jsdom)", "language": "JavaScript", "label": "Crawlee 3.18.2, JSDOMCrawler", "how": "Crawlee's charset rule, then jsdom", "kind": "http" }, { "id": "jsdom(bytes)", "language": "JavaScript", "label": "jsdom 30.1.3", "how": "new JSDOM(bytes, {contentType})", "kind": "http" }, { "id": "jsoup(bytes)", "language": "Java", "label": "jsoup 1.24.1", "how": "Jsoup.parse(bytes) with no charset", "kind": "http" }, { "id": "jsoup(http-charset)", "language": "Java", "label": "jsoup 1.24.1 with the HTTP charset", "how": "Jsoup.parse(bytes, )", "kind": "http" }, { "id": "jdk-http.ofString+jsoup", "language": "Java", "label": "java.net.http + jsoup 1.24.1 (JDK 21.0.3)", "how": "BodyHandlers.ofString() into Jsoup.parse", "kind": "http" }, { "id": "regex scraper", "language": "Python", "label": "a regular expression", "how": "the HTTP charset or UTF-8, then one regular expression per selector", "kind": "http" }, { "id": "playwright(chromium)", "language": "Python", "label": "Playwright 1.64.1, Chromium", "how": "in the page", "engine": "Chromium 156.0.8078.4", "kind": "browser" }, { "id": "playwright(firefox)", "language": "Python", "label": "Playwright 1.64.1, Firefox", "how": "in the page", "engine": "Firefox 157.0", "kind": "browser" }, { "id": "playwright(webkit)", "language": "Python", "label": "Playwright 1.64.1, WebKit", "how": "in the page", "engine": "WebKit 27.2", "kind": "browser" }, { "id": "crawlee.py(playwright)", "language": "Python", "label": "Crawlee for Python 1.11.0, PlaywrightCrawler", "how": "in the page", "engine": "Chromium 156.0.8078.4", "kind": "browser" }, { "id": "selenium(chrome)", "language": "Python", "label": "Selenium 4.52.0, Chrome", "how": "in the page", "engine": "Chrome 156.0.8078.80", "kind": "browser" }, { "id": "selenium(chrome)+bs4", "language": "Python", "label": "Selenium 4.52.0, Chrome, page_source into Beautiful Soup", "how": "re-parsed: driver.page_source into BeautifulSoup", "engine": "Chrome 156.0.8078.80", "kind": "browser" }, { "id": "scrapy-playwright", "language": "Python", "label": "Scrapy 2.19.0 + scrapy-playwright 0.0.48", "how": "re-parsed: page.content() into Scrapy's HtmlResponse", "engine": "Chromium 156.0.8078.4", "kind": "browser" }, { "id": "puppeteer", "language": "JavaScript", "label": "Puppeteer 25.16.0", "how": "in the page", "engine": "Chrome for Testing 156.0.8078.47", "kind": "browser" }, { "id": "crawlee(playwright)", "language": "JavaScript", "label": "Crawlee 3.18.2, PlaywrightCrawler", "how": "in the page", "engine": "Chromium 156.0.8078.4", "kind": "browser" }, { "id": "crawlee(playwright+cheerio)", "language": "JavaScript", "label": "Crawlee 3.18.2, PlaywrightCrawler + parseWithCheerio()", "how": "re-parsed: parseWithCheerio()", "engine": "Chromium 156.0.8078.4", "kind": "browser" }, { "id": "scrapy+w3lib-2.4.1", "language": "Python", "label": "Scrapy 2.19.0 with w3lib 2.4.1", "how": "HtmlResponse(...).css(), on the w3lib that `pip install -U scrapy` leaves behind", "kind": "http", "outdated": true } ], "cases": [ { "url": "/c/no-declaration", "readers": { "frostwork": [ "café — “Excellent”" ], "scrapy": [ "café — “Excellent”" ], "scrapy(w3lib+parsel)": [ "café — “Excellent”" ], "parsel(body=)": [ "café — “Excellent”" ], "requests.text+lxml": [ "café — “Excellent”" ], "requests.text+parsel": [ "café — “Excellent”" ], "requests.text+bs4": [ "café — “Excellent”" ], "requests+apparent": [ "café — “Excellent”" ], "crawlee.py(bs4)": [ "café — “Excellent”" ], "crawlee.py(parsel)": [ "café — “Excellent”" ], "lxml(bytes)": [ "café — “Excellent”" ], "html5lib": [ "café — “Excellentâ€�" ], "bs4+dammit": [ "café — “Excellent”" ], "selectolax(lexbor)": [ "café — “Excellent”" ], "fetch.text+cheerio": [ "café — “Excellent”" ], "crawlee(cheerio)": [ "café — “Excellent”" ], "crawlee(jsdom)": [ "café — “Excellentâ€�" ], "jsdom(bytes)": [ "café — “Excellent”" ], "jsoup(bytes)": [ "café — “Excellent”" ], "jsoup(http-charset)": [ "café — “Excellent”" ], "jdk-http.ofString+jsoup": [ "café — “Excellent”" ], "regex scraper": [ "café — “Excellent”" ], "playwright(chromium)": [ "café — “Excellent”" ], "playwright(firefox)": [ "café — “Excellent”" ], "playwright(webkit)": [ "café — “Excellent”" ], "crawlee.py(playwright)": [ "café — “Excellent”" ], "selenium(chrome)": [ "café — “Excellent”" ], "selenium(chrome)+bs4": [ "café — “Excellent”" ], "scrapy-playwright": [ "café — “Excellent”" ], "puppeteer": [ "café — “Excellent”" ], "crawlee(playwright)": [ "café — “Excellent”" ], "crawlee(playwright+cheerio)": [ "café — “Excellent”" ], "scrapy+w3lib-2.4.1": [ "café — “Excellent”" ] }, "id": "no-declaration", "title": "no charset anywhere, UTF-8 bytes", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "sniffing", "note": "the HTML spec's fallback is locale-dependent (windows-1252 in practice); every Python scraper stack defaults to UTF-8 instead", "nav": "top-level", "sha": "6ab01dcd774ee8c0", "size": 93, "browser": { "values": [ "café — “Excellent”" ], "charset": "windows-1252", "nav": "top-level", "build": "Chrome/156.0.8078.80", "locale": "en-GB" } }, { "url": "/c/comment-decoy", "readers": { "frostwork": [ "café — “Excellent”" ], "scrapy": [ "café — “Excellent”" ], "scrapy(w3lib+parsel)": [ "café — “Excellent”" ], "parsel(body=)": [ "café — “Excellent”" ], "requests.text+lxml": [ "café — “Excellent”" ], "requests.text+parsel": [ "café — “Excellent”" ], "requests.text+bs4": [ "café — “Excellent”" ], "requests+apparent": [ "café — “Excellent”" ], "crawlee.py(bs4)": [ "café — “Excellent”" ], "crawlee.py(parsel)": [ "café — “Excellent”" ], "lxml(bytes)": [ "café — “Excellent”" ], "html5lib": [ "café — “Excellent”" ], "bs4+dammit": [ "café — “Excellent”" ], "selectolax(lexbor)": [ "café — “Excellent”" ], "fetch.text+cheerio": [ "café — “Excellent”" ], "crawlee(cheerio)": [ "café — “Excellent”" ], "crawlee(jsdom)": [ "café — “Excellent”" ], "jsdom(bytes)": [ "café — “Excellent”" ], "jsoup(bytes)": [ "café — “Excellent”" ], "jsoup(http-charset)": [ "café — “Excellent”" ], "jdk-http.ofString+jsoup": [ "café — “Excellent”" ], "regex scraper": [ "café — “Excellent”" ], "playwright(chromium)": [ "café — “Excellent”" ], "playwright(firefox)": [ "café — “Excellent”" ], "playwright(webkit)": [ "café — “Excellent”" ], "crawlee.py(playwright)": [ "café — “Excellent”" ], "selenium(chrome)": [ "café — “Excellent”" ], "selenium(chrome)+bs4": [ "café — “Excellent”" ], "scrapy-playwright": [ "café — “Excellent”" ], "puppeteer": [ "café — “Excellent”" ], "crawlee(playwright)": [ "café — “Excellent”" ], "crawlee(playwright+cheerio)": [ "café — “Excellent”" ], "scrapy+w3lib-2.4.1": [ "caf矇 ��� ��諪xcellent���" ] }, "id": "comment-decoy", "title": "a inside a comment, before the real one", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "sniffing", "note": "w3lib has no comment handling and takes the FIRST regex hit, which is the decoy; WHATWG's prescan skips comments and reaches the real declaration", "nav": "iframe", "sha": "ca83a14008fb0f53", "size": 141, "browser": { "values": [ "café — “Excellent”" ], "charset": "UTF-8", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/stray-quote", "readers": { "frostwork": [ "中文" ], "scrapy": [ "¤¤¤å" ], "scrapy(w3lib+parsel)": [ "¤¤¤å" ], "parsel(body=)": [ "����" ], "requests.text+lxml": [ "¤¤¤å" ], "requests.text+parsel": [ "¤¤¤å" ], "requests.text+bs4": [ "¤¤¤å" ], "requests+apparent": [ "¤¤¤å" ], "crawlee.py(bs4)": [ "中文" ], "crawlee.py(parsel)": [ "中文" ], "lxml(bytes)": [ "中文" ], "html5lib": [ "中文" ], "bs4+dammit": [ "¤¤¤å" ], "selectolax(lexbor)": [ "¤¤¤å" ], "fetch.text+cheerio": [ "����" ], "crawlee(cheerio)": [ "����" ], "crawlee(jsdom)": [ "中文" ], "jsdom(bytes)": [ "中文" ], "jsoup(bytes)": [ "¤¤¤å" ], "jsoup(http-charset)": [ "¤¤¤å" ], "jdk-http.ofString+jsoup": [ "����" ], "regex scraper": [ "����" ], "playwright(chromium)": [ "中文" ], "playwright(firefox)": [ "中文" ], "playwright(webkit)": [ "中文" ], "crawlee.py(playwright)": [ "中文" ], "selenium(chrome)": [ "中文" ], "selenium(chrome)+bs4": [ "中文" ], "scrapy-playwright": [ "中文" ], "puppeteer": [ "中文" ], "crawlee(playwright)": [ "中文" ], "crawlee(playwright+cheerio)": [ "中文" ], "scrapy+w3lib-2.4.1": [ "¤¤¤å" ] }, "id": "stray-quote", "title": "stray quote inside an unquoted charset value", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "sniffing", "note": "an unquoted attribute value ends at whitespace or `>`, so the label is `windows-1252\"` -> invalid -> continue to big5. w3lib stops at the quote and takes windows-1252", "nav": "iframe", "sha": "672597cff1a240de", "size": 169, "browser": { "values": [ "中文" ], "charset": "Big5", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/meta-utf16", "readers": { "frostwork": [ "café — “Excellent”" ], "scrapy": [], "scrapy(w3lib+parsel)": [], "parsel(body=)": [ "café — “Excellent”" ], "requests.text+lxml": [ "café — “Excellent”" ], "requests.text+parsel": [ "café — “Excellent”" ], "requests.text+bs4": [ "café — “Excellent”" ], "requests+apparent": [ "café — “Excellent”" ], "crawlee.py(bs4)": [ "café — “Excellent”" ], "crawlee.py(parsel)": [ "café — “Excellent”" ], "lxml(bytes)": [], "html5lib": [ "café — “Excellent”" ], "bs4+dammit": [], "selectolax(lexbor)": [ "café — “Excellent”" ], "fetch.text+cheerio": [ "café — “Excellent”" ], "crawlee(cheerio)": [ "café — “Excellent”" ], "crawlee(jsdom)": [ "café — “Excellent”" ], "jsdom(bytes)": [ "café — “Excellent”" ], "jsoup(bytes)": [], "jsoup(http-charset)": [], "jdk-http.ofString+jsoup": [ "café — “Excellent”" ], "regex scraper": [ "café — “Excellent”" ], "playwright(chromium)": [ "café — “Excellent”" ], "playwright(firefox)": [ "café — “Excellent”" ], "playwright(webkit)": [ "café — “Excellent”" ], "crawlee.py(playwright)": [ "café — “Excellent”" ], "selenium(chrome)": [ "café — “Excellent”" ], "selenium(chrome)+bs4": [ "café — “Excellent”" ], "scrapy-playwright": [ "café — “Excellent”" ], "puppeteer": [ "café — “Excellent”" ], "crawlee(playwright)": [ "café — “Excellent”" ], "crawlee(playwright+cheerio)": [ "café — “Excellent”" ], "scrapy+w3lib-2.4.1": [] }, "id": "meta-utf16", "title": " on ASCII-compatible bytes", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "sniffing", "note": "the prescan could only READ it by assuming ASCII-compat, so WHATWG rewrites it to UTF-8", "nav": "iframe", "sha": "a61e2e1dc7ecb543", "size": 116, "browser": { "values": [ "café — “Excellent”" ], "charset": "UTF-8", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/http-x-user-defined", "readers": { "frostwork": [ "1.299,00 " ], "scrapy": [ "1.299,00 €" ], "scrapy(w3lib+parsel)": [ "1.299,00 �" ], "parsel(body=)": [ "ERR:LookupError: unknown encoding: x-user-defined" ], "requests.text+lxml": [ "1.299,00 �" ], "requests.text+parsel": [ "1.299,00 �" ], "requests.text+bs4": [ "1.299,00 �" ], "requests+apparent": [ "1.299,00 €" ], "crawlee.py(bs4)": [ "1.299,00 €" ], "crawlee.py(parsel)": [ "1.299,00 �" ], "lxml(bytes)": [ "1.299,00 €" ], "html5lib": [ "1.299,00 €" ], "bs4+dammit": [ "1.299,00 €" ], "selectolax(lexbor)": [ "1.299,00 �" ], "fetch.text+cheerio": [ "1.299,00 �" ], "crawlee(cheerio)": [ "ERR:Error: Resource http://127.0.0.1:8765/c/case served with unsupported charset/" ], "crawlee(jsdom)": [ "ERR:Error: Resource http://127.0.0.1:8765/c/case served with unsupported charset/" ], "jsdom(bytes)": [ "1.299,00 " ], "jsoup(bytes)": [ "1.299,00 �" ], "jsoup(http-charset)": [ "ERR:UnsupportedCharsetException: x-user-defined" ], "jdk-http.ofString+jsoup": [ "1.299,00 �" ], "regex scraper": [ "ERR:LookupError: unknown encoding: x-user-defined" ], "playwright(chromium)": [ "1.299,00 " ], "playwright(firefox)": [ "1.299,00 " ], "playwright(webkit)": [ "1.299,00 " ], "crawlee.py(playwright)": [ "1.299,00 " ], "selenium(chrome)": [ "1.299,00 " ], "selenium(chrome)+bs4": [ "1.299,00 " ], "scrapy-playwright": [ "1.299,00 " ], "puppeteer": [ "1.299,00 " ], "crawlee(playwright)": [ "1.299,00 " ], "crawlee(playwright+cheerio)": [ "1.299,00 " ], "scrapy+w3lib-2.4.1": [ "1.299,00 €" ] }, "id": "http-x-user-defined", "title": "HTTP charset=x-user-defined (no meta rewrite)", "content_type": "text/html; charset=x-user-defined", "declares_content_type": true, "selector": "#t::text", "mechanism": "sniffing", "note": "from HTTP the label is NOT rewritten: every high byte -> U+F780+b private use", "nav": "iframe", "sha": "9d2e5466ded30be3", "size": 78, "browser": { "values": [ "1.299,00 " ], "charset": "x-user-defined", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/replacement-enc", "readers": { "frostwork": [], "scrapy": [ "Members only: 4 seats left" ], "scrapy(w3lib+parsel)": [ "Members only: 4 seats left" ], "parsel(body=)": [ "Members only: 4 seats left" ], "requests.text+lxml": [ "Members only: 4 seats left" ], "requests.text+parsel": [ "Members only: 4 seats left" ], "requests.text+bs4": [ "Members only: 4 seats left" ], "requests+apparent": [ "Members only: 4 seats left" ], "crawlee.py(bs4)": [ "Members only: 4 seats left" ], "crawlee.py(parsel)": [ "Members only: 4 seats left" ], "lxml(bytes)": [ "Members only: 4 seats left" ], "html5lib": [ "Members only: 4 seats left" ], "bs4+dammit": [ "Members only: 4 seats left" ], "selectolax(lexbor)": [ "Members only: 4 seats left" ], "fetch.text+cheerio": [ "Members only: 4 seats left" ], "crawlee(cheerio)": [ "Members only: 4 seats left" ], "crawlee(jsdom)": [ "Members only: 4 seats left" ], "jsdom(bytes)": [], "jsoup(bytes)": [ "Members only: 4 seats left" ], "jsoup(http-charset)": [ "Members only: 4 seats left" ], "jdk-http.ofString+jsoup": [ "Members only: 4 seats left" ], "regex scraper": [ "Members only: 4 seats left" ], "playwright(chromium)": [], "playwright(firefox)": [], "playwright(webkit)": [], "crawlee.py(playwright)": [], "selenium(chrome)": [], "selenium(chrome)+bs4": [], "scrapy-playwright": [], "puppeteer": [], "crawlee(playwright)": [], "crawlee(playwright+cheerio)": [], "scrapy+w3lib-2.4.1": [ "Members only: 4 seats left" ] }, "id": "replacement-enc", "title": " (WHATWG 'replacement')", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "sniffing", "note": "WHATWG maps this label to the `replacement` decoder: the WHOLE document becomes one U+FFFD", "nav": "iframe", "sha": "963fc4499d546cfa", "size": 121, "browser": { "values": [], "charset": "replacement", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/xmldecl-late", "readers": { "frostwork": [ "café — “Excellent”" ], "scrapy": [ "caf矇 ��� ��諪xcellent���" ], "scrapy(w3lib+parsel)": [ "caf矇 ��� ��諪xcellent���" ], "parsel(body=)": [ "café — “Excellent”" ], "requests.text+lxml": [ "café — “Excellent”" ], "requests.text+parsel": [ "café — “Excellent”" ], "requests.text+bs4": [ "café — “Excellent”" ], "requests+apparent": [ "café — “Excellent”" ], "crawlee.py(bs4)": [ "café — “Excellent”" ], "crawlee.py(parsel)": [ "café — “Excellent”" ], "lxml(bytes)": [ "café — “Excellent”" ], "html5lib": [ "café — “Excellent”" ], "bs4+dammit": [ "café — “Excellent”" ], "selectolax(lexbor)": [ "café — “Excellent”" ], "fetch.text+cheerio": [ "café — “Excellent”" ], "crawlee(cheerio)": [ "café — “Excellent”" ], "crawlee(jsdom)": [ "café — “Excellent”" ], "jsdom(bytes)": [ "café — “Excellent”" ], "jsoup(bytes)": [ "café — “Excellent”" ], "jsoup(http-charset)": [ "café — “Excellent”" ], "jdk-http.ofString+jsoup": [ "café — “Excellent”" ], "regex scraper": [ "café — “Excellent”" ], "playwright(chromium)": [ "café — “Excellent”" ], "playwright(firefox)": [ "café — “Excellent”" ], "playwright(webkit)": [ "café — “Excellent”" ], "crawlee.py(playwright)": [ "café — “Excellent”" ], "selenium(chrome)": [ "café — “Excellent”" ], "selenium(chrome)+bs4": [ "café — “Excellent”" ], "scrapy-playwright": [ "café — “Excellent”" ], "puppeteer": [ "café — “Excellent”" ], "crawlee(playwright)": [ "café — “Excellent”" ], "crawlee(playwright+cheerio)": [ "café — “Excellent”" ], "scrapy+w3lib-2.4.1": [ "caf矇 ��� ��諪xcellent���" ] }, "id": "xmldecl-late", "title": " not at offset 0", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "sniffing", "note": "a ` (control)", "content_type": "text/html; charset=windows-1252", "declares_content_type": true, "selector": "#t::text", "mechanism": "sniffing", "note": "everyone should agree here — the control row", "nav": "iframe", "sha": "7cfa35a4ccfa304f", "size": 100, "browser": { "values": [ "1.299,00 €" ], "charset": "windows-1252", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/bom-vs-http", "readers": { "frostwork": [ "café — “Excellent”" ], "scrapy": [ "café — “Excellent”" ], "scrapy(w3lib+parsel)": [ "café — “Excellent”" ], "parsel(body=)": [ "café — “Excellentâ€�" ], "requests.text+lxml": [ "café — “Excellentâ€�" ], "requests.text+parsel": [ "café — “Excellentâ€�" ], "requests.text+bs4": [ "café — “Excellentâ€�" ], "requests+apparent": [ "café — “Excellent”" ], "crawlee.py(bs4)": [ "café — “Excellent”" ], "crawlee.py(parsel)": [ "café — “Excellent”" ], "lxml(bytes)": [ "café — “Excellent”" ], "html5lib": [ "café — “Excellent”" ], "bs4+dammit": [ "café — “Excellent”" ], "selectolax(lexbor)": [ "café — “Excellent”" ], "fetch.text+cheerio": [ "café — “Excellent”" ], "crawlee(cheerio)": [ "café — “Excellentâ€�" ], "crawlee(jsdom)": [ "café â€â€� “Excellent�" ], "jsdom(bytes)": [ "café — “Excellent”" ], "jsoup(bytes)": [ "café — “Excellent”" ], "jsoup(http-charset)": [ "café — “Excellent”" ], "jdk-http.ofString+jsoup": [ "café — “Excellentâ€�" ], "regex scraper": [ "café — “Excellentâ€�" ], "playwright(chromium)": [ "café — “Excellent”" ], "playwright(firefox)": [ "café — “Excellent”" ], "playwright(webkit)": [ "café — “Excellent”" ], "crawlee.py(playwright)": [ "café — “Excellent”" ], "selenium(chrome)": [ "café — “Excellent”" ], "selenium(chrome)+bs4": [ "café — “Excellent”" ], "scrapy-playwright": [ "café — “Excellent”" ], "puppeteer": [ "café — “Excellent”" ], "crawlee(playwright)": [ "café — “Excellent”" ], "crawlee(playwright+cheerio)": [ "café — “Excellent”" ], "scrapy+w3lib-2.4.1": [ "café — “Excellent”" ] }, "id": "bom-vs-http", "title": "UTF-8 BOM outranks a conflicting HTTP charset (control)", "content_type": "text/html; charset=windows-1252", "declares_content_type": true, "selector": "#t::text", "mechanism": "sniffing", "note": "control row: BOM first for everyone", "nav": "iframe", "sha": "3bfacae73fd79c9e", "size": 96, "browser": { "values": [ "café — “Excellent”" ], "charset": "UTF-8", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/latin1-label", "readers": { "frostwork": [ "99–149" ], "scrapy": [ "99–149" ], "scrapy(w3lib+parsel)": [ "99–149" ], "parsel(body=)": [ "99–149" ], "requests.text+lxml": [ "99–149" ], "requests.text+parsel": [ "99–149" ], "requests.text+bs4": [ "99–149" ], "requests+apparent": [ "99–149" ], "crawlee.py(bs4)": [ "99–149" ], "crawlee.py(parsel)": [ "99–149" ], "lxml(bytes)": [ "99–149" ], "html5lib": [ "99–149" ], "bs4+dammit": [ "99–149" ], "selectolax(lexbor)": [ "99�149" ], "fetch.text+cheerio": [ "99�149" ], "crawlee(cheerio)": [ "99–149" ], "crawlee(jsdom)": [ "99–149" ], "jsdom(bytes)": [ "99–149" ], "jsoup(bytes)": [ "99�149" ], "jsoup(http-charset)": [ "99–149" ], "jdk-http.ofString+jsoup": [ "99–149" ], "regex scraper": [ "99–149" ], "playwright(chromium)": [ "99–149" ], "playwright(firefox)": [ "99–149" ], "playwright(webkit)": [ "99–149" ], "crawlee.py(playwright)": [ "99–149" ], "selenium(chrome)": [ "99–149" ], "selenium(chrome)+bs4": [ "99–149" ], "scrapy-playwright": [ "99–149" ], "puppeteer": [ "99–149" ], "crawlee(playwright)": [ "99–149" ], "crawlee(playwright+cheerio)": [ "99–149" ], "scrapy+w3lib-2.4.1": [ "99–149" ] }, "id": "latin1-label", "title": "charset=iso-8859-1 with byte 0x96", "content_type": "text/html; charset=iso-8859-1", "declares_content_type": true, "selector": "#t::text", "mechanism": "decoder", "note": "the WHATWG label iso-8859-1 MEANS windows-1252 (0x96 = en dash). Python's literal latin_1 codec gives the invisible control U+0096", "nav": "iframe", "sha": "7d1b33d1fdf09d23", "size": 74, "browser": { "values": [ "99–149" ], "charset": "windows-1252", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/eucjp-wave", "readers": { "frostwork": [ "～ ①" ], "scrapy": [ "〜 ��" ], "scrapy(w3lib+parsel)": [ "〜 ��" ], "parsel(body=)": [ "〜 ��" ], "requests.text+lxml": [ "〜 ��" ], "requests.text+parsel": [ "〜 ��" ], "requests.text+bs4": [ "〜 ��" ], "requests+apparent": [ "℅ 迭" ], "crawlee.py(bs4)": [ "〜 ��" ], "crawlee.py(parsel)": [ "〜 ��" ], "lxml(bytes)": [ "¡Á ­¡" ], "html5lib": [ "〜 ��" ], "bs4+dammit": [ "℅ 迭" ], "selectolax(lexbor)": [ "�� ��" ], "fetch.text+cheerio": [ "�� ��" ], "crawlee(cheerio)": [ "～ ①" ], "crawlee(jsdom)": [ "～ â‘ " ], "jsdom(bytes)": [ "～ ①" ], "jsoup(bytes)": [ "�� ��" ], "jsoup(http-charset)": [ "〜 �" ], "jdk-http.ofString+jsoup": [ "〜 �" ], "regex scraper": [ "〜 ��" ], "playwright(chromium)": [ "～ ①" ], "playwright(firefox)": [ "～ ①" ], "playwright(webkit)": [ "～ ①" ], "crawlee.py(playwright)": [ "～ ①" ], "selenium(chrome)": [ "～ ①" ], "selenium(chrome)+bs4": [ "～ ①" ], "scrapy-playwright": [ "～ ①" ], "puppeteer": [ "～ ①" ], "crawlee(playwright)": [ "～ ①" ], "crawlee(playwright+cheerio)": [ "～ ①" ], "scrapy+w3lib-2.4.1": [ "〜 ��" ] }, "id": "eucjp-wave", "title": "euc-jp A1C1 (wave dash) + ADA1 (circled 1)", "content_type": "text/html; charset=euc-jp", "declares_content_type": true, "selector": "#t::text", "mechanism": "decoder", "note": "WHATWG/CP932: U+FF5E and U+2460. Python's strict euc_jp: U+301C and U+FFFD", "nav": "iframe", "sha": "5a27a77f4dc544a7", "size": 73, "browser": { "values": [ "～ ①" ], "charset": "EUC-JP", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/gb18030-pua", "readers": { "frostwork": [ "a　b" ], "scrapy": [ "ab" ], "scrapy(w3lib+parsel)": [ "ab" ], "parsel(body=)": [ "ab" ], "requests.text+lxml": [ "ab" ], "requests.text+parsel": [ "ab" ], "requests.text+bs4": [ "ab" ], "requests+apparent": [ "aŁ b" ], "crawlee.py(bs4)": [ "ab" ], "crawlee.py(parsel)": [ "ab" ], "lxml(bytes)": [ "a£ b" ], "html5lib": [ "ab" ], "bs4+dammit": [ "ab" ], "selectolax(lexbor)": [ "a��b" ], "fetch.text+cheerio": [ "a��b" ], "crawlee(cheerio)": [ "a　b" ], "crawlee(jsdom)": [ "a　b" ], "jsdom(bytes)": [ "a　b" ], "jsoup(bytes)": [ "a��b" ], "jsoup(http-charset)": [ "ab" ], "jdk-http.ofString+jsoup": [ "ab" ], "regex scraper": [ "ab" ], "playwright(chromium)": [ "a　b" ], "playwright(firefox)": [ "a　b" ], "playwright(webkit)": [ "a　b" ], "crawlee.py(playwright)": [ "a　b" ], "selenium(chrome)": [ "a　b" ], "selenium(chrome)+bs4": [ "a　b" ], "scrapy-playwright": [ "a　b" ], "puppeteer": [ "a　b" ], "crawlee(playwright)": [ "a　b" ], "crawlee(playwright+cheerio)": [ "a　b" ], "scrapy+w3lib-2.4.1": [ "ab" ] }, "id": "gb18030-pua", "title": "gb18030 A3A0", "content_type": "text/html; charset=gb18030", "declares_content_type": true, "selector": "#t::text", "mechanism": "decoder", "note": "GB18030-2005 moved this out of the PUA: U+3000 here, U+E5E5 in Python's older index", "nav": "iframe", "sha": "8cc2ef4aa554a5bf", "size": 72, "browser": { "values": [ "a　b" ], "charset": "gb18030", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/cp1252-c1", "readers": { "frostwork": [ "xyz" ], "scrapy": [ "x�y�z" ], "scrapy(w3lib+parsel)": [ "x�y�z" ], "parsel(body=)": [ "x�y�z" ], "requests.text+lxml": [ "x�y�z" ], "requests.text+parsel": [ "x�y�z" ], "requests.text+bs4": [ "x�y�z" ], "requests+apparent": [ "x【鉱" ], "crawlee.py(bs4)": [ "x�y�z" ], "crawlee.py(parsel)": [ "x�y�z" ], "lxml(bytes)": [ "xyz" ], "html5lib": [ "x�y�z" ], "bs4+dammit": [ "x【鉱" ], "selectolax(lexbor)": [ "x�y�z" ], "fetch.text+cheerio": [ "x�y�z" ], "crawlee(cheerio)": [ "x�y�z" ], "crawlee(jsdom)": [ "x�y�z" ], "jsdom(bytes)": [ "xyz" ], "jsoup(bytes)": [ "x�y�z" ], "jsoup(http-charset)": [ "x�y�z" ], "jdk-http.ofString+jsoup": [ "x�y�z" ], "regex scraper": [ "x�y�z" ], "playwright(chromium)": [ "xyz" ], "playwright(firefox)": [ "xyz" ], "playwright(webkit)": [ "xyz" ], "crawlee.py(playwright)": [ "xyz" ], "selenium(chrome)": [ "xyz" ], "selenium(chrome)+bs4": [ "xyz" ], "scrapy-playwright": [ "xyz" ], "puppeteer": [ "xyz" ], "crawlee(playwright)": [ "xyz" ], "crawlee(playwright+cheerio)": [ "xyz" ], "scrapy+w3lib-2.4.1": [ "x�y�z" ] }, "id": "cp1252-c1", "title": "windows-1252 bytes 81 8D 8F 90 9D", "content_type": "text/html; charset=windows-1252", "declares_content_type": true, "selector": "#t::text", "mechanism": "decoder", "note": "a WHATWG index is TOTAL; Python's cp1252 leaves these five undefined -> U+FFFD", "nav": "iframe", "sha": "f6dffdebbcadb1cb", "size": 73, "browser": { "values": [ "xyz" ], "charset": "windows-1252", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/indented-bom", "readers": { "frostwork": [ "Real Title" ], "scrapy": [ "Real Title" ], "scrapy(w3lib+parsel)": [ "Real Title" ], "parsel(body=)": [ "Real Title" ], "requests.text+lxml": [ "Real Title" ], "requests.text+parsel": [ "Real Title" ], "requests.text+bs4": [ "Real Title" ], "requests+apparent": [ "Real Title" ], "crawlee.py(bs4)": [ "Real Title" ], "crawlee.py(parsel)": [ "Real Title" ], "lxml(bytes)": [ "Real Title" ], "html5lib": [ "Real Title" ], "bs4+dammit": [ "Real Title" ], "selectolax(lexbor)": [ "Real Title" ], "fetch.text+cheerio": [ "Real Title" ], "crawlee(cheerio)": [ "Real Title" ], "crawlee(jsdom)": [ "Real Title" ], "jsdom(bytes)": [ "Real Title" ], "jsoup(bytes)": [ "Real Title" ], "jsoup(http-charset)": [ "Real Title" ], "jdk-http.ofString+jsoup": [ "Real Title" ], "regex scraper": [ "Real Title" ], "playwright(chromium)": [ "Real Title" ], "playwright(firefox)": [ "Real Title" ], "playwright(webkit)": [ "Real Title" ], "crawlee.py(playwright)": [ "Real Title" ], "selenium(chrome)": [ "Real Title" ], "selenium(chrome)+bs4": [ "Real Title" ], "scrapy-playwright": [ "Real Title" ], "puppeteer": [ "Real Title" ], "crawlee(playwright)": [ "Real Title" ], "crawlee(playwright+cheerio)": [ "Real Title" ], "scrapy+w3lib-2.4.1": [ "Real Title" ] }, "id": "indented-bom", "title": "a U+FEFF preceded by whitespace", "content_type": "text/html", "declares_content_type": false, "selector": "title::text", "mechanism": "normalization", "note": "parsel parses text.strip(), which PROMOTES the U+FEFF to offset 0 where libxml2 eats it as a BOM. On the raw bytes it is a character before the frame -> the head is lost", "nav": "iframe", "sha": "68dc8cadac567139", "size": 117, "browser": { "values": [ "Real Title" ], "charset": "UTF-8", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/vertical-tab", "readers": { "frostwork": [ "Real Title" ], "scrapy": [ "Real Title" ], "scrapy(w3lib+parsel)": [ "Real Title" ], "parsel(body=)": [ "Real Title" ], "requests.text+lxml": [ "Real Title" ], "requests.text+parsel": [ "Real Title" ], "requests.text+bs4": [ "Real Title" ], "requests+apparent": [ "Real Title" ], "crawlee.py(bs4)": [ "Real Title" ], "crawlee.py(parsel)": [ "Real Title" ], "lxml(bytes)": [ "Real Title" ], "html5lib": [ "ERR:ValueError: All strings must be XML compatible: Unicode or ASCII, no NULL bytes or" ], "bs4+dammit": [ "Real Title" ], "selectolax(lexbor)": [ "Real Title" ], "fetch.text+cheerio": [ "Real Title" ], "crawlee(cheerio)": [ "Real Title" ], "crawlee(jsdom)": [ "Real Title" ], "jsdom(bytes)": [ "Real Title" ], "jsoup(bytes)": [ "Real Title" ], "jsoup(http-charset)": [ "Real Title" ], "jdk-http.ofString+jsoup": [ "Real Title" ], "regex scraper": [ "Real Title" ], "playwright(chromium)": [ "Real Title" ], "playwright(firefox)": [ "Real Title" ], "playwright(webkit)": [ "Real Title" ], "crawlee.py(playwright)": [ "Real Title" ], "selenium(chrome)": [ "Real Title" ], "selenium(chrome)+bs4": [ "Real Title" ], "scrapy-playwright": [ "Real Title" ], "puppeteer": [ "Real Title" ], "crawlee(playwright)": [ "Real Title" ], "crawlee(playwright+cheerio)": [ "Real Title" ], "scrapy+w3lib-2.4.1": [ "Real Title" ] }, "id": "vertical-tab", "title": "a leading VERTICAL TAB (0x0B)", "content_type": "text/html", "declares_content_type": false, "selector": "title::text", "mechanism": "normalization", "note": "0x0B is whitespace to Python's str.strip() and NOT to libxml2 or HTML5", "nav": "iframe", "sha": "785b39252151fb42", "size": 103, "browser": { "values": [ "Real Title" ], "charset": "UTF-8", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/nul-split", "readers": { "frostwork": [], "scrapy": [ "deleted-nul text" ], "scrapy(w3lib+parsel)": [ "deleted-nul text" ], "parsel(body=)": [ "deleted-nul text" ], "requests.text+lxml": [], "requests.text+parsel": [ "deleted-nul text" ], "requests.text+bs4": [], "requests+apparent": [], "crawlee.py(bs4)": [], "crawlee.py(parsel)": [ "deleted-nul text" ], "lxml(bytes)": [], "html5lib": [], "bs4+dammit": [], "selectolax(lexbor)": [], "fetch.text+cheerio": [], "crawlee(cheerio)": [], "crawlee(jsdom)": [], "jsdom(bytes)": [], "jsoup(bytes)": [], "jsoup(http-charset)": [], "jdk-http.ofString+jsoup": [], "regex scraper": [], "playwright(chromium)": [], "playwright(firefox)": [], "playwright(webkit)": [], "crawlee.py(playwright)": [], "selenium(chrome)": [], "selenium(chrome)+bs4": [], "scrapy-playwright": [], "puppeteer": [], "crawlee(playwright)": [], "crawlee(playwright+cheerio)": [], "scrapy+w3lib-2.4.1": [ "deleted-nul text" ] }, "id": "nul-split", "title": "a NUL inside a tag name", "content_type": "text/html", "declares_content_type": false, "selector": "div::text", "mechanism": "normalization", "note": "parsel deletes NUL document-wide before parsing; HTML5 keeps it (U+FFFD in a tag name ends the name)", "nav": "iframe", "sha": "f4b3063803ce922a", "size": 90, "browser": { "values": [], "charset": "UTF-8", "nav": "iframe", "build": "Chrome/148", "locale": null } }, { "url": "/c/iso2022jp", "readers": { "frostwork": [ "社会" ], "scrapy": [ "社会" ], "scrapy(w3lib+parsel)": [ "社会" ], "parsel(body=)": [ "社会" ], "requests.text+lxml": [ "社会" ], "requests.text+parsel": [ "社会" ], "requests.text+bs4": [ "社会" ], "requests+apparent": [ "社会" ], "crawlee.py(bs4)": [ "社会" ], "crawlee.py(parsel)": [ "社会" ], "lxml(bytes)": [ "\u001b$B" ], "html5lib": [ "社会" ], "bs4+dammit": [ "社会" ], "selectolax(lexbor)": [ "\u001b$B" ], "fetch.text+cheerio": [ "\u001b$B" ], "crawlee(cheerio)": [ "ERR:Error: Resource http://127.0.0.1:8765/c/case served with unsupported charset/" ], "crawlee(jsdom)": [ "ERR:Error: Resource http://127.0.0.1:8765/c/case served with unsupported charset/" ], "jsdom(bytes)": [ "社会" ], "jsoup(bytes)": [ "\u001b$B" ], "jsoup(http-charset)": [ "社会" ], "jdk-http.ofString+jsoup": [ "社会" ], "regex scraper": [ "社会" ], "playwright(chromium)": [ "社会" ], "playwright(firefox)": [ "社会" ], "playwright(webkit)": [ "社会" ], "crawlee.py(playwright)": [ "社会" ], "selenium(chrome)": [ "社会" ], "selenium(chrome)+bs4": [ "社会" ], "scrapy-playwright": [ "社会" ], "puppeteer": [ "社会" ], "crawlee(playwright)": [ "社会" ], "crawlee(playwright+cheerio)": [ "社会" ], "scrapy+w3lib-2.4.1": [ "社会" ] }, "id": "iso2022jp", "title": "ISO-2022-JP: the word 社会 is the bytes ` is raw text only when scripting is ENABLED", "content_type": "text/html", "declares_content_type": false, "selector": "#t::text", "mechanism": "structure", "note": "with scripting on (a browser)