#!/usr/bin/env python3 """Public-signal audit, not a Google index-status test. Usage: python3 search-discovery-audit.py CHINESE_URL JAPANESE_URL The two arguments must be HTTPS URLs on the same origin. This script only downloads their public HTML, robots.txt, and at most five sitemap files. """ import json import sys from html.parser import HTMLParser from urllib.error import HTTPError from urllib.parse import urljoin, urlsplit from urllib.request import Request, urlopen from urllib.robotparser import RobotFileParser from xml.etree import ElementTree as ET MAX_BYTES = 2_000_000 USER_AGENT = "Gumi-public-discovery-audit/1.0" class Head(HTMLParser): def __init__(self): super().__init__() self.in_head = False self.lang = None self.links = [] self.meta = [] def handle_starttag(self, tag, attrs): attrs = dict(attrs) if tag == "html": self.lang = attrs.get("lang") if tag == "head": self.in_head = True if not self.in_head: return if tag == "link": self.links.append(attrs) elif tag == "meta": self.meta.append(attrs) def handle_endtag(self, tag): if tag == "head": self.in_head = False def link(self, relation): return [item.get("href") for item in self.links if relation in item.get("rel", "").lower().split()] def meta_value(self, name, key="name"): return [item.get("content", "") for item in self.meta if item.get(key, "").lower() == name] def download(url): try: response = urlopen(Request(url, headers={"User-Agent": USER_AGENT}), timeout=15) except HTTPError as error: response = error with response: body = response.read(MAX_BYTES + 1) if len(body) > MAX_BYTES: raise ValueError(f"Response exceeds {MAX_BYTES} bytes: {url}") return response.status, response.geturl(), response.headers, body def local_name(element): return element.tag.rsplit("}", 1)[-1] def sitemap_urls(origin, sitemap_url): status, _, _, body = download(sitemap_url) if status != 200: raise ValueError(f"Sitemap HTTP {status}: {sitemap_url}") root = ET.fromstring(body) if local_name(root) == "sitemapindex": children = [child.text for node in root if local_name(node) == "sitemap" for child in node if local_name(child) == "loc"] if not 1 <= len(children) <= 5: raise ValueError("This audit supports 1–5 sitemap files") combined = {} for child_url in children: if urlsplit(child_url).netloc != urlsplit(origin).netloc: raise ValueError("Cross-origin sitemap is outside this audit") combined.update(sitemap_urls(origin, child_url)) return combined if local_name(root) != "urlset": raise ValueError(f"Unexpected sitemap root: {local_name(root)}") result = {} for node in root: if local_name(node) != "url": continue location = next((child.text for child in node if local_name(child) == "loc"), None) if location: result[location] = { child.get("hreflang"): child.get("href") for child in node if local_name(child) == "link" and child.get("rel") == "alternate" } return result def audit(chinese_url, japanese_url): urls = [chinese_url, japanese_url] origins = {urlsplit(url).scheme + "://" + urlsplit(url).netloc for url in urls} if len(origins) != 1 or any(urlsplit(url).scheme != "https" for url in urls): raise ValueError("Provide two HTTPS URLs on the same origin") origin = origins.pop() robots_status, _, _, robots_body = download(origin + "/robots.txt") if robots_status != 200: raise ValueError(f"robots.txt HTTP {robots_status}") robots_text = robots_body.decode("utf-8") robot_parser = RobotFileParser() robot_parser.parse(robots_text.splitlines()) sitemap_candidates = [line.split(":", 1)[1].strip() for line in robots_text.splitlines() if line.lower().startswith("sitemap:")] if len(sitemap_candidates) != 1: raise ValueError("Expected one Sitemap line in robots.txt") sitemap_url = sitemap_candidates[0] sitemap = sitemap_urls(origin, sitemap_url) expected_alternates = {"zh-CN": chinese_url, "ja-JP": japanese_url} pages = [] for url, language in zip(urls, ("zh-CN", "ja-JP")): status, final_url, headers, body = download(url) head = Head() if status == 200: head.feed(body.decode("utf-8")) canonical_links = [urljoin(final_url, href) for href in head.link("canonical") if href] sitemap_links = [urljoin(final_url, href) for href in head.link("sitemap") if href] rss_links = [urljoin(final_url, item.get("href")) for item in head.links if item.get("type") == "application/rss+xml" and item.get("href")] metadata_noindex = any("noindex" in content.lower() for name in ("robots", "googlebot") for content in head.meta_value(name)) header_noindex = "noindex" in headers.get("X-Robots-Tag", "").lower() og_url = head.meta_value("og:url", key="property") advertised = {link: download(link)[0] for link in set(sitemap_links + rss_links)} record = { "url": url, "http": status, "finalUrl": final_url, "htmlLang": head.lang, "canonical": canonical_links, "ogUrl": og_url, "metaNoindex": metadata_noindex, "headerNoindex": header_noindex, "robotsAllowsGooglebotByPythonParser": robot_parser.can_fetch("Googlebot", url), "sitemapListed": url in sitemap, "sitemapAlternatesCorrect": all(sitemap.get(url, {}).get(lang) == href for lang, href in expected_alternates.items()), "advertisedLinkStatus": advertised, "googleIndexStatus": "unknown: Search Console not queried", } record["publicSignalsPass"] = ( status == 200 and final_url == url and head.lang == language and canonical_links == [url] and og_url == [url] and not metadata_noindex and not header_noindex and record["robotsAllowsGooglebotByPythonParser"] and record["sitemapListed"] and record["sitemapAlternatesCorrect"] and all(link_status == 200 for link_status in advertised.values()) ) pages.append(record) return {"sitemap": sitemap_url, "sitemapUrlCount": len(sitemap), "pages": pages, "allPublicSignalsPass": all(page["publicSignalsPass"] for page in pages)} def self_test(): clean = Head() clean.feed('' '') assert clean.lang == "ja-JP" and clean.link("canonical") == ["https://e.test/ja/"] assert clean.meta_value("og:url", "property") == ["https://e.test/ja/"] blocked = Head() blocked.feed('' '') assert "noindex" in blocked.meta_value("robots")[0] assert blocked.link("canonical") == [] and blocked.link("sitemap") == ["/missing.xml"] xml = ET.fromstring('https://e.test/' '' '') assert local_name(xml) == "urlset" and local_name(xml[0][1]) == "link" print("Parser boundaries passed: valid head, noindex/missing canonical, XML namespace.") if __name__ == "__main__": if sys.argv[1:] == ["--self-test"]: self_test() elif len(sys.argv) == 3: try: report = audit(sys.argv[1], sys.argv[2]) print(json.dumps(report, ensure_ascii=False, indent=2)) sys.exit(0 if report["allPublicSignalsPass"] else 1) except Exception as error: print(f"Audit failed: {error}", file=sys.stderr) sys.exit(2) else: print("Usage: python3 search-discovery-audit.py CHINESE_URL JAPANESE_URL | --self-test", file=sys.stderr) sys.exit(2)