1. urlparse() - URL解析

将URL字符串解析为6个组成部分的元组：

from urllib.parse import urlparse

url = "https://www.example.com:8080/path/to/page?name=value#section"
parsed = urlparse(url)

print(parsed.scheme)    # https
print(parsed.netloc)    # www.example.com:8080
print(parsed.path)      # /path/to/page
print(parsed.params)    # (空)
print(parsed.query)     # name=value
print(parsed.fragment)  # section

应用场景： 当需要获取URL中的特定部分（如域名、路径或查询参数）时，使用urlparse是最有效的方式。

2. urlunparse() - URL重组

将URL的组成部分重新组合成完整URL：

from urllib.parse import urlunparse

components = ('https', 'www.example.com', '/path/to/page', '', 'name=value', 'section')
full_url = urlunparse(components)
print(full_url)  # https://www.example.com/path/to/page?name=value#section

3. urlsplit() 与 urlunsplit()

类似urlparse，但将params合并到path中，返回5个元素的元组：

from urllib.parse import urlsplit, urlunsplit

url = "https://www.example.com/path;param?query=value#frag"
splitted = urlsplit(url)
print(splitted)  # SplitResult(scheme='https', netloc='www.example.com', path='/path;param', query='query=value', fragment='frag')

# 重组URL
unsplitted = urlunsplit(splitted)
print(unsplitted)  # https://www.example.com/path;param?query=value#frag

4. urljoin() - URL拼接

将基础URL与相对URL组合成绝对URL：

from urllib.parse import urljoin

base = "https://www.example.com/path/to/page.html"
print(urljoin(base, "another.html"))          # https://www.example.com/path/to/another.html
print(urljoin(base, "../other.html"))         # https://www.example.com/path/other.html
print(urljoin(base, "//otherdomain.com/img")) # https://otherdomain.com/img

应用场景： 在网页爬虫开发中，处理相对路径链接时特别有用。

5. parse_qs() 和 parse_qsl() - 查询参数解析

解析URL中的查询字符串：

from urllib.parse import parse_qs, parse_qsl

query = "name=John&age=30&city=New+York&hobbies=reading&hobbies=traveling"
print(parse_qs(query))  # {'name': ['John'], 'age': ['30'], 'city': ['New York'], 'hobbies': ['reading', 'traveling']}
print(parse_qsl(query)) # [('name', 'John'), ('age', '30'), ('city', 'New York'), ('hobbies', 'reading'), ('hobbies', 'traveling')]

案例1：提取URL中的域名

from urllib.parse import urlparse

def get_domain(url):
    parsed = urlparse(url)
    return parsed.netloc.split(':')[0]  # 移除端口部分

print(get_domain("https://subdomain.example.com:8080/path"))  # subdomain.example.com

案例2：修改URL的查询参数

from urllib.parse import urlparse, parse_qs, urlencode, urlunparse

def update_query_param(url, param, value):
    parsed = urlparse(url)
    query_dict = parse_qs(parsed.query)
    query_dict[param] = [value]  # 更新参数值
    new_query = urlencode(query_dict, doseq=True)
    return urlunparse(parsed._replace(query=new_query))

original = "https://example.com/page?name=John&age=30"
updated = update_query_param(original, "age", "35")
print(updated)  # https://example.com/page?name=John&age=35

案例3：网页爬虫中的URL规范化

from urllib.parse import urljoin, urlparse

def normalize_url(base, url):
    # 拼接URL
    full_url = urljoin(base, url)
    # 解析并重组以标准化
    parsed = urlparse(full_url)
    # 移除片段、默认端口等
    if parsed.scheme == "http" and parsed.port == 80:
        netloc = parsed.netloc.replace(":80", "")
    elif parsed.scheme == "https" and parsed.port == 443:
        netloc = parsed.netloc.replace(":443", "")
    else:
        netloc = parsed.netloc
        
    # 重新构建标准化URL
    return urlunparse((
        parsed.scheme,
        netloc,
        parsed.path.rstrip('/') or '/',  # 确保路径至少为'/'
        parsed.params,
        parsed.query,
        ''  # 移除片段
    ))

base = "https://example.com/path/to/page.html"
print(normalize_url(base, "../other.html#section"))  # https://example.com/path/other.html