import argparse
import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import zipfile
import re
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
SAVE_DIR = 'sites'
def make_request(url, refer='', proxy_url=''):
"""Make HTTP request with optional refer and proxy."""
headers = {
'User-Agent': USER_AGENT
}
if refer:
headers['Referer'] = refer
if proxy_url:
url = proxy_url + url
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
return response.text
except requests.exceptions.RequestException as e:
print(f"Error making request: {e}")
return None
def is_file_link(url):
"""Check if URL points to a file."""
path = urlparse(url).path
return bool(re.search(r'\.\w+$', path))
def save_file(file_url, base_url, save_dir, refer='', proxy_url=''):
"""Download and save file locally."""
if file_url.startswith('//'):
full_url = urlparse(base_url).scheme + ':' + file_url
elif file_url.startswith(('http://', 'https://')):
full_url = file_url
else:
full_url = urljoin(base_url.rstrip('/') + '/', file_url.lstrip('/'))
parsed_url = urlparse(full_url)
path = parsed_url.path
extension = os.path.splitext(path)[1][1:] or 'unknown'
filename = os.path.basename(path)
extension_dir = os.path.join(save_dir, extension)
os.makedirs(extension_dir, exist_ok=True)
file_path = os.path.join(extension_dir, filename)
try:
response = requests.get(full_url, headers={'User-Agent': USER_AGENT, 'Referer': refer} if refer else {'User-Agent': USER_AGENT})
response.raise_for_status()
with open(file_path, 'wb') as f:
f.write(response.content)
print(f"Saved file: {full_url}")
return file_path
except requests.exceptions.RequestException as e:
print(f"Error saving file {full_url}: {e}")
return None
def process_file_tags(soup, base_url, save_dir, refer, proxy_url):
"""Process HTML tags containing file references."""
file_tags = {
'script': 'src',
'link': 'href',
'img': 'src',
'audio': 'src',
'video': 'src',
'source': 'src',
'img': 'data-original'
}
modified_content = str(soup)
for tag, attribute in file_tags.items():
elements = soup.find_all(tag, attrs={attribute: True})
for element in elements:
file_url = element[attribute]
print(f"Detected link: {file_url}")
if is_file_link(file_url):
new_file_path = save_file(file_url, base_url, save_dir, refer, proxy_url)
if new_file_path:
extension = os.path.splitext(new_file_path)[1][1:]
relative_path = f'./{extension}/{os.path.basename(new_file_path)}'
modified_content = modified_content.replace(file_url, relative_path)
else:
print(f"Skipping link: {file_url}")
return modified_content
def zip_directory(source_dir, zip_path):
"""Create ZIP archive of downloaded files."""
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
for root, _, files in os.walk(source_dir):
for file in files:
file_path = os.path.join(root, file)
arcname = os.path.relpath(file_path, source_dir)
zipf.write(file_path, arcname)
def main():
parser = argparse.ArgumentParser(description='Website Mirror Tool')
parser.add_argument('url', help='URL to mirror')
parser.add_argument('--refer', help='Referrer URL', default='')
parser.add_argument('--proxy', help='Proxy URL', default='')
args = parser.parse_args()
parsed_url = urlparse(args.url)
host = parsed_url.netloc.replace('.', '')
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
save_dir = os.path.join(SAVE_DIR, host)
os.makedirs(save_dir, exist_ok=True)
html_content = make_request(args.url, args.refer, args.proxy)
if not html_content:
print("Failed to retrieve webpage")
return
soup = BeautifulSoup(html_content, 'html.parser')
modified_content = process_file_tags(soup, base_url, save_dir, args.refer, args.proxy)
html_file_path = os.path.join(save_dir, f"{host}.html")
with open(html_file_path, 'w', encoding='utf-8') as f:
f.write(modified_content)
print(f"Saved HTML file: {html_file_path}")
zip_path = os.path.join(SAVE_DIR, f"{host}.zip")
zip_directory(save_dir, zip_path)
print(f"Created ZIP archive: {zip_path}")
if __name__ == '__main__':
main()
主要功能:
在当前目录下创建一个 sites 文件夹
在 sites 文件夹中按照域名创建子文件夹
下载的文件会按类型分类存放 (如 images 文件夹存放图片,css 文件夹存放样式表等)
自动修改 HTML 文件中的链接,指向本地文件
最后生成一个 ZIP 压缩包,包含所有下载的文件
例如,如果你下载 https://example.com:
HTML 文件会保存为 sites/examplecom/examplecom.htm
所有文件会压缩为 sites/examplecom.zip
图片会保存在 sites/examplecom/images/ 目录
CSS 文件会保存在 sites/examplecom/css/ 目录
JavaScript 文件会保存在 sites/examplecom/js/ 目录
运行时会在控制台显示
: 检测到的文件链接
下载保存的文件路径
最终生成的 HTML 和 ZIP 文件位置