import os
import re
import json
import threading
from queue import Queue, Empty
import requests
from lxml import etree
from fake_useragent import UserAgent
from loguru import logger
from concurrent.futures import ThreadPoolExecutor
from tenacity import retry, stop_after_attempt, wait_exponential, before_sleep_log
# 初始化队列
first_queue = Queue() # 存储分类页URL
second_queue = Queue() # 存储详情页URL
# 配置日志
logger.remove()
logger.add("Record_url.log", rotation="500 MB", level="INFO")
logger.add(lambda msg: print(msg, end=""), level="INFO")
# User-Agent 生成器
ua = UserAgent()
# 线程安全的临时存储
temp_file_lock = threading.Lock()
TEMP_FILE = "temp_urls.jsonl" # 临时存储商品 URL 的 JSONL 文件
# 输出文件和锁
OUTPUT_FILE = "all_products.json"
file_lock = threading.Lock()
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10),
before_sleep=before_sleep_log(logger, logger.level("INFO").no))
def fetch_response(target_url):
"""获取网页响应，失败时重试3次"""
headers = {"User-Agent": ua.random}
response = requests.get(target_url, headers=headers, allow_redirects=False, timeout=10)
response.raise_for_status()
return response
def save_to_json(product_data):
"""线程安全地将产品数据保存到JSON文件"""
with file_lock:
if not os.path.exists(OUTPUT_FILE):
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
json.dump([], f)
try:
with open(OUTPUT_FILE, "r", encoding="utf-8") as f:
existing_products = json.load(f)
except (json.JSONDecodeError, FileNotFoundError):
existing_products = []
existing_products.append(product_data) # 直接添加，假设每次清空文件
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
json.dump(existing_products, f, ensure_ascii=False, indent=2)
logger.info(f"Saved product: {product_data['Title']} (URL: {product_data['URL']})")
def parse_data():
"""解析详情页数据"""
while True:
try:
item = second_queue.get(timeout=5)
url, category, mark, Other_list = item["url"], item["category"], item["mark"], item["Other_list"]
try:
response = fetch_response(url).text
tree = etree.HTML(response)
matcher = re.search('<script type="application/ld\+json">\s*(.*?)\s*</script>', response).group(1)
json_data = json.loads(matcher)
title = json_data.get("name", f"{url} - No Title")
body = json_data.get('description', "")
try:
price = json_data["offers"]["price"]
except:
price = "N/A"
color = json_data.get("color", "As show")
img_list = json_data.get("image", [])
if not img_list or price == "N/A":
logger.warning(f"Parse Data Error Images and Price:{item}")
continue
size_list = tree.xpath(
'//*[@class="c-product-attributes__item__selector button-select-size js-attr-selector "]/@data-pdp-attr-value') or [
"Standard Size"]
variant_image = img_list[0]
product_data = {
"Mark": mark,
"Title": title,
"Price": price,
"Color": color,
"Body": body,
"Img_list": img_list,
"Size_list": size_list,
"Other_link": Other_list,
"Category": category,
"variant_image": variant_image,
"URL": url
}
save_to_json(product_data)
except:
logger.error(f'Parse Data Error:{item}')
finally:
second_queue.task_done()
except Empty:
logger.info("- - - - - - - - - - - - - 详细队列为空，退出解析线程")
break
def get_detail_url():
"""从分类页提取详情页URL，写入临时文件"""
while True:
try:
item = first_queue.get(timeout=5)
category, baseurl = item["category"], item["url"]
start = 0
goods_link_num = 0
while True:
if "prefn1" in baseurl:
url = f'{baseurl}&start={start}&sz=36'
else:
url = f'{baseurl}?start={start}&sz=36'
try:
response = fetch_response(url).text
tree = etree.HTML(response)
div_list = tree.xpath('//*[contains(@class, "swatches c-product-tile__color-swatches__swatches")]')
if not div_list:
break
with temp_file_lock:
goods_link_num += len(div_list)
for div in div_list:
a_list = div.xpath('./a')
other_list = div.xpath('./a/@href')
for a in a_list:
href = a.xpath('./@href')[0]
filtered_other_list = [link for link in other_list if link != href]
# 输出结果
# 标记最好从域名后面开始切
mark = href
detail_url = "https://www.skechers.com" + href
data = {"mark": mark, "url": detail_url, "category": category,
"Other_list": filtered_other_list}
second_queue.put(data)
start += 36
except Exception as e:
logger.error(f"Frist Repeat Error:{url} -{e}")
break
logger.info(f"Processed {category} - Found {goods_link_num} items")
first_queue.task_done()
except Empty:
print("- - - - - - - - - - - - - - 类别队列为空，退出详细线程")
break
def get_classify():
"""加载分类URL到队列"""
items_list = [
{'category': 'Women>New Arrivals', 'url': 'https://www.skechers.com/new-arrivals/women/'},
{'category': 'Women>Hands Free Slip-ins',
'url': 'https://www.skechers.com/women/shoes/hands-free-slip-ins/?prefn1=gender&prefv1=W'},
{'category': 'Women>Sandals', 'url': 'https://www.skechers.com/women/shoes/sandals/?prefn1=gender&prefv1=W'},
{'category': 'Women>Best Sellers', 'url': 'https://www.skechers.com/best-sellers/women/'},
{'category': "Women>Women's Sale", 'url': 'https://www.skechers.com/sale/?prefn1=gender&prefv1=W'},
{'category': 'Women>Shoes>Athletic Sneakers', 'url': 'https://www.skechers.com/women/shoes/athletic-sneakers/'},
{'category': 'Women>Shoes>CasualSneakers', 'url': 'https://www.skechers.com/women/shoes/casual-sneakers/'},
{'category': 'Women>Shoes>Boots', 'url': 'https://www.skechers.com/women/shoes/boots/'},
{'category': 'Women>Shoes>Canvas Shoes', 'url': 'https://www.skechers.com/women/shoes/canvas-shoes/'},
{'category': 'Women>Shoes>Flats', 'url': 'https://www.skechers.com/women/shoes/flats/'},
{'category': 'Women>Shoes>Sandals', 'url': 'https://www.skechers.com/women/shoes/sandals/'},
{'category': 'Women>Shoes>Hands Free Slip-ins',
'url': 'https://www.skechers.com/women/shoes/hands-free-slip-ins/'},
{'category': 'Women>Shoes>Slip-Ons', 'url': 'https://www.skechers.com/women/shoes/slip-ons/'},
{'category': 'Women>Shoes>Lace Up', 'url': 'https://www.skechers.com/women/shoes/lace-up/'},
{'category': 'Women>Shoes>Wedges', 'url': 'https://www.skechers.com/women/shoes/wedges/'},
{'category': 'Women>Shoes>Boat Shoes', 'url': 'https://www.skechers.com/women/shoes/boat-shoes/'},
{'category': 'Women>Shoes>Slippers', 'url': 'https://www.skechers.com/women/shoes/slippers/'},
{'category': 'Women>Shoes>Work & Safety', 'url': 'https://www.skechers.com/women/shoes/work-safety/'},
{'category': 'Women>Shoes>Waterproof',
'url': 'https://www.skechers.com/technologies/features/waterproof/?prefn1=gender&prefv1=W'},
{'category': 'Women>Shop by Width>Wide',
'url': 'https://www.skechers.com/technologies/comfort-technologies/wides/?prefn1=gender&prefv1=W'},
{'category': 'Women>Shop by Activity>Basketball',
'url': 'https://www.skechers.com/technologies/collections/skechers-basketball/'},
{'category': 'Women>Shop by Activity>Golf', 'url': 'https://www.skechers.com/women/shop-by-activity/golf/'},
{'category': 'Women>Shop by Activity>Running',
'url': 'https://www.skechers.com/women/shop-by-activity/running/'},
{'category': 'Women>Shop by Activity>Trail & Hiking',
'url': 'https://www.skechers.com/women/shop-by-activity/trail-hiking/'},
{'category': 'Women>Shop by Activity>Training',
'url': 'https://www.skechers.com/women/shop-by-activity/training/'},
{'category': 'Women>Shop by Activity>Walking',
'url': 'https://www.skechers.com/women/shop-by-activity/walking/'},
{'category': 'Women>Shop by Activity>Pickleball',
'url': 'https://www.skechers.com/technologies/collections/pickleball/?prefn1=gender&prefv1=W'},
{'category': 'Women>Shop by Activity>Tennis',
'url': 'https://www.skechers.com/technologies/collections/tennis/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Air Cooled Memory Foam',
'url': 'https://www.skechers.com/technologies/comfort-technologies/air-cooled-memory-foam/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Arch Fit',
'url': 'https://www.skechers.com/women/comfort-technologies/arch-fit/'},
{'category': 'Women>Comfort Technologies>Cozy Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/cozy-fit/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Glide Step',
'url': 'https://www.skechers.com/technologies/comfort-technologies/glide-step/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Goga Mat',
'url': 'https://www.skechers.com/technologies/comfort-technologies/goga-mat/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Hands Free Slip-ins',
'url': 'https://www.skechers.com/technologies/comfort-technologies/slip-ins/?prefn1=gender&prefv1=W'},
{'category':'Women>Comfort Technologies>Hyper Burst',
'url': 'https://www.skechers.com/technologies/comfort-technologies/hyper-burst/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Luxe Foam',
'url': 'https://www.skechers.com/technologies/comfort-technologies/luxe-foam/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Massage Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/massage-fit/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Max Cushioning',
'url': 'https://www.skechers.com/women/comfort-technologies/max-cushioning/'},
{'category': 'Women>Comfort Technologies>Relaxed Fit',
'url': 'https://www.skechers.com/women/comfort-technologies/relaxed-fit/'},
{'category': 'Women>Comfort Technologies>Skech Air',
'url': 'https://www.skechers.com/technologies/comfort-technologies/skech-air/?prefn1=gender&prefv1=W'},
{'category': 'Women>Comfort Technologies>Stretch Fit',
'url': 'https://www.skechers.com/women/comfort-technologies/stretch-fit/'},
{'category': 'Women>Comfort Technologies>Ultra Go',
'url': 'https://www.skechers.com/technologies/comfort-technologies/ultra-go/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>BOBS',
'url': 'https://www.skechers.com/technologies/collections/bobs/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>Cali', 'url': 'https://www.skechers.com/technologies/collections/cali/'},
{'category': 'Women>Collections>Court & Classics',
'url': 'https://www.skechers.com/technologies/collections/court-classics/?prefn1=gender&prefv1=W'},
{'category': "Women>Collections>D'Lites",
'url': 'https://www.skechers.com/technologies/collections/dlites/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>Foamies', 'url': 'https://www.skechers.com/women/collections/foamies/'},
{'category': 'Women>Collections>GO FOAM',
'url': 'https://www.skechers.com/technologies/collections/go-foam/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>GO RUN',
'url': 'https://www.skechers.com/technologies/collections/go-run/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>GO WALK', 'url': 'https://www.skechers.com/women/collections/go-walk/'},
{'category': 'Women>Collections>GO RECOVER',
'url': 'https://www.skechers.com/technologies/collections/go-recover/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>Mark Nason', 'url': 'https://www.skechers.com/women/collections/mark-nason/'},
{'category': 'Women>Collections>Modern Comfort',
'url': 'https://www.skechers.com/women/collections/modern-comfort/'},
{'category': 'Women>Collections>Our Planet Matters',
'url': 'https://www.skechers.com/technologies/collections/our-planet-matters/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>SKECHERS Street™',
'url': 'https://www.skechers.com/women/collections/skechers-street/'}, {'category': 'Women>Collections>UNOs',
'url': 'https://www.skechers.com/technologies/collections/unos/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collections>Work & Safety', 'url': 'https://www.skechers.com/women/shoes/work-safety/'},
{'category': 'Women>Collaborations>Goldcrown',
'url': 'https://www.skechers.com/technologies/collections/goldcrown/?prefn1=gender&prefv1=W'},
{'category': 'Women>Collaborations>Martha Stewart',
'url': 'https://www.skechers.com/collaborations/martha-stewart/martha-stewart-styles/'},
{'category': 'Women>Collaborations>Snoop Dogg',
'url': 'https://www.skechers.com/technologies/collections/snoop-dogg/?prefn1=gender&prefv1=W'},
{'category': 'Women>Clothing & Accessories>Tops', 'url': 'https://www.skechers.com/clothing/womens/tops/'},
{'category': 'Women>Clothing & Accessories>Bottoms',
'url': 'https://www.skechers.com/clothing/womens/bottoms/'}, {'category': 'Women>Clothing & Accessories>Bags',
'url': 'https://www.skechers.com/clothing/accessories/bags/?prefn1=gender&prefv1=W%7CU'},
{'category': 'Women>Clothing & Accessories>Hats',
'url': 'https://www.skechers.com/clothing/accessories/hats/?prefn1=gender&prefv1=W%7CU'},
{'category': 'Women>Clothing & Accessories>Socks',
'url': 'https://www.skechers.com/clothing/accessories/socks/?prefn1=gender&prefv1=W%7CU'},
{'category': 'Men>New Arrivals', 'url': 'https://www.skechers.com/new-arrivals/men/'},
{'category': 'Men>Hands Free Slip-ins', 'url': 'https://www.skechers.com/men/shoes/hands-free-slip-ins/'},
{'category': 'Men>Sandals', 'url': 'https://www.skechers.com/men/shoes/sandals/?prefn1=gender&prefv1=M'},
{'category': 'Men>Best Sellers', 'url': 'https://www.skechers.com/best-sellers/men/'},
{'category': "Men>Men's Sale", 'url': 'https://www.skechers.com/sale/?prefn1=gender&prefv1=M'},
{'category': 'Men>Shoes>Athletic Sneakers', 'url': 'https://www.skechers.com/men/shoes/athletic-sneakers/'},
{'category': 'Men>Shoes>Casual Sneakers', 'url': 'https://www.skechers.com/men/shoes/casual-sneakers/'},
{'category': 'Men>Shoes>Boat Shoes', 'url': 'https://www.skechers.com/men/shoes/boat-shoes/'},
{'category': 'Men>Shoes>Boots', 'url': 'https://www.skechers.com/men/shoes/boots/'},
{'category': 'Men>Shoes>Hands Free Slip-ins', 'url': 'https://www.skechers.com/men/shoes/hands-free-slip-ins/'},
{'category': 'Men>Shoes>Slip-Ons', 'url': 'https://www.skechers.com/men/shoes/slip-ons/'},
{'category': 'Men>Shoes>Lace Up', 'url': 'https://www.skechers.com/men/shoes/lace-up/'},
{'category': 'Men>Shoes>Dress Shoes', 'url': 'https://www.skechers.com/men/shoes/dress-shoes/'},
{'category': 'Men>Shoes>Sandals', 'url': 'https://www.skechers.com/men/shoes/sandals/'},
{'category': 'Men>Shoes>Slippers', 'url': 'https://www.skechers.com/men/shoes/slippers/'},
{'category': 'Men>Shoes>Canvas Shoes', 'url': 'https://www.skechers.com/men/shoes/canvas-shoes/'},
{'category': 'Men>Shoes>Work & Safety', 'url': 'https://www.skechers.com/men/shoes/work-safety/'},
{'category': 'Men>Shoes>Waterproof',
'url': 'https://www.skechers.com/technologies/features/waterproof/?prefn1=gender&prefv1=M'},
{'category': 'Men>Shop by Width>Wide',
'url': 'https://www.skechers.com/technologies/comfort-technologies/wides/?prefn1=gender&prefv1=M'},
{'category': 'Men>Shop by Width>Extra-Wide', 'url': 'https://www.skechers.com/men/shop-by-width/extra-wide/'},
{'category': 'Men>Shop by Activity>Basketball',
'url': 'https://www.skechers.com/technologies/collections/skechers-basketball/'},
{'category': 'Men>Shop by Activity>Golf', 'url': 'https://www.skechers.com/men/shop-by-activity/golf/'},
{'category': 'Men>Shop by Activity>Running', 'url': 'https://www.skechers.com/men/shop-by-activity/running/'},
{'category': 'Men>Shop by Activity>Trail & Hiking',
'url': 'https://www.skechers.com/men/shop-by-activity/trail-hiking/'},
{'category': 'Men>Shop by Activity>Training', 'url': 'https://www.skechers.com/men/shop-by-activity/training/'},
{'category': 'Men>Shop by Activity>Walking', 'url': 'https://www.skechers.com/men/shop-by-activity/walking/'},
{'category': 'Men>Shop by Activity>Pickleball',
'url': 'https://www.skechers.com/technologies/collections/pickleball/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Men>Shop by Activity>Soccer',
'url': 'https://www.skechers.com/technologies/collections/soccer/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Men>Shop by Activity>Tennis',
'url': 'https://www.skechers.com/technologies/collections/tennis/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Air Cooled Memory Foam',
'url': 'https://www.skechers.com/technologies/comfort-technologies/air-cooled-memory-foam/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Arch Fit',
'url': 'https://www.skechers.com/men/comfort-technologies/arch-fit/'},
{'category': 'Men>Comfort Technologies>Cozy Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/cozy-fit/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Glide Step',
'url': 'https://www.skechers.com/technologies/comfort-technologies/glide-step/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Goga Mat',
'url': 'https://www.skechers.com/technologies/comfort-technologies/goga-mat/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Hands Free Slip-ins',
'url': 'https://www.skechers.com/technologies/comfort-technologies/slip-ins/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Hyper Burst',
'url': 'https://www.skechers.com/technologies/comfort-technologies/hyper-burst/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Luxe Foam',
'url': 'https://www.skechers.com/technologies/comfort-technologies/luxe-foam/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Massage Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/massage-fit/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Max Cushioning',
'url': 'https://www.skechers.com/men/comfort-technologies/max-cushioning/'},
{'category': 'Men>Comfort Technologies>Relaxed Fit',
'url': 'https://www.skechers.com/men/comfort-technologies/relaxed-fit/'},
{'category': 'Men>Comfort Technologies>Skech Air',
'url': 'https://www.skechers.com/technologies/comfort-technologies/skech-air/?prefn1=gender&prefv1=M'},
{'category': 'Men>Comfort Technologies>Stretch Fit',
'url': 'https://www.skechers.com/men/comfort-technologies/stretch-fit/'},
{'category': 'Men>Comfort Technologies>Ultra Go',
'url': 'https://www.skechers.com/technologies/comfort-technologies/ultra-go/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>Foamies', 'url': 'https://www.skechers.com/men/collections/foamies/'},
{'category': 'Men>Collections>Court & Classics',
'url': 'https://www.skechers.com/technologies/collections/court-classics/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>GO FOAM',
'url': 'https://www.skechers.com/technologies/collections/go-foam/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>GO GOLF',
'url': 'https://www.skechers.com/technologies/collections/go-golf/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>GO RUN',
'url': 'https://www.skechers.com/technologies/collections/go-run/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>GO WALK', 'url': 'https://www.skechers.com/men/collections/go-walk/'},
{'category': 'Men>Collections>GO RECOVER',
'url': 'https://www.skechers.com/technologies/collections/go-recover/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>John Deere',
'url': 'https://www.skechers.com/collaborations/john-deere/john-deere-styles/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>Mark Nason', 'url': 'https://www.skechers.com/men/collections/mark-nason/'},
{'category': 'Men>Collections>Our Planet Matters',
'url': 'https://www.skechers.com/technologies/collections/our-planet-matters/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>Pickleball',
'url': 'https://www.skechers.com/technologies/collections/pickleball/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Men>Collections>Snoop Dogg',
'url': 'https://www.skechers.com/technologies/collections/snoop-dogg/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>SKECHERS Streetwear™',
'url': 'https://www.skechers.com/men/collections/skechers-streetwear/'},
{'category': 'Men>Collections>SKECHERS USA', 'url': 'https://www.skechers.com/men/collections/skechers-usa/'},
{'category': 'Men>Collections>UNOs',
'url': 'https://www.skechers.com/technologies/collections/unos/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collections>Work & Safety', 'url': 'https://www.skechers.com/men/shoes/work-safety/'},
{'category': 'Men>Collaborations>Snoop Dogg',
'url': 'https://www.skechers.com/technologies/collections/snoop-dogg/?prefn1=gender&prefv1=M'},
{'category': 'Men>Collaborations>John Deere',
'url': 'https://www.skechers.com/collaborations/john-deere/john-deere-styles/?prefn1=gender&prefv1=M'},
{'category': 'Men>Clothing & Accessories>Tops', 'url': 'https://www.skechers.com/clothing/mens/tops/'},
{'category': 'Men>Clothing & Accessories>Bottoms', 'url': 'https://www.skechers.com/clothing/mens/bottoms/'},
{'category': 'Men>Clothing & Accessories>Hats',
'url': 'https://www.skechers.com/clothing/accessories/hats/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Men>Clothing & Accessories>Socks',
'url': 'https://www.skechers.com/clothing/accessories/socks/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Men>Clothing & Accessories>Watches',
'url': 'https://www.skechers.com/clothing/accessories/watches/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Kids>Shop Kids’ All', 'url': 'https://www.skechers.com/kids/shoes/'},
{'category': 'Kids>New Arrivals', 'url': 'https://www.skechers.com/new-arrivals/kids/'},
{'category': 'Kids>Best Sellers', 'url': 'https://www.skechers.com/best-sellers/kids/'},
{'category': 'Kids>John Deere', 'url': 'https://www.skechers.com/collaborations/john-deere/'},
{'category': "Kids>Kids' Sale", 'url': 'https://www.skechers.com/sale/?prefn1=gender&prefv1=G%7CB'},
{'category': 'Kids>Girls>Athletic Shoes & Sneakers',
'url': 'https://www.skechers.com/kids/girls/athletic-shoes-sneakers/'},
{'category': 'Kids>Girls>Sandals', 'url': 'https://www.skechers.com/kids/girls/sandals/'},
{'category': 'Kids>Girls>Boots', 'url': 'https://www.skechers.com/kids/girls/boots/'},
{'category': 'Kids>Girls>Slip-ons', 'url': 'https://www.skechers.com/kids/girls/slip-ons/'},
{'category': 'Kids>Girls>Lace Ups', 'url': 'https://www.skechers.com/kids/girls/lace-ups/'},
{'category': 'Kids>Girls>Hands Free Slip-ins',
'url': 'https://www.skechers.com/kids/girls/hands-free-slip-ins/'},
{'category': 'Kids>Girls>Light-up', 'url': 'https://www.skechers.com/kids/girls/light-up/'},
{'category': 'Kids>Girls Age Groups>Toddlers (Sizes 4-10)',
'url': 'https://www.skechers.com/kids/girls/?prefn1=kidsSizeBucket&prefv1=Toddlers%20%28Sizes%204-10%29'},
{'category': 'Kids>Girls Age Groups>Little Kids (Sizes 10.5-3.0)',
'url': 'https://www.skechers.com/kids/girls/?prefn1=kidsSizeBucket&prefv1=Little%20Kids%20%28Sizes%2010.5-3.0%29'},
{'category': 'Kids>Girls Age Groups>Big Kids (Sizes 3.5-7)',
'url': 'https://www.skechers.com/kids/girls/?prefn1=kidsSizeBucket&prefv1=Big%20Kids%20%28Sizes%203.5-7%29'},
{'category': 'Kids>Boys>Athletic Shoes & Sneakers',
'url': 'https://www.skechers.com/kids/boys/athletic-shoes-sneakers/'},
{'category': 'Kids>Boys>Sandals', 'url': 'https://www.skechers.com/kids/boys/sandals/'},
{'category': 'Kids>Boys>Boots', 'url': 'https://www.skechers.com/kids/boys/boots/'},
{'category': 'Kids>Boys>Slip-ons', 'url': 'https://www.skechers.com/kids/boys/slip-ons/'},
{'category': 'Kids>Boys>Lace Ups', 'url': 'https://www.skechers.com/kids/boys/lace-ups/'},
{'category': 'Kids>Boys>Hands Free Slip-ins', 'url': 'https://www.skechers.com/kids/boys/hands-free-slip-ins/'},
{'category': 'Kids>Boys>Light-up', 'url': 'https://www.skechers.com/kids/boys/light-up/'},
{'category': 'Kids>Boys Age Groups>Toddlers (Sizes 4-10)',
'url': 'https://www.skechers.com/kids/boys/?prefn1=kidsSizeBucket&prefv1=Toddlers%20%28Sizes%204-10%29'},
{'category': 'Kids>Boys Age Groups>Little Kids (Sizes 10.5-3.0)',
'url': 'https://www.skechers.com/kids/boys/?prefn1=kidsSizeBucket&prefv1=Little%20Kids%20%28Sizes%2010.5-3.0%29'},
{'category': 'Kids>Boys Age Groups>Big Kids (Sizes 3.5-7)',
'url': 'https://www.skechers.com/kids/boys/?prefn1=kidsSizeBucket&prefv1=Big%20Kids%20%28Sizes%203.5-7%29'},
{'category': 'Kids>Collections & Features>Foamies',
'url': 'https://www.skechers.com/kids/collections-features/foamies/'},
{'category': 'Kids>Collections & Features>UNOs',
'url': 'https://www.skechers.com/technologies/collections/unos/?prefn1=gender&prefv1=G%7CB'},
{'category': 'Kids>Collections & Features>Sport',
'url': 'https://www.skechers.com/kids/collections-features/sport/'},
{'category': 'Kids>Collections & Features>Twinkle Toes',
'url': 'https://www.skechers.com/kids/collections-features/twinkle-toes/'},
{'category': 'Kids>Collections & Features>SKECHERS Street',
'url': 'https://www.skechers.com/kids/collections-features/skechers-street/'},
{'category': 'Kids>Collections & Features>Uniform School Shoes',
'url': 'https://www.skechers.com/kids/collections-features/uniform-school-shoes/'},
{'category': 'Kids>Collections & Features>Light-ups',
'url': 'https://www.skechers.com/kids/collections-features/light-ups/'},
{'category': 'Kids>Collections & Features>Character & Themed',
'url': 'https://www.skechers.com/kids/collections-features/character-themed/'},
{'category': 'Kids>Collections & Features>John Deere',
'url': 'https://www.skechers.com/collaborations/john-deere/john-deere-styles/?prefn1=gender&prefv1=B'},
{'category': 'Kids>Collections & Features>Wide Width',
'url': 'https://www.skechers.com/technologies/comfort-technologies/wides/?prefn1=gender&prefv1=B%7CG'},
{'category': 'Kids>Collections & Features>Machine Washable',
'url': 'https://www.skechers.com/kids/collections-features/machine-washable/'},
{'category': 'Kids>Collections & Features>Water Friendly',
'url':'https://www.skechers.com/kids/collections-features/water-friendly/'},
{'category': 'Kids>Collections & Features>Adaptable',
'url': 'https://www.skechers.com/kids/collections-features/adaptable/'},
{'category': 'Kids>Accessories>Backpacks',
'url': 'https://www.skechers.com/clothing/accessories/bags/?prefn1=gender&prefv1=G%7CU'},
{'category': 'Kids>Accessories>Hats',
'url': 'https://www.skechers.com/clothing/accessories/hats/?prefn1=gender&prefv1=G%7CB'},
{'category': 'Kids>Accessories>Socks',
'url': 'https://www.skechers.com/clothing/accessories/socks/?prefn1=gender&prefv1=G%7CB'},
{'category': 'Technologies>Snoop Dogg Styles',
'url': 'https://www.skechers.com/technologies/collections/snoop-dogg/'},
{'category': 'Technologies>Martha Stewart Collection',
'url': 'https://www.skechers.com/collaborations/martha-stewart/'},
{'category': 'Technologies>John Deere', 'url': 'https://www.skechers.com/collaborations/john-deere/'},
{'category': 'Technologies>Collaborations', 'url': 'https://www.skechers.com/collaborations/'},
{'category': 'Technologies>Comfort Technologies>Air Cooled Memory Foam',
'url': 'https://www.skechers.com/technologies/comfort-technologies/air-cooled-memory-foam/'},
{'category': 'Technologies>Comfort Technologies>Arch Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/arch-fit/'},
{'category': 'Technologies>Comfort Technologies>Cozy Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/cozy-fit/'},
{'category': 'Technologies>Comfort Technologies>Glide-Step',
'url': 'https://www.skechers.com/technologies/comfort-technologies/glide-step/'},
{'category': 'Technologies>Comfort Technologies>Goga Mat',
'url': 'https://www.skechers.com/technologies/comfort-technologies/goga-mat/'},
{'category': 'Technologies>Comfort Technologies>Hands Free Slip-ins',
'url': 'https://www.skechers.com/technologies/comfort-technologies/slip-ins/'},
{'category': 'Technologies>Comfort Technologies>Hyper Burst',
'url': 'https://www.skechers.com/technologies/comfort-technologies/hyper-burst/'},
{'category': 'Technologies>Comfort Technologies>Luxe Foam',
'url': 'https://www.skechers.com/technologies/comfort-technologies/luxe-foam/'},
{'category': 'Technologies>Comfort Technologies>Massage Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/massage-fit/'},
{'category': 'Technologies>Comfort Technologies>Max Cushioning',
'url': 'https://www.skechers.com/technologies/comfort-technologies/max-cushioning/'},
{'category': 'Technologies>Comfort Technologies>Relaxed Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/relaxed-fit/'},
{'category': 'Technologies>Comfort Technologies>Skech-Air',
'url': 'https://www.skechers.com/technologies/comfort-technologies/skech-air/'},
{'category': 'Technologies>Comfort Technologies>Stretch Fit',
'url': 'https://www.skechers.com/technologies/comfort-technologies/stretch-fit/'},
{'category': 'Technologies>Comfort Technologies>Ultra Go',
'url': 'https://www.skechers.com/technologies/comfort-technologies/ultra-go/'},
{'category': 'Technologies>Features>Breathable Comfort',
'url': 'https://www.skechers.com/technologies/features/breathable-comfort/'},
{'category': 'Technologies>Features>Electrical Hazard',
'url': 'https://www.skechers.com/technologies/features/electrical-hazard/'},
{'category': 'Technologies>Features>Faux Fur Lining',
'url': 'https://www.skechers.com/technologies/features/faux-fur-lining/'},
{'category': 'Technologies>Features>Goodyear Rubber',
'url': 'https://www.skechers.com/technologies/features/goodyear-rubber/'},
{'category': 'Technologies>Features>Lights', 'url': 'https://www.skechers.com/technologies/features/lights/'},
{'category': 'Technologies>Features>Machine Washable',
'url': 'https://www.skechers.com/technologies/features/machine-washable/'},
{'category': 'Technologies>Features>No Tie Fit',
'url': 'https://www.skechers.com/technologies/features/no-tie-fit/'},
{'category': 'Technologies>Features>Recycled',
'url': 'https://www.skechers.com/technologies/features/recycled/'},
{'category': 'Technologies>Features>Safety Toe',
'url': 'https://www.skechers.com/technologies/features/safety-toe/'},
{'category': 'Technologies>Features>Slip Resistant',
'url': 'https://www.skechers.com/technologies/features/slip-resistant/'},
{'category': 'Technologies>Features>Vegan', 'url': 'https://www.skechers.com/technologies/features/vegan/'},
{'category': 'Technologies>Features>Waterproof',
'url': 'https://www.skechers.com/technologies/features/waterproof/'},
{'category': 'Technologies>Features>Water Repellent',
'url': 'https://www.skechers.com/technologies/features/water-repellent/'},
{'category': 'Technologies>Collections>BOBS', 'url': 'https://www.skechers.com/technologies/collections/bobs/'},
{'category': 'Technologies>Collections>Cali', 'url': 'https://www.skechers.com/technologies/collections/cali/'},
{'category': 'Technologies>Collections>Court & Classics',
'url': 'https://www.skechers.com/technologies/collections/court-classics/'},
{'category': "Technologies>Collections>D'Lites",
'url': 'https://www.skechers.com/technologies/collections/dlites/'},
{'category': 'Technologies>Collections>Foamies',
'url': 'https://www.skechers.com/technologies/collections/foamies/'},
{'category': 'Technologies>Collections>GO FOAM',
'url': 'https://www.skechers.com/technologies/collections/go-foam/'},
{'category': 'Technologies>Collections>GO GOLF',
'url': 'https://www.skechers.com/technologies/collections/go-golf/'},
{'category': 'Technologies>Collections>GO RUN',
'url': 'https://www.skechers.com/technologies/collections/go-run/'},
{'category': 'Technologies>Collections>GO WALK',
'url': 'https://www.skechers.com/technologies/collections/go-walk/'},
{'category': 'Technologies>Collections>GO RECOVER',
'url': 'https://www.skechers.com/technologies/collections/go-recover/'},
{'category': 'Technologies>Collections>Mark Nason',
'url': 'https://www.skechers.com/technologies/collections/mark-nason/'},
{'category': 'Technologies>Collections>Our Planet Matters',
'url': 'https://www.skechers.com/technologies/collections/our-planet-matters/'},
{'category': 'Technologies>Collections>Snoop Dogg',
'url': 'https://www.skechers.com/technologies/collections/snoop-dogg/'},
{'category': 'Technologies>Collections>UNOs', 'url': 'https://www.skechers.com/technologies/collections/unos/'},
{'category': 'Technologies>Collections>Work & Safety',
'url': 'https://www.skechers.com/technologies/collections/work-safety/'},
{'category': 'Technologies>Collections>Skechers Basketball',
'url': 'https://www.skechers.com/technologies/collections/skechers-basketball/'},
{'category': 'Clothing>Slip-ins Pants', 'url': 'https://www.skechers.com/clothing/slip-ins-pants/'},
{'category': 'Clothing>GO WALK Pants', 'url': 'https://www.skechers.com/clothing/go-walk-pants/'},
{'category': 'Clothing>Spring/Summer Apparel',
'url': 'https://www.skechers.com/clothing/spring-and-summer-apparel/'},
{'category': 'Clothing>Performance Apparel', 'url': 'https://www.skechers.com/clothing/performance-apparel/'},
{'category': "Clothing>Women's Matching Sets", 'url': 'https://www.skechers.com/clothing/matching-sets/'},
{'category': "Clothing>Women's Extended Sizes",
'url': 'https://www.skechers.com/clothing/womens/?prefn1=size&prefv1=XL%7CXXL%7C3XL&start=0&sz=24'},
{'category': 'Clothing>Clothing Sale', 'url': 'https://www.skechers.com/sale/clothing-accessories/'},
{'category': "Clothing>Women's>Tops", 'url': 'https://www.skechers.com/clothing/womens/tops/'},
{'category': "Clothing>Women's>Bottoms", 'url': 'https://www.skechers.com/clothing/womens/bottoms/'},
{'category': "Clothing>Women's>Accessories", 'url': 'https://www.skechers.com/clothing/womens/accessories/'},
{'category': "Clothing>Men's>Tops", 'url': 'https://www.skechers.com/clothing/mens/tops/'},
{'category': "Clothing>Men's>Bottoms", 'url': 'https://www.skechers.com/clothing/mens/bottoms/'},
{'category': "Clothing>Men's>Accessories", 'url': 'https://www.skechers.com/clothing/mens/accessories/'},
{'category': 'Clothing>Accessories>Socks', 'url': 'https://www.skechers.com/clothing/accessories/socks/'},
{'category': 'Clothing>Accessories>Watches', 'url': 'https://www.skechers.com/clothing/accessories/watches/'},
{'category': 'Clothing>Accessories>Bags', 'url': 'https://www.skechers.com/clothing/accessories/bags/'},
{'category': 'Clothing>Accessories>Hats', 'url': 'https://www.skechers.com/clothing/accessories/hats/'},
{'category': 'Clothing>Accessories>Shoe Care',
'url': 'https://www.skechers.com/clothing/accessories/shoe-care/'},
{'category': 'Clothing>Accessories>Sunglasses',
'url': 'https://www.skechers.com/clothing/accessories/sunglasses/'},
{'category': 'Clothing>Accessories>Sporting Goods',
'url': 'https://www.skechers.com/clothing/accessories/sporting-goods/'},
{'category': 'Sale>Slip-in Sale',
'url': 'https://www.skechers.com/sale/?prefn1=technologyFeatureName&prefv1=Slip-Ins'},
{'category': 'Sale>Women>Shoes',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=W%7CU&prefv2=FOOTWEAR'},
{'category': 'Sale>Women>Clothing',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=W&prefv2=APPAREL'},
{'category': 'Sale>Women>Shop All', 'url': 'https://www.skechers.com/sale/?prefn1=gender&prefv1=W%7CU'},
{'category': 'Sale>Men>Shoes',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=M%7CU&prefv2=FOOTWEAR'},
{'category': 'Sale>Men>Clothing',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=M&prefv2=APPAREL'},
{'category': 'Sale>Men>Accessories',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=M%7CU&prefv2=ACCESSORIES'},
{'category': 'Sale>Men>Shop All', 'url': 'https://www.skechers.com/sale/?prefn1=gender&prefv1=M%7CU'},
{'category': 'Sale>Kids>Girls Shoes',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=G&prefv2=FOOTWEAR'},
{'category': 'Sale>Kids>Boys Shoes',
'url': 'https://www.skechers.com/sale/?prefn1=gender&prefn2=productLine&prefv1=B&prefv2=FOOTWEAR'},
{'category': 'Sale>Kids>Shop All', 'url': 'https://www.skechers.com/sale/?prefn1=gender&prefv1=B%7CG'}]
for item in items_list:
first_queue.put(item)
def main():
"""主函数"""
get_classify()
with ThreadPoolExecutor(max_workers=8) as executor:
detail_futures = [executor.submit(get_detail_url) for _ in range(8)]
for future in detail_futures:
future.result()
with ThreadPoolExecutor(max_workers=12) as executor:
parse_futures = [executor.submit(parse_data) for _ in range(12)]
for future in parse_futures:
future.result()
print("- - - - - - - - - - - - - - Crawling completed - - - - - - - - - - - - - -")
if __name__ == "__main__":
main()