Skip to content

D
doudian-py
Commit 7ac7efa1 by baiquan
Options

添加上传视频代理验证

parent ebe6d9a1
Showing with 420 additions and 17 deletions
requirements.txt
......@@ -28,3 +28,5 @@ retrying~=1.4.0
python-dateutil
python-socks~=2.7.1
PySocks
requests-toolbelt~=1.0.0
\ No newline at end of file
service/create_goods.py
import json
import random
from urllib.parse import urlencode
......@@ -135,18 +136,45 @@ def upload_image_by_bytes(cookies, image_bytes):
return response.json()['data']
def get_prettify_info(cookies, category_id, url_list):
def get_prettify_info(category_id, url_list):
prettify_info = []
for url in url_list:
for i, url in enumerate(url_list):
component_front_data = {
"imgList": [url],
"uploadSource": "local_upload",
'droppedEventTriggered': True,
"image": {
"url": url,
"width": 1080,
"height": 980
},
"$$name$$": f"图片{i+1}"
}
component_front_data = json.dumps(component_front_data)
component_data = {
"url": url
}
component_data = json.dumps(component_data)
prettify_info.append({
'id': 2,
'component_type_id': 2,
'component_front_data': component_front_data,
'component_data': component_data,
'image': {
'url': url,
'width': 1080,
'height': 980,
},
})
json_data = {
'category_id': int(category_id),
'prettify_info': prettify_info
'category_id': category_id,
'prettify_info': prettify_info,
'check_status': 2,
'appid': 1,
}
print(json_data)
response = requests.post(
'https://fxg.jinritemai.com/product/prettify/formatPrettifyForProduct',
cookies=cookies,
......
service/doudian_request.py
......@@ -22,7 +22,7 @@ HEADERS = {
@retry(stop_max_attempt_number=3, wait_fixed=3000)
def doudian_request(method: str, url: str, proxies:dict, params:dict=None, data: str = None, json: dict = None,headers=None, cookies=None, match_str="") -> any:
def doudian_request(method: str, url: str, proxies:dict, params:dict=None, data: any = None, json: dict = None,headers=None, cookies=None, match_str="") -> any:
if headers is None:
headers = HEADERS
logger.info(f'doudian_request-->{url}')
......
service/upload_image_and_video.py 0 → 100644
import asyncio
import hashlib
import json
import os
import requests
from loguru import logger
from requests_toolbelt.multipart.encoder import MultipartEncoder
from config import settings
from service.doudian_request import doudian_request
from service.upload_video import upload_video_with_multithreading, download_video
from utils.common import check_proxy
def get_local_path(item_id, url):
folder_path = os.path.join(settings.BASE_PATH, str(item_id))
if not os.path.exists(folder_path):
raise FileNotFoundError(f"文件夹不存在: {folder_path}")
if "?" in url:
url = url.split("?")[0]
file_path = os.path.join(folder_path, os.path.basename(url))
if not os.path.exists(file_path):
if file_path.endswith(".mp4"):
raise FileNotFoundError(f"文件不存在: {file_path}")
logger.info(f"{file_path} 文件不存在，开始下载")
img_bytes = requests.get(url, stream=True)
with open(file_path, 'wb') as f:
for chunk in img_bytes.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
return file_path
def upload_image_by_bytes(cookies, headers, proxies, image_path_list):
result_dict = {}
# 按每10张分批处理
for batch_start in range(0, len(image_path_list), 10):
batch_end = min(batch_start + 10, len(image_path_list))
batch = image_path_list[batch_start:batch_end]
# 准备多部分表单数据
fields = {}
file_handles = [] # 用于保存打开的文件对象
for idx, img_dict in enumerate(batch):
# 获取原始字典中的键和本地路径
original_key = next(iter(img_dict.keys()))
local_path = img_dict[original_key]
# 打开文件并添加到表单
file_handle = open(local_path, 'rb')
file_handles.append(file_handle)
fields[f'image[{idx}]'] = ('image.jpg',file_handle,'image/jpeg')
# 创建多部分编码器
multipart_data = MultipartEncoder(fields=fields)
headers_batch = headers.copy()
headers_batch['Content-Type'] = multipart_data.content_type
# 执行上传请求
url = 'https://fxg.jinritemai.com/product/img/batchupload?_bid=ffa_goods'
try:
response = doudian_request('POST', url, proxies, data=multipart_data, headers=headers_batch,cookies=cookies)
batch_urls = response['data']
# 收集结果到字典
for idx_in_batch, img_url in enumerate(batch_urls):
list_index = batch_start + idx_in_batch
img_dict = image_path_list[list_index]
original_key = next(iter(img_dict.keys()))
result_dict[original_key] = img_url
except Exception as e:
logger.error(e)
return None
finally:
# 确保关闭所有打开的文件
for fh in file_handles:
fh.close()
# 返回结果字典
logger.info(result_dict)
return result_dict
async def uploadImageAndVideo(task: dict = None):
"""
上传图片和视频
:param task:
:return:
"""
addr = task.get("proxies")["addr"]
port = task.get("proxies")["port"]
username = task.get("proxies")["username"]
password = task.get("proxies")["password"]
proxy_url = f"socks5h://{username}:{password}@{addr}:{port}"
proxies = check_proxy(proxy_url)
cookies = task.get('cookie')
headers = task.get('headers')
item_id = task.get('id')
skus = task.get('skus')
# 准备SKU图片上传
sku_image_list = []
for sku in skus:
for key, value in sku.items():
if isinstance(value, dict):
img_url = value.get('image')
if img_url:
md5_key = hashlib.md5(img_url.encode()).hexdigest()
local_path = get_local_path(item_id, img_url)
sku_image_list.append({md5_key: local_path})
# 准备主图上传
image_list = []
for url in task.get('images', []):
md5_key = hashlib.md5(url.encode()).hexdigest()
local_path = get_local_path(item_id, url)
image_list.append({md5_key: local_path})
# 准备详情图上传
description_list = []
for url in task.get('description', []):
md5_key = hashlib.md5(url.encode()).hexdigest()
local_path = get_local_path(item_id, url)
description_list.append({md5_key: local_path})
try:
# 并行处理所有上传任务
sku_image_dict, image_dict, description_dict, video_dict = await asyncio.gather(
run_in_executor(upload_image_by_bytes, cookies, headers, proxies, sku_image_list),
run_in_executor(upload_image_by_bytes, cookies, headers, proxies, image_list),
run_in_executor(upload_image_by_bytes, cookies, headers, proxies, description_list),
upload_videos(task, item_id)
)
except Exception as e:
logger.error(f"上传过程中发生错误: {str(e)}")
return None
# 构建回调数据结构
callback_data = {
"id": item_id,
"skus": sku_image_dict,
"images": image_dict,
"description": description_dict,
"video_list": video_dict
}
logger.info(json.dumps(callback_data))
# await callback_task(callback_data)
async def run_in_executor(func, *args):
"""在异步环境中运行同步函数"""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, func, *args)
async def upload_videos(task: dict, item_id: str):
"""异步上传所有视频"""
video_dict = {}
video_tasks = []
for video_url in task.get('video_list', []):
logger.info(f"开始处理视频：{video_url}")
local_video_url = get_local_path(item_id, video_url)
video_task = upload_single_video(task.copy(), local_video_url, video_url)
video_tasks.append(video_task)
# 并行执行所有视频上传任务
results = await asyncio.gather(*video_tasks)
# 合并结果
for md5_key, result in results:
if result and 'video_info' in result and 'MainPlayUrl' in result['video_info']:
video_dict[md5_key] = result['video_info']['MainPlayUrl']
return video_dict
async def upload_single_video(task: dict, local_path: any, original_url: str):
"""上传单个视频"""
task['file_path'] = local_path
md5_key = hashlib.md5(original_url.encode()).hexdigest()
try:
result = await run_in_executor(upload_video_with_multithreading, task)
logger.success(f'视频上传成功: {result}')
return md5_key, result
except Exception as e:
logger.error(f'视频上传失败: {original_url}, 错误: {str(e)}')
return md5_key, None
service/upload_video.py
......@@ -684,8 +684,10 @@ def prepare_video_file(task):
download_video(task['video_url'], file_path, headers=task['headers'])
else:
file_path = task.get("file_path")
if not os.path.exists(file_path):
raise Exception(f"视频文件 {file_path} 不存在")
if is_video_corrupted(file_path):
raise Exception("视频文件已损坏")
raise Exception(f"视频文件 {file_path} 已损坏")
video_duration = get_video_duration(file_path)
if video_duration > 60:
logger.error("视频时长大于60秒，上传失败")
......
settings.toml
[development]
HUB_APP_ID = "password"
[production]
DEBUG = true
......@@ -30,3 +31,5 @@ DB_USER = "root"
DB_PASSWORD = "123456"
DB_NAME = "doudian"
DB_CHARSET = "utf8mb4"
BASE_PATH = "D://"
\ No newline at end of file
shop.py
......@@ -8,6 +8,7 @@ from loguru import logger
from service.hub_ import closeBrowser, envList, exportCookie
from service.page_login import page_login
from service.upload_image_and_video import uploadImageAndVideo
HUB_DOMAIN = "http://127.0.0.1:6873"
DOMAIN = "http://159.75.92.198:8809"
......@@ -146,6 +147,16 @@ async def syncShop(task: dict = None):
await closeBrowser(browser_default_id)
return
def run_upload_image_and_video(task):
"""在新的线程中运行 uploadImageAndVideo 的包装函数"""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(uploadImageAndVideo(task))
except Exception as e:
logger.error(f"上传图片和视频任务执行失败: {e}")
finally:
loop.close()
def run_sync_shop(task):
"""在新的线程中运行 syncShop 的包装函数"""
......@@ -171,6 +182,16 @@ async def handle_task(task):
logger.error(f"同步店铺任务超时: {task}")
except Exception as e:
logger.error(f"同步店铺任务异常: {e}")
elif type_ == 2:
# 上传图片和视频任务 - 使用线程池执行
with concurrent.futures.ThreadPoolExecutor(max_workers=INNER_MAX_WORKERS) as inner_executor:
future = inner_executor.submit(run_upload_image_and_video, task)
try:
future.result(timeout=300) # 设置5分钟超时
except concurrent.futures.TimeoutError:
logger.error(f"上传图片和视频任务超时: {task}")
except Exception as e:
logger.error(f"上传图片和视频任务异常: {e}")
else:
logger.warning(f"未知任务类型: {type_}, 跳过处理")
......@@ -180,17 +201,183 @@ async def run():
'number': 1
}
try:
tasks = await get_task(upload_data)
# tasks = {"data": [{"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1075653218"},]}
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1075653218"},
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1075653218"},
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1075653218"},
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1075653218"},]}
# tasks = await get_task(upload_data)
# tasks = {"data": [{"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1111095672"},]}
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1111095694"},
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1180779018"},
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1180779004"},
# {"app_name": "admin", "type": 1, "browser_type": 1, "browser_id": "1180779061"},]}
tasks = {
"id": "808160465909",
"headers": {
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Connection': 'keep-alive',
'Origin': 'https://fxg.jinritemai.com',
'Referer': 'https://fxg.jinritemai.com/',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-site',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0',
'X-Requested-With': 'XMLHttpRequest',
'sec-ch-ua': '"Not(A:Brand";v="99", "Microsoft Edge";v="133", "Chromium";v="133"',
},
"cookie": {
"PHPSESSID": "cd870e80d2c587d7cc9c457e8f7efb4f",
"PHPSESSID_SS": "cd870e80d2c587d7cc9c457e8f7efb4f"
},
"proxies": {
"addr": "58.251.251.235",
"port": "9001",
"username": "WeGhswcQujYZ33tQ",
"password": "jQCvhZdckPu4VjxG"
},
"skus": [
{
"颜色分类": {
"title": "粉色-铃铛立体猫咪",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d22eb8e3.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "3.12",
"end_price": "3.12",
"skuid": "5743344376309"
},
{
"颜色分类": {
"title": "黄色-铃铛立体猫咪",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d2323dfb.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "3.12",
"end_price": "3.12",
"skuid": "5743344376311"
},
{
"颜色分类": {
"title": "蓝色-铃铛立体猫咪",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d2350619.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "3.12",
"end_price": "3.12",
"skuid": "5743344376307"
},
{
"颜色分类": {
"title": "米色-铃铛立体猫咪",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d237c62c.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "3.12",
"end_price": "3.12",
"skuid": "5743344376312"
},
{
"颜色分类": {
"title": "白色-铃铛立体猫咪",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d23a93a0.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "3.12",
"end_price": "3.12",
"skuid": "5743344376308"
},
{
"颜色分类": {
"title": "黑色-铃铛立体猫咪",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d23d62d5.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "3.12",
"end_price": "3.12",
"skuid": "5743344376310"
},
{
"颜色分类": {
"title": "3个铃铛立体猫咪（可备注）",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d24267e2.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "6.62",
"end_price": "6.62",
"skuid": "5932955163526"
},
{
"颜色分类": {
"title": "全家福6个",
"kinds": "颜色分类",
"image": "http://159.75.92.198:8808/uploads/20250619/893838919796/68537d2453d59.jpg"
},
"quantityErrorMsg": "无",
"quantityText": "有货(限购100件)",
"quantity": "200",
"start_price": "10.05",
"end_price": "10.05",
"skuid": "5932955163525"
}
],
"images": [
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d22053f3.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d222eaa3.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d225a093.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d2285d3d.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d22bf1dc.jpg"
],
"description": [
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d249c380.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d24af867.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d24ebd93.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d25323d7.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d256f116.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d25a9c93.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d25e66c2.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d262d20d.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d267bb74.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d26b7b8f.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d26f0e8e.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d273833c.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d2768681.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d27d28ee.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d280d893.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d28517e4.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d288b03a.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d28c69cf.jpg",
"http://159.75.92.198:8808/uploads/20250619/893838919796/68537d29231bf.jpg"
],
"video_list": [
"https://cloud.video.taobao.com/play/u/3708306281/p/2/e/6/t/1/469092503042.mp4?appKey=38829"
],
"type": 2
}
except:
logger.error('获取任务失败')
return
logger.info(json.dumps(tasks))
tasks = tasks.get('data', {})
tasks = tasks.get('data', tasks)
if type(tasks) == dict:
tasks = [tasks]
if not tasks:
logger.error('没有任务')
return
......@@ -199,6 +386,7 @@ async def run():
with concurrent.futures.ThreadPoolExecutor(max_workers=OUTER_MAX_WORKERS) as outer_executor:
# 为每个任务创建处理线程
futures = []
for task in tasks:
# 提交任务处理到线程池
future = outer_executor.submit(
......@@ -223,14 +411,14 @@ if __name__ == '__main__':
argv = sys.argv
if len(argv) != 2:
logger.error("请传入参数")
sys.exit(0)
# sys.exit(0)
else:
CODE = argv[1]
DEFAULT_HEADER = {
"Content-Type": "application/json",
"Accept": "application/json",
"Authorization-Code": CODE,
"Authorization-Code": 'CODE',
}
while True:
# while True:
asyncio.run(run())
time.sleep(10)
# time.sleep(10)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment