Skip to content

D
doudian-py
Commit d2108791 by baiquan
Options

init

parent 21a0795d
Showing with 38 additions and 21 deletions
api.py
import execjs
from loguru import logger
from celery.result import AsyncResult
from fastapi import FastAPI, APIRouter
from fastapi import status
from pydantic import BaseModel
from celery_app import celery_app
from errors import *
import os
# 创建 logs 目录（如果不存在）
os.makedirs("logs", exist_ok=True)
# 添加日志文件输出，按7天滚动
logger.add("logs/api.log", rotation="7 day", level="INFO", encoding="utf-8", backtrace=True, diagnose=True)
app = FastAPI(title="doudian数据同步平台")
sync_router = APIRouter(prefix="/sync", tags=["同步接口"])
......@@ -19,7 +22,8 @@ class SyncShopRequest(BaseModel):
class SyncShopInfoRequest(BaseModel):
browser_id: str
listen_data: dict
listen_url: str
open_url: str
class CreateTemplateRequest(BaseModel):
cookies: dict
......
create_goods.py
......@@ -83,13 +83,15 @@ def search_category(cookies, key):
print(response.text)
return response.json()['data']
def get_a_bogus(a_bogus_js_path, params,data, userAgent):
def get_a_bogus(a_bogus_js_path, params, data, userAgent):
with open(a_bogus_js_path, 'r', encoding='gbk') as f:
a_bogus_js = f.read()
a_bogus = execjs.compile(a_bogus_js).call('get_ab', params,data, userAgent)
a_bogus = execjs.compile(a_bogus_js).call('get_ab', params, data, userAgent)
# print(f'a_bogus: {a_bogus}')
return a_bogus
def get_schema(cookies, category_id):
params = {
'appid': '1',
......@@ -107,9 +109,9 @@ def get_schema(cookies, category_id):
'_bid': 'ffa_goods',
}
url = 'https://fxg.jinritemai.com/product/tproduct/getSchema'
a_bogus = get_a_bogus('./js/a_bogus.js', urlencode(params), json_data, headers['User-Agent'])
a_bogus = get_a_bogus('a_bogus.js', urlencode(params), json_data, headers['User-Agent'])
params['a_bogus'] = a_bogus
response = requests.post(url,params=params,cookies=cookies,headers=headers,json=json_data,impersonate="chrome131")
response = requests.post(url, params=params, cookies=cookies, headers=headers, json=json_data, impersonate="chrome131")
print(response.text)
return response.json()['data']
......@@ -128,8 +130,8 @@ def upload_image_by_bytes(cookies, image_bytes):
url = 'https://fxg.jinritemai.com/product/img/batchupload?_bid=ffa_goods'
mp = CurlMime()
for i, byte_data in enumerate(image_bytes):
mp.addpart(name=f"image[{i}]", content_type="image/jpeg",filename=f"image_{i}.jpg", data=byte_data)
response = requests.post(url,cookies=cookies,headers=headers,data={"foo":"bar"},multipart=mp,impersonate="chrome131")
mp.addpart(name=f"image[{i}]", content_type="image/jpeg", filename=f"image_{i}.jpg", data=byte_data)
response = requests.post(url, cookies=cookies, headers=headers, data={"foo": "bar"}, multipart=mp, impersonate="chrome131")
print(response.text)
mp.close()
return response.json()['data']
......@@ -156,6 +158,7 @@ def get_prettify_info(cookies, category_id, url_list):
print(response.text)
return response.json()['data']
def generate_id(length=18):
"""生成18位随机数字ID"""
return str(random.randint(10 ** (length - 1), 10 ** length - 1))
......@@ -199,7 +202,7 @@ def build_spec_structure(spec_params):
{'id': value_id, 'name': value,
'image_url': spec_params.get('specs_images')['specs_images_values'][value]})
else:
spec_values.append({'id': value_id, 'name': value })
spec_values.append({'id': value_id, 'name': value})
value_mapping[value] = value_id
spec_mapping[spec_name] = value_mapping
result['spec_detail']['value'].append({
......@@ -228,6 +231,7 @@ def build_spec_structure(spec_params):
return result
def search_brand_prefix(cookies, category_id, brand_name):
params = {
'cid': str(category_id),
......@@ -235,13 +239,14 @@ def search_brand_prefix(cookies, category_id, brand_name):
'appid': '1',
}
url = 'https://fxg.jinritemai.com/product/tproduct/brandPrefix'
response = requests.get(url,params=params,cookies=cookies,headers=headers)
response = requests.get(url, params=params, cookies=cookies, headers=headers)
search_result = response.json()['data']
print(search_result)
if not search_result:
return None
return search_result[0]
def add_with_schema(cookies, schema, goods_params):
if goods_params.get('detail_prettify_image', None):
prettify_info = get_prettify_info(cookies, schema['context']['category_id'], goods_params['detail_prettify_image'])
......@@ -796,7 +801,6 @@ def add_with_schema(cookies, schema, goods_params):
print(response.text)
if __name__ == '__main__':
account = 'dybaihuo58@163.com'
password = 'Cc123123'
......@@ -806,7 +810,6 @@ if __name__ == '__main__':
'PHPSESSID_SS': '232b5651dcd01b596b830d540362643d',
}
pic_image_path_list = ['image1.jpg', 'image2.jpg', 'image3.jpg', 'image4.jpg', 'image5.jpg']
image_bytes = []
for image_path in pic_image_path_list:
......
hub_.py
......@@ -31,6 +31,8 @@ async def sendRequest(method: HttpMethod, url: str, json_data: dict = None, **kw
# 判断是否有code字段
if 'code' in response:
if response['code'] != 0:
logger.error(f'hub接口异常返回值:{response}')
logger.error(json_data)
raise HubAPIError(msg=response['msg'])
if response['code'] != 0:
......@@ -102,8 +104,11 @@ async def closeBrowser(browser_id: str = ""):
if browser_status_data:
containers = browser_status_data.get('containers', {})
if containers:
try:
if (containers[0]['status'] == 1 or containers[0]['status'] == 0) and containers[0]['containerCode'] == browser_id:
return await sendRequest("POST", url=url, json_data=data)
except Exception as e:
logger.error(f"关闭浏览器异常-->{e}")
async def checkBrowserStatus(browser_id: str = ""):
"""
......
js/a_bogus.js
......@@ -11193,8 +11193,6 @@ window.bdms || function () {
function get_ab(params,data,ua) {
if (typeof data == "object")
data = JSON.stringify(data)
return window.bdms.init._v[2].p[48].apply(null, [0, 1, 14, params, data, ua])
}
......
main.py
......@@ -3,8 +3,8 @@ from asyncio.log import logger
from DrissionPage import Chromium
from DrissionPage.common import Settings
from common import setup_proxy
from config import settings
from errors import *
from hub_ import *
......@@ -30,6 +30,7 @@ HEADERS = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0',
}
async def syncShop(container_name: str):
"""
同步店铺
......@@ -134,6 +135,7 @@ async def syncShopInfo(parameter: dict = None):
except:
raise NotFoundError('获取数据失败')
def getRegionNodeList():
params = {
'level': '4',
......@@ -142,8 +144,10 @@ def getRegionNodeList():
response = requests.get('https://ecomuser.snssdk.com/area/getdivision', params=params, headers=HEADERS)
return response.json()['data']['region_node_list']
REGION_NODE_LIST = getRegionNodeList()
def getProductName(product_type, product_name):
if product_type == 'province':
for region_node in REGION_NODE_LIST:
......@@ -159,6 +163,7 @@ def getProductName(product_type, product_name):
else:
raise NotFoundError("未找到城市")
async def createTemplate(cookies, template_params, proxies):
addr = proxies['addr']
port = proxies['port']
......@@ -564,7 +569,7 @@ async def createTemplate(cookies, template_params, proxies):
}
logger.info(rule_list)
url = 'https://fxg.jinritemai.com/freight/template/createTemplate'
response = session.post(url,params=params,cookies=cookies,headers=HEADERS,json=json_data).json()
response = session.post(url, params=params, cookies=cookies, headers=HEADERS, json=json_data).json()
logger.info(response)
if response['code'] != 0:
msg = response.get('msg', '')
......
task_worker.py
......@@ -40,12 +40,13 @@ def execute_sync_shop(container_name: str):
}
@shared_task(name='sync_shop_info')
def execute_sync_shop_info(browser_id: str, listen_data: list):
def execute_sync_shop_info(browser_id: str, listen_url: str, open_url: str):
"""执行店铺信息同步任务"""
try:
parameter = {
'browser_id': browser_id,
'listen_data': listen_data
'listen_url': listen_url,
'open_url': open_url,
}
result = asyncio.run(syncShopInfo(parameter))
return {
......@@ -168,6 +169,7 @@ def execute_generate_a_bogus(params:str | dict,data: dict,ua: str):
ctx = execjs.compile(js_code)
if type(params) == dict:
params = urlencode(params)
data = json.dumps(data)
result = ctx.call('get_ab', params, data, ua)
return {
'code': 200,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment