Skip to content

W
wfs_export
Commit 8e606b5f by yexing
Options

更新

parent 3db14115
Showing with 618 additions and 142 deletions
.gitignore
log
.vscode
old
fu_data
nuitka-crash-report.xml 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
out/walmart_demo.dist/certifi/cacert.pem 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
out/walmart_demo.dist/fake_useragent/data/browsers.jsonl 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
out/walmart_sup.dist/certifi/cacert.pem 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
plugin/.gitignore
node_modules
pnpm-lock.yaml
test
tmp
plugin/.version
2.5.1
\ No newline at end of file
3.0.8
\ No newline at end of file
plugin/dist.crx
No preview for this file type
plugin/dist/js/background.js
......@@ -19445,9 +19445,35 @@
var jszip_minExports = requireJszip_min();
var JSZip = /*@__PURE__*/getDefaultExportFromCjs(jszip_minExports);
function sleep(ms) {
ms += 1000 * Math.random(); // 1s+ delay
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function retry(asyncFunc, options = {}, args = []) {
let { retries = 3, factor = 1, delayMs = 2000, isThrow = true } = options;
let attempt = 0;
while (attempt < retries) {
try {
return await asyncFunc(...args);
} catch (e) {
attempt++;
await sleep(delayMs);
delayMs *= factor;
console.log(`${delayMs}ms 后重试, ${attempt}/${retries}, ${e.message}`);
if (attempt >= retries && isThrow) throw e;
}
}
return null;
}
const checkResponse = async (resp) => {
if (resp.status !== 200) {
const text = await resp.text();
throw new Error(`异常响应: ${text}`);
}
};
async function createZip(blobs, fileNames) {
if (blobs.length !== fileNames.length) {
throw new Error("数组长度不一致");
if (blobs.length !== fileNames.length || blobs.length === 0) {
throw new Error("数组长度不一致或为空");
}
const zip = new JSZip();
blobs.forEach((blob, idx) => {
......@@ -19479,9 +19505,10 @@
month = month == 0 ? 11 : month - 1;
year = month == 0 ? year - 1 : year;
}
month++;
const endDate = new Date(year, month, 0).getDate() + '';
x2 = x2 === 0 ? endDate : x2;
month = (month + 1 + '').padStart(2, '0');
month = (month + '').padStart(2, '0');
value.period = [
`${year}-${month}-${x1}`,
`${year}-${month}-${x2}`
......@@ -19523,19 +19550,18 @@
}
async function upload(url, body, headers) {
console.log(`上传 ${url}: ${body}`);
console.log(`上传 ${url} ${body}`);
let response = await fetch(url, {
method: "POST", body, headers
});
console.log(response);
try {
let data = await response.json();
if (data.status.toUpperCase() === "OK") {
console.log(data);
if (data.status === "OK") return true;
} catch (e) {
console.error(e.message);
return true;
} else {
throw new Error(`上传失败, ${url}, ${JSON.stringify(data)}`);
}
return false;
}
async function checkLog(fn, insert = false) {
const url = "https://walmart.meinuosha.com/index.php/index/index/Getspluginexporttablerecords?accessvalue=Walmart2025PY0307";
......@@ -19559,49 +19585,50 @@
const linkURL = "https://walmart.meinuosha.com/index.php/index/index/url_download_list?accessvalue=Walmart2025PY0307";
const url = options.isJson ? jsonURL : options.isLink ? linkURL : fileURL;
if (await checkLog(fn)) {
if (await upload(url, data, options.headers)) {
await checkLog(fn, true);
}
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5 }
);
if (insert) await checkLog(fn, true);
}
}
const TABLE = {
WFS: {
name: "WFS",
uri: "https://seller.walmart.com/aurora/v1/wfs/reports/",
params: [
const p1 = [
["11 22:", ["01", "10"]],
["21 22:", ['11', '20']],
["01 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
];
const p2 = [
["11 22:", ["01", "10"]],
["21 22:", ['11', '20']],
["03 22:", ['01', 0]]
];
const TABLE = {
WFS: {
name: "WFS",
uri: "https://seller.walmart.com/aurora/v1/wfs/reports/",
params: p1
},
PAYMENT: {
name: "放款表",
uri: "https://seller.walmart.com/aurora/v1/",
params: [
["05 22:", ['01', 0]]
]
params: p2
},
GQL: {
name: "营销表",
uri: "https://seller.walmart.com/aurora/v2/marketing-self-serve/gql",
params: p1
},
ORDER: {
name: "订单表",
uri: "https://seller.walmart.com/aurora/v1/reports/",
params: [
["12 22:", ["01", "10"]],
["22 22:", ['11', '20']],
["02 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
params: p1
},
ADVERT: {
name: "广告表",
uri: "https://advertising.walmart.com/sp/api/",
params: [
["12 22:", ["01", "10"]],
["22 22:", ['11', '20']],
["02 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
params: p1
}
};
......@@ -19614,7 +19641,8 @@
chrome.tabs.query({}, (tabs) => {
const tab = tabs.filter(x => checkOrigin(x.url, uri))[0];
if (!tab) {
console.error(`tab not found, by ${JSON.stringify(msg)}`);
const errorMsg = `tab not found, msg: ${JSON.stringify(msg)}`;
callback ? callback(errorMsg) : console.error(errorMsg);
return;
}
console.log(`send ${JSON.stringify(msg)} to ${tab.url}`);
......@@ -19647,6 +19675,7 @@
console.log(`GET ${url}`);
let response = await fetch(url);
console.log(response);
await checkResponse(response);
let blob = await response.blob();
const csv = xlsx2csv(await blob.arrayBuffer(), { type: "array" });
if (!csv.includes("No data found")) {
......@@ -19670,7 +19699,20 @@
sendResponse();
} else if (msg.type === "addLog") {
logs.push(msg.data);
if (msg.push) {
chrome.runtime.sendMessage({ type: 'progress', logs });
}
sendResponse();
} else if (msg.type === "executeScript") {
const results = await chrome.scripting.executeScript({
target: { tabId: sender.tab.id },
world: "MAIN",
func: (wAttr) => {
return window[wAttr];
},
args: [msg.data]
});
sendResponse(results[0].result);
} else if (msg.type === "sopLink") {
try {
await sopLink(msg.data);
......@@ -19688,7 +19730,6 @@
taskNum++;
if (taskInterval) return;
chrome.storage.local.set({ isRunning: true });
// chrome.storage.local.get(null, console.log);
taskInterval = setInterval(() => {
// 推送更新
const ackLen = logs.filter(x => x.message.includes("完成")).length;
......@@ -19723,6 +19764,7 @@
});
// 初始化
// initTimedTask();
// chrome.storage.local.get(null, console.log);
// chrome.storage.local.clear();
// chrome.alarms.getAll(console.log);
chrome.alarms.clearAll();
......
plugin/dist/js/popup.js
(function () {
'use strict';
const TABLE = {
WFS: {
name: "WFS",
uri: "https://seller.walmart.com/aurora/v1/wfs/reports/",
params: [
const p1 = [
["11 22:", ["01", "10"]],
["21 22:", ['11', '20']],
["01 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
];
const p2 = [
["11 22:", ["01", "10"]],
["21 22:", ['11', '20']],
["03 22:", ['01', 0]]
];
const TABLE = {
WFS: {
name: "WFS",
uri: "https://seller.walmart.com/aurora/v1/wfs/reports/",
params: p1
},
PAYMENT: {
name: "放款表",
uri: "https://seller.walmart.com/aurora/v1/",
params: [
["05 22:", ['01', 0]]
]
params: p2
},
GQL: {
name: "营销表",
uri: "https://seller.walmart.com/aurora/v2/marketing-self-serve/gql",
params: p1
},
ORDER: {
name: "订单表",
uri: "https://seller.walmart.com/aurora/v1/reports/",
params: [
["12 22:", ["01", "10"]],
["22 22:", ['11', '20']],
["02 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
params: p1
},
ADVERT: {
name: "广告表",
uri: "https://advertising.walmart.com/sp/api/",
params: [
["12 22:", ["01", "10"]],
["22 22:", ['11', '20']],
["02 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
params: p1
}
};
......@@ -99,9 +98,10 @@
month = month == 0 ? 11 : month - 1;
year = month == 0 ? year - 1 : year;
}
month++;
const endDate = new Date(year, month, 0).getDate() + '';
x2 = x2 === 0 ? endDate : x2;
month = (month + 1 + '').padStart(2, '0');
month = (month + '').padStart(2, '0');
value.period = [
`${year}-${month}-${x1}`,
`${year}-${month}-${x2}`
......@@ -141,6 +141,13 @@
this.params.push(arguments);
}
}
const addLog = (message, push) => {
const logEntry = {
time: new Date().toLocaleTimeString(),
message
};
chrome.runtime.sendMessage({ type: "addLog", data: logEntry, push });
};
var dayjs_min$1 = {exports: {}};
......@@ -196,13 +203,62 @@
var timezoneExports = requireTimezone();
var timezone = /*@__PURE__*/getDefaultExportFromCjs(timezoneExports);
var isSameOrAfter$2 = {exports: {}};
var isSameOrAfter$1 = isSameOrAfter$2.exports;
var hasRequiredIsSameOrAfter;
function requireIsSameOrAfter () {
if (hasRequiredIsSameOrAfter) return isSameOrAfter$2.exports;
hasRequiredIsSameOrAfter = 1;
(function (module, exports) {
!function(e,t){module.exports=t();}(isSameOrAfter$1,(function(){return function(e,t){t.prototype.isSameOrAfter=function(e,t){return this.isSame(e,t)||this.isAfter(e,t)};}}));
} (isSameOrAfter$2));
return isSameOrAfter$2.exports;
}
var isSameOrAfterExports = requireIsSameOrAfter();
var isSameOrAfter = /*@__PURE__*/getDefaultExportFromCjs(isSameOrAfterExports);
var isSameOrBefore$2 = {exports: {}};
var isSameOrBefore$1 = isSameOrBefore$2.exports;
var hasRequiredIsSameOrBefore;
function requireIsSameOrBefore () {
if (hasRequiredIsSameOrBefore) return isSameOrBefore$2.exports;
hasRequiredIsSameOrBefore = 1;
(function (module, exports) {
!function(e,i){module.exports=i();}(isSameOrBefore$1,(function(){return function(e,i){i.prototype.isSameOrBefore=function(e,i){return this.isSame(e,i)||this.isBefore(e,i)};}}));
} (isSameOrBefore$2));
return isSameOrBefore$2.exports;
}
var isSameOrBeforeExports = requireIsSameOrBefore();
var isSameOrBefore = /*@__PURE__*/getDefaultExportFromCjs(isSameOrBeforeExports);
dayjs.extend(utc);
dayjs.extend(timezone);
dayjs.extend(isSameOrAfter);
dayjs.extend(isSameOrBefore);
function getTimeDiff(input, offset = 0) {
const time = dayjs.tz(input, "YYYY-MM-DD", "America/Los_Angeles");
time.add(offset, "day");
const CST = "Asia/Shanghai";
const PT = "America/Los_Angeles";
const checkTime = (time) => {
if (!time.isValid()) throw new Error("无效时间");
};
function getLocalTime(dt) {
const time = dayjs.tz(new Date(), CST);
checkTime(time);
return time.toDate();
}
function getTimeDiff(dt, offset = 0) {
const time = dayjs
.tz(dt ?? new Date(), PT)
.add(offset, "day");
checkTime(time);
return (dayjs().valueOf() - time.valueOf()) / 1000 / 60 / 60; // h
}
......@@ -224,20 +280,24 @@
const boxes = document.querySelectorAll("input[name='option']:checked");
boxes.forEach((box) => {
const { uri, params } = TABLE[box.value.toUpperCase()];
const { name, uri, params } = TABLE[box.value.toUpperCase()];
// const row = box.closest(".form-row");
// const fromInput = row.querySelector("input[name='fromDate']");
// const toInput = row.querySelector("input[name='toDate']");
// const period = [fromInput.value, toInput.value];
// const period = ['2025-01-01', '2025-02-01'];
const period = new Plan(null, params).get()?.period;
const diff = getTimeDiff(period[1], 1);
let select = true;
if (diff < 0 && diff > -12) {
select = confirm("时间未到, 是否确认继续");
const period = new Plan(getLocalTime(), params).get()?.period;
if (period === undefined) {
addLog(`${name} 未到指定时间`, true);
return;
}
// Deprecated
const diff = getTimeDiff(period[1], 1);
if (-12 < diff && diff < 0) {
const select = confirm("时间未到, 是否确认继续");
if (!select) return;
}
chrome.storage.local.get(['isRunning'], ({ isRunning }) => {
if (!isRunning) {
chrome.runtime.sendMessage({ type: "run", data: { period, uri } });
......@@ -279,7 +339,6 @@
renderLogs(msg.logs);
}
});
// 到点触发
btn.addEventListener("click", onclick);
};
......
plugin/dist/manifest.json
{"name":"WalmartExports","description":"导出沃尔玛报表","version":"2.5.1","manifest_version":3,"background":{"service_worker":"./js/background.js"},"permissions":["tabs","activeTab","scripting","notifications","storage","alarms"],"action":{"default_title":"WalmartExports","default_popup":"popup.html","default_icon":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"}},"icons":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"},"web_accessible_resources":[{"resources":["src/main.js"],"matches":["<all_urls>"]}],"host_permissions":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"content_scripts":[{"matches":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"js":["./js/main.js"]}]}
\ No newline at end of file
{"name":"WalmartExports","description":"导出沃尔玛报表","version":"3.0.8","manifest_version":3,"background":{"service_worker":"./js/background.js"},"permissions":["tabs","activeTab","scripting","notifications","storage","alarms"],"action":{"default_title":"WalmartExports","default_popup":"popup.html","default_icon":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"}},"icons":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"},"web_accessible_resources":[{"resources":["src/main.js"],"matches":["<all_urls>"]}],"host_permissions":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"content_scripts":[{"matches":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"js":["./js/main.js"]}]}
\ No newline at end of file
plugin/dist/popup.html
......@@ -19,6 +19,10 @@
<label><input type="checkbox" value="Payment" name="option" checked /> 放款表</label>
<!-- <input type="text" name="fromDate">-<input type="text" name="toDate"> -->
</div>
<div class="form-row">
<label><input type="checkbox" value="GQL" name="option" checked /> 营销表</label>
<!-- <input type="text" name="fromDate">-<input type="text" name="toDate"> -->
</div>
<!-- <div class="form-row">
<label><input type="checkbox" value="Order" name="option" checked /> 订单表</label>
<input type="text" name="fromDate">-<input type="text" name="toDate">
......
plugin/eslint.config.js
......@@ -4,6 +4,19 @@ import pluginJs from "@eslint/js";
/** @type {import('eslint').Linter.Config[]} */
export default [
{languageOptions: { globals: {...globals.browser, ...globals.node, chrome: "readonly"} }},
{
languageOptions: { globals: { ...globals.browser, ...globals.node, chrome: "readonly" } },
rules: {
"no-unused-vars": [
"error",
{
argsIgnorePattern: "^_", // 忽略函数参数
varsIgnorePattern: "^_", // 忽略普通变量
destructuredArrayIgnorePattern: "^_", // 忽略解构数组中的变量
caughtErrorsIgnorePattern: "^_" // 忽略 catch 错误变量
}
]
}
},
pluginJs.configs.recommended,
];
\ No newline at end of file
plugin/src/background.js
import { xlsx2csv } from "./xlsx.js";
import { Plan, createZip } from "./util.js";
import { Plan, createZip, checkResponse } from "./util.js";
import { uploadFile } from "./upload.js";
import { TABLE } from "./conf.js";
......@@ -12,7 +12,8 @@ const sendTabMsg = (msg, callback) => {
chrome.tabs.query({}, (tabs) => {
const tab = tabs.filter(x => checkOrigin(x.url, uri))[0];
if (!tab) {
console.error(`tab not found, by ${JSON.stringify(msg)}`);
const errorMsg = `tab not found, msg: ${JSON.stringify(msg)}`
callback ? callback(errorMsg) : console.error(errorMsg);
return;
}
console.log(`send ${JSON.stringify(msg)} to ${tab.url}`);
......@@ -45,6 +46,7 @@ async function sopLink(data) {
console.log(`GET ${url}`);
let response = await fetch(url);
console.log(response);
await checkResponse(response);
let blob = await response.blob();
const csv = xlsx2csv(await blob.arrayBuffer(), { type: "array" });
if (!csv.includes("No data found")) {
......@@ -68,7 +70,20 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
sendResponse();
} else if (msg.type === "addLog") {
logs.push(msg.data);
if (msg.push) {
chrome.runtime.sendMessage({ type: 'progress', logs });
}
sendResponse();
} else if (msg.type === "executeScript") {
const results = await chrome.scripting.executeScript({
target: { tabId: sender.tab.id },
world: "MAIN",
func: (wAttr) => {
return window[wAttr];
},
args: [msg.data]
});
sendResponse(results[0].result);
} else if (msg.type === "sopLink") {
try {
await sopLink(msg.data);
......
plugin/src/conf.js
export const TABLE = {
WFS: {
name: "WFS",
uri: "https://seller.walmart.com/aurora/v1/wfs/reports/",
params: [
const p1 = [
["11 22:", ["01", "10"]],
["21 22:", ['11', '20']],
["01 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
];
const p2 = [
["11 22:", ["01", "10"]],
["21 22:", ['11', '20']],
["03 22:", ['01', 0]]
];
export const TABLE = {
WFS: {
name: "WFS",
uri: "https://seller.walmart.com/aurora/v1/wfs/reports/",
params: p1
},
PAYMENT: {
name: "放款表",
uri: "https://seller.walmart.com/aurora/v1/",
params: [
["05 22:", ['01', 0]]
]
params: p2
},
GQL: {
name: "营销表",
uri: "https://seller.walmart.com/aurora/v2/marketing-self-serve/gql",
params: p1
},
ORDER: {
name: "订单表",
uri: "https://seller.walmart.com/aurora/v1/reports/",
params: [
["12 22:", ["01", "10"]],
["22 22:", ['11', '20']],
["02 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
params: p1
},
ADVERT: {
name: "广告表",
uri: "https://advertising.walmart.com/sp/api/",
params: [
["12 22:", ["01", "10"]],
["22 22:", ['11', '20']],
["02 22:", ['21', 0]],
["05 22:", ['01', 0]]
]
params: p1
}
}
plugin/src/header.js
......@@ -17,10 +17,6 @@ export const rrHeaders = () => {
}));
return rr;
}
export const rHeaders = Object.freeze({
"accept": "application/json",
"wm_aurora.market": "US",
});
export const headers = Object.freeze({
"wm_aurora.market": "US",
});
plugin/src/main.js
import { sleep, xpath } from "./util.js";
import { sleep, xpath, createZip, fmt0, addLog } from "./util.js";
import { createTasks } from "./base.js";
import { Task } from "./task.js";
import { uploadFile } from "./upload.js";
import { TABLE } from "./conf.js";
......@@ -14,18 +15,14 @@ async function webJump() {
).click();
}
};
const addLog = (message) => {
const logEntry = {
time: new Date().toLocaleTimeString(),
message
};
chrome.runtime.sendMessage({ type: "addLog", data: logEntry });
}
async function run(options) {
const name = Object.values(TABLE).find(x => x.uri === options.uri).name;
let tasks = await createTasks(options.uri, options.period);
tasks = options.sn ? [tasks.at(options.sn)] : tasks;
async function run(options = {}) {
const { uri, period, sn } = options;
const [fromDate, toDate] = period || [fmt0(new Date(), -10), fmt0(new Date(), -1)];
const [key, { name }] = Object.entries(TABLE).find((item) => item[1].uri === uri);
let tasks = await createTasks(uri, period);
tasks = sn ? [tasks.at(sn)] : tasks;
let moment = 3000, idx = 0, len = tasks.length;
const allData = [], allFn = [], zipFn = `${Task.partnerId} #${key}# ${fromDate}_${toDate}.zip`;
for (const task of tasks) {
const pf = `${++idx}/${len}`;
addLog(`${name} 运行中 ${pf}`);
......@@ -39,19 +36,24 @@ async function run(options) {
const isJson = fn.endsWith('.json');
if (isJson) {
const headers = { 'Content-Type': 'application/json' }
uploadFile(fn, data, { isJson, headers });
await uploadFile(fn, data, { isJson, headers });
// const blob = new Blob([data], { "type": "application/json" });
// downloadFile(fn, blob);
} else {
fn = fn.replace(/\.\w+$/g, ".zip");
const form = new FormData();
form.append("zipfile", data, fn);
uploadFile(fn, form);
allData.push(data);
allFn.push(fn);
// downloadFile(fn, data);
}
console.log(`${pf} end: ${fn}`);
await sleep(moment);
}
if (allData.length > 0) {
const zip = await createZip(allData, allFn);
const form = new FormData();
form.append("zipfile", zip, zipFn);
await uploadFile(zipFn, form);
}
addLog(`${name} 运行完成`);
}
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
......
plugin/src/popup.js
import { TABLE } from "./conf.js";
import { Plan } from "./util.js";
import { getTimeDiff } from "./timezone.js";
import { Plan, addLog } from "./util.js";
import { getTimeDiff, getLocalTime } from "./timezone.js";
function showModal() {
document.querySelector('.modal').style.display = 'flex';
......@@ -20,20 +20,24 @@ function onclick(e) {
const boxes = document.querySelectorAll("input[name='option']:checked");
boxes.forEach((box) => {
const { uri, params } = TABLE[box.value.toUpperCase()];
const { name, uri, params } = TABLE[box.value.toUpperCase()];
// const row = box.closest(".form-row");
// const fromInput = row.querySelector("input[name='fromDate']");
// const toInput = row.querySelector("input[name='toDate']");
// const period = [fromInput.value, toInput.value];
// const period = ['2025-01-01', '2025-02-01'];
const period = new Plan(null, params).get()?.period;
const diff = getTimeDiff(period[1], 1);
let select = true;
if (diff < 0 && diff > -12) {
select = confirm("时间未到, 是否确认继续");
const period = new Plan(getLocalTime(), params).get()?.period;
if (period === undefined) {
addLog(`${name} 未到指定时间`, true);
return;
}
// Deprecated
const diff = getTimeDiff(period[1], 1);
if (-12 < diff && diff < 0) {
const select = confirm("时间未到, 是否确认继续");
if (!select) return;
}
chrome.storage.local.get(['isRunning'], ({ isRunning }) => {
if (!isRunning) {
chrome.runtime.sendMessage({ type: "run", data: { period, uri } });
......@@ -75,6 +79,5 @@ window.onload = function () {
renderLogs(msg.logs);
}
});
// 到点触发
btn.addEventListener("click", onclick);
}
plugin/src/task.js 0 → 100644
import {
sleep, JSON2CSV, createZip, renameZipFile,
} from "./util.js";
import { headers } from "./header.js";
const DELAY = 2000; // 2s
export class TaskGroup {
constructor(tasks, options = {}) {
this.tasks = tasks;
this.task0 = tasks[0];
this.unique = options.unique;
this.len = this.unique ? 1 : this.tasks.length;
}
getFileName() {
const pf = this.len > 1 ? this.len : null;
return this.task0.getFileName(pf).replace(/\.\w+$/g, ".zip");
}
async send() {
let fns = [];
let blobs = (
await Promise.all(
this.tasks
.map((task, idx) => {
if (this.unique) {
task.options.retText = true;
}
const pf = this.len > 1 ? `${this.len}-${idx + 1}` : null;
fns.push(task.getFileName(pf));
return task.send(true);
})))
.filter(x => x !== false && x !== true);
if (this.unique) {
const reg = /\n|\r/g; // 换行符
blobs = [blobs.reduce((prev, cur) => {
return (prev.match(reg)?.length || 0) > (cur.match(reg)?.length || 0) ? prev : cur;
})];
}
if (blobs.length > 0) {
return createZip(blobs, fns);
} else {
return false;
}
}
}
export class Task {
static partnerId = null;
static uri = null;
static desc = null;
constructor(names, params, options) {
this.names = names;
this.params = params;
this.options = options || {};
this.uri = names.uri || Task.uri;
this.partnerId = names.partnerId || Task.partnerId;
this.desc = names.desc || Task.desc;
if (!(this.uri && this.partnerId)) {
throw new Error(`uri: ${this.uri}, partnerId: ${this.partnerId}`);
}
this._retryLimit = 6
this._retryCount = 0
}
getFileName(sf) {
const ext = this.options.ext || 'csv';
let name = this.names.showName;
name = this.options.noLowerCase ? name : name.toLowerCase();
sf = sf === undefined || sf === null ? '' : ' ' + sf;
return `${this.partnerId} #${name}# ${this.desc}${sf}.${ext}`;
}
async _retry(args) {
if (++this._retryCount < this._retryLimit) {
await sleep(DELAY * this._retryCount);
const result = await this.send(...args); // bind
this._retryCount = 0;
return result;
} else {
console.error(`重试失败: ${JSON.stringify(this)}`);
this._retryCount = 0;
return false;
}
}
async send(next = false) {
const url = new URL(`${this.uri}${this.names.apiName || ''}`);
const method = this.options.method || 'GET';
const callback = this.options.callback;
const ext = this.options.ext || 'csv';
next |= this.options.next;
let body = this.options.body;
body = (body && JSON.stringify(body)) || null;
url.search = new URLSearchParams(this.params).toString();
console.log(`${method} ${url}`);
try {
let response = await fetch(url, {
body,
method,
headers: this.options.headers || headers,
...this.options.requestInit
});
console.log(response);
if (response.status !== 200) {
const text = await response.text();
console.log(`重试请求, ${response.url}, ${text}`);
return this._retry(arguments);
}
let blob;
if (callback) {
try {
blob = await callback.call(this, response);
} catch (e) {
console.error(`重试回调, ${e.message}`);
return this._retry(arguments);
}
} else if (ext == 'json') {
const json = await response.json();
if (Object.keys(json).length === 0) {
return true;
}
const csv = JSON2CSV(json);
blob = new Blob([csv], { type: "text/csv" });
this.options.ext = "csv";
} else {
blob = await response.blob();
}
if (blob === undefined || blob === null) return false;
if (next) return blob;
const fn = this.getFileName();
if (ext === 'zip') {
const newFn = fn.replace(/\.\w+$/g, ".csv");
return renameZipFile(blob, newFn);
}
return createZip([blob], [fn]);
} catch (e) {
console.error(`send ${e.message} ${JSON.stringify(this)}`);
return false;
}
}
}
\ No newline at end of file
plugin/src/timezone.js
import dayjs from "dayjs";
import utc from "dayjs/plugin/utc.js";
import timezone from "dayjs/plugin/timezone.js";
import isSameOrAfter from "dayjs/plugin/isSameOrAfter.js";
import isSameOrBefore from "dayjs/plugin/isSameOrBefore.js";
dayjs.extend(utc);
dayjs.extend(timezone);
dayjs.extend(isSameOrAfter);
dayjs.extend(isSameOrBefore);
export function getTimeDiff(input, offset = 0) {
const time = dayjs.tz(input, "YYYY-MM-DD", "America/Los_Angeles");
time.add(offset, "day");
const CST = "Asia/Shanghai";
const PT = "America/Los_Angeles";
const checkTime = (time) => {
if (!time.isValid()) throw new Error("无效时间");
}
export function getLocalTime(dt) {
const time = dayjs.tz(dt ?? new Date(), CST);
checkTime(time);
return time.toDate();
}
export function getMouth(dt, offset = 0) {
let time = dayjs.tz(dt ?? new Date(), CST).add(offset, "month");
checkTime(time);
return time.toDate();
}
export function cutDate(period, date) {
const d1 = dayjs(period[0]);
const d2 = dayjs(period[1]);
if (!date || (d1.isSameOrAfter(date, "day") && d2.isSameOrAfter(date, "day"))) {
return [[1, 2], false];
} else if (d1.isSameOrBefore(date, "day") && d2.isSameOrBefore(date, "day")) {
return [[0, 1], false];
} else {
return [[0, 2], true];
}
}
export function isLatestDate(dt) {
const time = dayjs.tz(dt ?? new Date(), PT);
checkTime(time);
const now = dayjs.tz(undefined, PT);
return time.isSame(now, "day")
|| time.isSame(now.subtract(1, "day"), "day")
|| time.isSame(now.add(1, "day"), "day");
}
export function getTimeDiff(dt, offset = 0) {
const time = dayjs
.tz(dt ?? new Date(), PT)
.add(offset, "day");
checkTime(time);
return (dayjs().valueOf() - time.valueOf()) / 1000 / 60 / 60; // h
}
plugin/src/upload.js
import { retry } from "./util.js";
async function upload(url, body, headers) {
console.log(`上传 ${url}: ${body}`);
console.log(`上传 ${url} ${body}`);
let response = await fetch(url, {
method: "POST", body, headers
});
console.log(response);
try {
let data = await response.json();
if (data.status.toUpperCase() === "OK") {
console.log(data);
if (data.status === "OK") return true;
} catch (e) {
console.error(e.message);
return true;
} else {
throw new Error(`上传失败, ${url}, ${JSON.stringify(data)}`);
}
return false;
}
export async function checkLog(fn, insert = false) {
const url = "https://walmart.meinuosha.com/index.php/index/index/Getspluginexporttablerecords?accessvalue=Walmart2025PY0307";
......@@ -35,8 +36,10 @@ export async function uploadFile(fn, data, options = {}) {
const linkURL = "https://walmart.meinuosha.com/index.php/index/index/url_download_list?accessvalue=Walmart2025PY0307";
const url = options.isJson ? jsonURL : options.isLink ? linkURL : fileURL;
if (await checkLog(fn)) {
if (await upload(url, data, options.headers)) {
await checkLog(fn, true);
}
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5 }
);
if (insert) await checkLog(fn, true);
}
}
plugin/src/util.js
......@@ -14,11 +14,11 @@ export function getValue(key, str, sep = ';') {
item = item.trim();
if (item.startsWith(key)) {
let value = item.substring(key.length + 1, item.length);
console.log(`已找到${key}: ${value}`);
console.log(`已找到 ${key}: ${value}`);
return value;
}
}
console.warn(`未找到${key}`);
console.warn(`未找到 ${key}`);
return '';
}
const replacer = (value) => {
......@@ -34,6 +34,7 @@ export function xpath(str) {
return document.evaluate(str, document).iterateNext();
}
export function JSON2CSV(json) {
try {
const cols = Object.keys(json[0]);
let csv = cols.join(',') + '\r\n';
json.forEach(row => {
......@@ -41,10 +42,65 @@ export function JSON2CSV(json) {
csv += values.join(',') + '\r\n';
});
return csv;
} catch (e) {
console.error(`json2csv 失败: ${e.message}, json: ${JSON.stringify(json)}`);
return null;
}
}
export const decodeState = (str) => {
try {
const base64 = str.replace(/[\n\s_-]/g, ch =>
ch === '-' ? '+' :
ch === '_' ? '/' :
''
).replace(/[^A-Za-z0-9+/]/g, '');
return JSON.parse(
new TextDecoder().decode(
Uint8Array.from(
atob(base64), c => c.charCodeAt(0)
)));
} catch (e) {
console.error(`decodeState 失败: ${e.message}, str: ${str}`);
return null;
}
}
export const decodeState2 = (str) => {
try {
const payload = str.split('.')[1];
const pad = '='.repeat((4 - payload.length % 4) % 4);
return JSON.parse(atob(
payload.replace(/-/g, '+').replace(/_/g, '/') + pad
));
} catch (e) {
console.error(`decodeState2 失败: ${e.message}, str: ${str}`);
return null;
}
}
export async function retry(asyncFunc, options = {}, args = []) {
let { retries = 3, factor = 1, delayMs = 2000, isThrow = true } = options;
let attempt = 0;
while (attempt < retries) {
try {
return await asyncFunc(...args);
} catch (e) {
attempt++;
await sleep(delayMs);
delayMs *= factor;
console.log(`${delayMs}ms 后重试, ${attempt}/${retries}, ${e.message}`);
if (attempt >= retries && isThrow) throw e;
}
}
return null;
}
export const checkResponse = async (resp) => {
if (resp.status !== 200) {
const text = await resp.text();
throw new Error(`异常响应: ${text}`);
}
}
export async function createZip(blobs, fileNames) {
if (blobs.length !== fileNames.length) {
throw new Error("数组长度不一致");
if (blobs.length !== fileNames.length || blobs.length === 0) {
throw new Error("数组长度不一致或为空");
}
const zip = new JSZip();
blobs.forEach((blob, idx) => {
......@@ -96,9 +152,10 @@ export class Plan {
month = month == 0 ? 11 : month - 1;
year = month == 0 ? year - 1 : year;
}
month++;
const endDate = new Date(year, month, 0).getDate() + '';
x2 = x2 === 0 ? endDate : x2;
month = (month + 1 + '').padStart(2, '0');
month = (month + '').padStart(2, '0');
value.period = [
`${year}-${month}-${x1}`,
`${year}-${month}-${x2}`
......@@ -138,7 +195,7 @@ export class Plan {
this.params.push(arguments);
}
}
export function fmt0(d1, diff) {
export function fmt0(d1, diff = 0) {
const dt1 = new Date(d1);
dt1.setDate(d1.getDate() + diff);
const str = dt1.toISOString();
......@@ -167,3 +224,10 @@ export function ffmt1(d1, d2) {
const dt2 = new Date(d2);
return `${dt1.getMonth() + 1}月${dt1.getDate()}-${dt2.getDate()}`;
}
export const addLog = (message, push) => {
const logEntry = {
time: new Date().toLocaleTimeString(),
message
};
chrome.runtime.sendMessage({ type: "addLog", data: logEntry, push });
}
src/walmart_sup.py 0 → 100644
import asyncio
import os
from pathlib import Path
from datetime import datetime, time
import redis.asyncio as redis
from loguru import logger
from curl_cffi.requests import AsyncSession, Response
IS_DEBUG = int(os.environ.get("IS_DEBUG", False))
LOG_PATH = f"./log/{Path(__file__).stem}.log"
if not IS_DEBUG: logger.remove()
logger.add(LOG_PATH, level="INFO", rotation="1 day", retention="3 months")
class Subsidiary:
def __init__(self):
self._tasks = []
self.s = AsyncSession()
self.r: redis.Redis = None
async def connect(self):
if self.r is None:
self.r = redis.Redis(host="localhost", port=6380, db=0)
if await self.r.ping():
logger.info(f"redis 连接成功")
async def bind(self):
while True:
try:
url = "http://www.zdopen.com/ShortProxy/BindIP?api=202406041824314753&akey=4fee8e19764876e1&i=2"
resp: Response = await self.s.get(url)
logger.debug(resp.json())
await asyncio.sleep(10)
except Exception as e:
logger.warning(f"绑定失败: {e}")
async def cache_ips(self):
while True:
try:
url = "https://20tools.net/api/proxies/all?sign=ftd*kcm.ygh4mjp7ERJ"
resp: Response = await self.s.get(url)
ips = resp.json()
await self.r.sadd("walmart_ips", *ips)
await self.r.expire("walmart_ips", 1)
logger.debug(f"缓存ip ({len(ips)}): {ips}")
await asyncio.sleep(1)
except Exception as e:
logger.warning(f"缓存ip失败: {e}")
async def close(self):
await self.r.aclose()
await self.s.close()
logger.info("资源已回收")
async def run_during_night(self):
"""在晚上运行任务
"""
while True:
now = datetime.now().time()
if time(22, 0) > now and now > time(7, 0):
for task in self._tasks:
if task and not task.done():
task.cancel()
logger.info("不在指定时间范围内, 停止运行")
break
await asyncio.sleep(60)
async def run(self, monitor: bool = False):
try:
await self.connect()
self._tasks = [
asyncio.create_task(self.bind()),
asyncio.create_task(self.cache_ips()),
]
if monitor:
monitor_task = asyncio.create_task(self.run_during_night())
await asyncio.gather(*self._tasks, monitor_task)
else:
await asyncio.gather(*self._tasks)
except asyncio.CancelledError:
logger.info("任务已取消")
# raise
finally:
await self.close()
if __name__ == "__main__":
loop = asyncio.get_event_loop()
if IS_DEBUG:
loop.run_until_complete(Subsidiary().run())
else:
loop.run_until_complete(Subsidiary().run(monitor=True))
{75C21EF1-43D8-4E80-8EB6-A205EAF38138} 0 → 100644
356220ce9ae494f6ce2668faebef9f69
fc0fe2bafa800f057ae1660385496d2b
4fa82dcab0ee0309eb7bd1b6e89ec994
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment