Skip to content

W
wfs_export
Commit cf92a603 by yexing
Options

重试优化

parent 748ce478
Showing with 124 additions and 66 deletions
plugin/.version
3.4.8
\ No newline at end of file
3.4.9
\ No newline at end of file
plugin/dist.crx
No preview for this file type
plugin/dist/js/background.js
......@@ -19628,13 +19628,12 @@
} else {
url = options.isLink ? linkURL : fileURL;
}
if (await checkLog(fn)) {
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5 }
);
if (insert) await checkLog(fn, true);
}
if (options.isCheck && !await checkLog(fn)) return;
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5, retries: 10 }
);
if (insert) await checkLog(fn, true);
}
const checkURL = (u1, u2) => {
......
plugin/dist/js/main.js
(function () {
var G = (function (exports) {
'use strict';
var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {};
......@@ -144,6 +144,15 @@
compressionOptions: { level: 9 }
});
}
function downloadFile(fn, blob) {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = fn;
a.click();
a.remove();
URL.revokeObjectURL(url);
}
function fmt0(d1, diff = 0) {
const dt1 = new Date(d1);
dt1.setDate(d1.getDate() + diff);
......@@ -263,7 +272,11 @@
message
}
};
const addLog = (message, push) => chrome.runtime.sendMessage({ type: "addLog", data: makeLogEntry(message), push });
const addLog = (message, push) => {
if (typeof chrome !== 'undefined' && chrome.runtime) {
chrome.runtime.sendMessage({ type: "addLog", data: makeLogEntry(message), push });
}
};
const checkTime = (time) => {
if (!time.isValid()) throw new Error("无效时间");
......@@ -445,6 +458,10 @@
}
return createZip([blob], [fn]);
} catch (e) {
if (e.message === 'Failed to fetch') {
console.info(`重试 fetch`);
return this._retry(arguments);
}
console.error(`send ${e.message} ${JSON.stringify(this)}`);
return false;
}
......@@ -1265,13 +1282,12 @@
} else {
url = options.isLink ? linkURL : fileURL;
}
if (await checkLog(fn)) {
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5 }
);
if (insert) await checkLog(fn, true);
}
if (options.isCheck && !await checkLog(fn)) return;
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5, retries: 10 }
);
if (insert) await checkLog(fn, true);
}
// fetch("https://advertising.walmart.com/extend-sso")
......@@ -1285,16 +1301,15 @@
).click();
}
}async function run(options = {}) {
const { uri, plan, sn } = options;
const { uri, plan, sn, isDownload = false } = options;
plan.period ||= [fmt0(new Date(), -10), fmt0(new Date(), -1)];
const [fromDate, toDate] = plan.period;
const [key, { name }] = Object.entries(TABLE).find((item) => item[1].uri === uri);
let tasks = await createTasks(uri, fromDate, toDate);
tasks = sn ? [tasks.at(sn)] : tasks;
// 在季度周期过滤掉月度任务
// tasks = tasks.filter((x)=>!x.isMonthly);
let moment = 2000, idx = 0, len = tasks.length;
const allData = [], allFn = [], zipFn = `${Task.partnerId} #${key}# ${fromDate}_${toDate}.zip`;
const isSingle = uri == TABLE.PAYMENT.uri;
for (const task of tasks) {
const pf = `${++idx}/${len}`;
addLog(`${name} 运行中 ${pf}`);
......@@ -1303,26 +1318,41 @@
continue;
}
console.log(`${pf} start: ${JSON.stringify(task)}`);
let fn = task.getFileName();
const isJson = fn.endsWith('.json');
if (!isJson) fn = fn.replace(/\.\w+$/g, ".zip");
if ((isSingle || isJson) && !await checkLog(fn)) {
console.log(`${pf} skip2`);
continue;
}
let data = await task.send();
if (data === true || data === false) {
console.log(`${pf} skip2`);
console.log(`${pf} skip3`);
continue;
}
let fn = task.getFileName();
const isJson = fn.endsWith('.json');
if (isJson) {
const route = data.__route;
delete data.__route;
const headers = { 'Content-Type': 'application/json' };
await uploadFile(fn, data, { isJson, headers, route });
// const blob = new Blob([data], { "type": "application/json" });
// downloadFile(fn, blob);
if (isDownload) {
// const blob = new Blob([data], { "type": "application/json" });
// downloadFile(fn, blob);
console.log(data);
}
} else {
fn = fn.replace(/\.\w+$/g, ".zip");
allData.push(data);
allFn.push(fn);
// downloadFile(fn, data);
if (isSingle) {
const zip = await createZip([data], [fn]);
const form = new FormData();
form.append("zipfile", zip, fn);
await uploadFile(fn, form);
} else {
allData.push(data);
allFn.push(fn);
}
}
console.log(`${pf} end: ${fn}`);
await sleep(moment);
......@@ -1331,24 +1361,31 @@
const zip = await createZip(allData, allFn);
const form = new FormData();
form.append("zipfile", zip, zipFn);
await uploadFile(zipFn, form);
await uploadFile(zipFn, form, { isCheck: true });
isDownload && downloadFile(zipFn, zip);
}
addLog(`${name} 运行完成`);
}
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
(async () => {
try {
if (msg.type === "jump") {
await webJump();
} else if (msg.type === "run") {
await run(msg.data); // 默认
if (typeof chrome !== 'undefined' && chrome.runtime) {
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
(async () => {
try {
if (msg.type === "jump") {
await webJump();
} else if (msg.type === "run") {
await run(msg.data); // 默认
}
sendResponse({ status: "ok" });
} catch (err) {
sendResponse({ status: "error", error: err.message });
}
sendResponse({ status: "ok" });
} catch (err) {
sendResponse({ status: "error", error: err.message });
}
})();
return true;
});
})();
return true;
});
}
exports.run = run;
return exports;
})();
})({});
plugin/dist/js/popup.js
......@@ -213,7 +213,11 @@
message
}
};
const addLog = (message, push) => chrome.runtime.sendMessage({ type: "addLog", data: makeLogEntry(message), push });
const addLog = (message, push) => {
if (typeof chrome !== 'undefined' && chrome.runtime) {
chrome.runtime.sendMessage({ type: "addLog", data: makeLogEntry(message), push });
}
};
const checkTime = (time) => {
if (!time.isValid()) throw new Error("无效时间");
......
plugin/dist/manifest.json
{"name":"WalmartExports","description":"导出沃尔玛报表","version":"3.4.8","manifest_version":3,"background":{"service_worker":"./js/background.js"},"permissions":["tabs","activeTab","scripting","notifications","storage","alarms"],"action":{"default_title":"WalmartExports","default_popup":"popup.html","default_icon":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"}},"icons":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"},"web_accessible_resources":[{"resources":["src/main.js"],"matches":["<all_urls>"]}],"host_permissions":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"content_scripts":[{"matches":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"js":["./js/main.js"]}]}
\ No newline at end of file
{"name":"WalmartExports","description":"导出沃尔玛报表","version":"3.4.9","manifest_version":3,"background":{"service_worker":"./js/background.js"},"permissions":["tabs","activeTab","scripting","notifications","storage","alarms"],"action":{"default_title":"WalmartExports","default_popup":"popup.html","default_icon":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"}},"icons":{"16":"icon.png","32":"icon.png","48":"icon.png","128":"icon.png"},"web_accessible_resources":[{"resources":["src/main.js"],"matches":["<all_urls>"]}],"host_permissions":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"content_scripts":[{"matches":["https://seller.walmart.com/*","https://advertising.walmart.com/*","https://login.account.wal-mart.com/*","https://marketplace.walmartapis.com/*","https://walmart.meinuosha.com/*"],"js":["./js/main.js"]}]}
\ No newline at end of file
plugin/src/main.js
......@@ -2,7 +2,7 @@ import { sleep, xpath, createZip, fmt0, downloadFile } from "./util.js";
import { addLog } from "./timezone.js";
import { createTasks } from "./base.js";
import { Task } from "./task.js";
import { uploadFile } from "./upload.js";
import { uploadFile, checkLog } from "./upload.js";
import { TABLE } from "./conf.js";
// fetch("https://advertising.walmart.com/extend-sso")
......@@ -25,6 +25,7 @@ export async function run(options = {}) {
tasks = sn ? [tasks.at(sn)] : tasks;
let moment = 2000, idx = 0, len = tasks.length;
const allData = [], allFn = [], zipFn = `${Task.partnerId} #${key}# ${fromDate}_${toDate}.zip`;
const isSingle = uri == TABLE.PAYMENT.uri;
for (const task of tasks) {
const pf = `${++idx}/${len}`;
addLog(`${name} 运行中 ${pf}`);
......@@ -33,27 +34,41 @@ export async function run(options = {}) {
continue;
}
console.log(`${pf} start: ${JSON.stringify(task)}`);
let fn = task.getFileName();
const isJson = fn.endsWith('.json');
if (!isJson) fn = fn.replace(/\.\w+$/g, ".zip");
if ((isSingle || isJson) && !await checkLog(fn)) {
console.log(`${pf} skip2`);
continue;
}
let data = await task.send();
if (data === true || data === false) {
console.log(`${pf} skip2`);
console.log(`${pf} skip3`);
continue;
}
let fn = task.getFileName();
const isJson = fn.endsWith('.json');
if (isJson) {
const route = data.__route;
delete data.__route;
const headers = { 'Content-Type': 'application/json' }
await uploadFile(fn, data, { isJson, headers, route });
if (isDownload) {
const blob = new Blob([data], { "type": "application/json" });
downloadFile(fn, blob);
// const blob = new Blob([data], { "type": "application/json" });
// downloadFile(fn, blob);
console.log(data);
}
} else {
fn = fn.replace(/\.\w+$/g, ".zip");
allData.push(data);
allFn.push(fn);
if (isSingle) {
const zip = await createZip([data], [fn]);
const form = new FormData();
form.append("zipfile", zip, fn);
await uploadFile(fn, form);
} else {
allData.push(data);
allFn.push(fn);
}
}
console.log(`${pf} end: ${fn}`);
await sleep(moment);
......@@ -62,7 +77,7 @@ export async function run(options = {}) {
const zip = await createZip(allData, allFn);
const form = new FormData();
form.append("zipfile", zip, zipFn);
await uploadFile(zipFn, form);
await uploadFile(zipFn, form, { isCheck: true });
isDownload && downloadFile(zipFn, zip);
}
addLog(`${name} 运行完成`);
......
plugin/src/task.js
......@@ -134,6 +134,10 @@ export class Task {
}
return createZip([blob], [fn]);
} catch (e) {
if (e.message === 'Failed to fetch') {
console.info(`重试 fetch`);
return this._retry(arguments);
}
console.error(`send ${e.message} ${JSON.stringify(this)}`);
return false;
}
......
plugin/src/upload.js
......@@ -40,11 +40,10 @@ export async function uploadFile(fn, data, options = {}) {
} else {
url = options.isLink ? linkURL : fileURL;
}
if (await checkLog(fn)) {
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5 }
);
if (insert) await checkLog(fn, true);
}
if (options.isCheck && !await checkLog(fn)) return;
const insert = await retry(
async () => upload(url, data, options.headers),
{ delayMs: 1000 * 60 * 1.5, retries: 10 }
);
if (insert) await checkLog(fn, true);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment