mirror of
https://github.com/caty21/forex-dashboard.git
synced 2026-08-25 18:38:07 +00:00
fix: RBNZ MPS — le retry curl ne se déclenchait jamais sur un 403
curl sans --fail sort avec le code 0 même sur une erreur HTTP (il écrit juste la page d'erreur sur stdout), donc le try/catch du wrapper de retry ne capturait rien : le blocage Cloudflare éventuel des IPs GitHub Actions n'aurait jamais bénéficié des tentatives supplémentaires. Ajoute --fail (+ un court backoff entre tentatives) pour que le retry serve réellement à quelque chose.
This commit is contained in:
@@ -27,22 +27,45 @@ import { execFileSync } from "child_process";
|
|||||||
|
|
||||||
const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
|
const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
|
||||||
|
|
||||||
function curlGet(url) {
|
function curlGetOnce(url) {
|
||||||
return execFileSync("curl", [
|
return execFileSync("curl", [
|
||||||
"-s", "-L", "--max-time", "30",
|
"-sS", "-L", "--fail", "--max-time", "30",
|
||||||
"-A", UA,
|
"-A", UA,
|
||||||
"-H", "Accept-Language: en-US,en;q=0.9",
|
"-H", "Accept-Language: en-US,en;q=0.9",
|
||||||
url,
|
url,
|
||||||
], { maxBuffer: 1024 * 1024 * 50 }); // Buffer (binaire OK pour le PDF)
|
], { maxBuffer: 1024 * 1024 * 50 }); // Buffer (binaire OK pour le PDF)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sleep(ms) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Le blocage Cloudflare sur les IPs des runners GitHub Actions est
|
||||||
|
// intermittent (constaté ailleurs dans ce repo pour investing.com) : quelques
|
||||||
|
// tentatives supplémentaires suffisent parfois à passer. --fail est
|
||||||
|
// indispensable ici : sans lui, curl sort avec le code 0 même sur un 403 (il
|
||||||
|
// affiche juste la page d'erreur Cloudflare sur stdout), donc le try/catch
|
||||||
|
// ci-dessous ne déclenchait jamais de retry — l'échec n'était détecté que
|
||||||
|
// plus loin, hors de cette fonction, par la validation de contenu.
|
||||||
|
async function curlGet(url, attempts = 3) {
|
||||||
|
let lastErr;
|
||||||
|
for (let i = 0; i < attempts; i++) {
|
||||||
|
try { return curlGetOnce(url); }
|
||||||
|
catch (e) {
|
||||||
|
lastErr = e;
|
||||||
|
if (i < attempts - 1) await sleep(2000 * (i + 1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastErr;
|
||||||
|
}
|
||||||
|
|
||||||
const MONTHS = {
|
const MONTHS = {
|
||||||
jan: 1, feb: 2, mar: 3, apr: 4, may: 5, jun: 6,
|
jan: 1, feb: 2, mar: 3, apr: 4, may: 5, jun: 6,
|
||||||
jul: 7, aug: 8, sep: 9, oct: 10, nov: 11, dec: 12,
|
jul: 7, aug: 8, sep: 9, oct: 10, nov: 11, dec: 12,
|
||||||
};
|
};
|
||||||
|
|
||||||
async function findLatestMpsPageUrl() {
|
async function findLatestMpsPageUrl() {
|
||||||
const xml = curlGet("https://www.rbnz.govt.nz/sitemap.xml").toString("utf8");
|
const xml = (await curlGet("https://www.rbnz.govt.nz/sitemap.xml")).toString("utf8");
|
||||||
if (!xml.includes("<urlset")) throw new Error("sitemap.xml : réponse inattendue (bloqué ?)");
|
if (!xml.includes("<urlset")) throw new Error("sitemap.xml : réponse inattendue (bloqué ?)");
|
||||||
|
|
||||||
const re = /<loc>(https:\/\/www\.rbnz\.govt\.nz\/monetary-policy\/monetary-policy-statement\/monetary-policy-statement-filtered-listing-page\/(\d{4})\/([a-z]+)-\d+\/monetary-policy-statement-[a-z]+-\d{4})<\/loc>/g;
|
const re = /<loc>(https:\/\/www\.rbnz\.govt\.nz\/monetary-policy\/monetary-policy-statement\/monetary-policy-statement-filtered-listing-page\/(\d{4})\/([a-z]+)-\d+\/monetary-policy-statement-[a-z]+-\d{4})<\/loc>/g;
|
||||||
@@ -61,7 +84,7 @@ async function findLatestMpsPageUrl() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function findPdfUrl(pageUrl) {
|
async function findPdfUrl(pageUrl) {
|
||||||
const html = curlGet(pageUrl).toString("utf8");
|
const html = (await curlGet(pageUrl)).toString("utf8");
|
||||||
const m = html.match(/href="(https:\/\/www\.rbnz\.govt\.nz\/-\/media\/[^"]+monetary-policy-statement-[^"]+\.pdf)"/i)
|
const m = html.match(/href="(https:\/\/www\.rbnz\.govt\.nz\/-\/media\/[^"]+monetary-policy-statement-[^"]+\.pdf)"/i)
|
||||||
?? html.match(/href="(\/-\/media\/[^"]+monetary-policy-statement-[^"]+\.pdf)"/i);
|
?? html.match(/href="(\/-\/media\/[^"]+monetary-policy-statement-[^"]+\.pdf)"/i);
|
||||||
if (!m) throw new Error("Lien PDF introuvable sur la page MPS");
|
if (!m) throw new Error("Lien PDF introuvable sur la page MPS");
|
||||||
@@ -69,7 +92,7 @@ async function findPdfUrl(pageUrl) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function fetchPdfText(pdfUrl) {
|
async function fetchPdfText(pdfUrl) {
|
||||||
const buf = curlGet(pdfUrl);
|
const buf = await curlGet(pdfUrl);
|
||||||
if (buf.length < 1000 || buf.subarray(0, 4).toString("latin1") !== "%PDF") {
|
if (buf.length < 1000 || buf.subarray(0, 4).toString("latin1") !== "%PDF") {
|
||||||
throw new Error(`PDF invalide (${buf.length} bytes, bloqué ?)`);
|
throw new Error(`PDF invalide (${buf.length} bytes, bloqué ?)`);
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"updated_at": "2026-07-08",
|
"updated_at": "2026-07-09",
|
||||||
"sourcePageUrl": "https://www.rbnz.govt.nz/monetary-policy/monetary-policy-statement/monetary-policy-statement-filtered-listing-page/2026/may-270/monetary-policy-statement-may-2026",
|
"sourcePageUrl": "https://www.rbnz.govt.nz/monetary-policy/monetary-policy-statement/monetary-policy-statement-filtered-listing-page/2026/may-270/monetary-policy-statement-may-2026",
|
||||||
"sourcePdfUrl": "https://www.rbnz.govt.nz/-/media/project/sites/rbnz/files/publications/monetary-policy-statements/2026/may-272605/monetary-policy-statement-may-2026.pdf",
|
"sourcePdfUrl": "https://www.rbnz.govt.nz/-/media/project/sites/rbnz/files/publications/monetary-policy-statements/2026/may-272605/monetary-policy-statement-may-2026.pdf",
|
||||||
"mpsYear": 2026,
|
"mpsYear": 2026,
|
||||||
|
|||||||
Reference in New Issue
Block a user