Node.js에서 회전하는 레지덴셜 프록시: undici, axios, got, SOCKS5

import { ProxyAgent, fetch } from "undici";

const credentials = Buffer.from("USERNAME-country-us:PASSWORD").toString("base64");
const dispatcher = new ProxyAgent({
  uri: "http://gw.crawlrelay.com:7777",
  token: `Basic ${credentials}`,
});

const res = await fetch("https://api.ipify.org?format=json", {
  dispatcher,
  signal: AbortSignal.timeout(30000),
});
console.log(res.status, await res.json());
await dispatcher.close();

import axios from "axios";
import { HttpsProxyAgent } from "https-proxy-agent";

const agent = new HttpsProxyAgent("http://USERNAME-country-de:[email protected]:7777");
const client = axios.create({ httpsAgent: agent, httpAgent: agent, proxy: false, timeout: 30000 });

const { data } = await client.get("https://api.ipify.org?format=json");
console.log(data.ip);

import got from "got";
import { HttpsProxyAgent } from "hpagent";

const proxy = "http://USERNAME-country-fr:[email protected]:7777";
const body = await got("https://api.ipify.org", {
  agent: { https: new HttpsProxyAgent({ proxy, keepAlive: false }) },
  timeout: { request: 30000 },
  retry: { limit: 2, statusCodes: [429, 502, 503, 504] },
}).text();
console.log(body);

import https from "node:https";
import { SocksProxyAgent } from "socks-proxy-agent";

const agent = new SocksProxyAgent("socks5h://USERNAME-country-gb:[email protected]:7778");

const req = https.get("https://api.ipify.org", { agent, timeout: 30000 }, (res) => {
  let body = "";
  res.on("data", (chunk) => (body += chunk));
  res.on("end", () => console.log(res.statusCode, body));
});
req.on("timeout", () => req.destroy(new Error("timeout after 30 s")));
req.on("error", (err) => console.error(err.code, err.message));

import { ProxyAgent, fetch } from "undici";

const PER_HOST = 4;
const active = new Map();
const waiting = new Map();
const dispatcher = new ProxyAgent("http://USERNAME-country-us:[email protected]:7777");

async function withHostSlot(url, task) {
  const host = new URL(url).hostname;
  while ((active.get(host) ?? 0) >= PER_HOST) {
    await new Promise((resolve) => {
      if (!waiting.has(host)) waiting.set(host, []);
      waiting.get(host).push(resolve);
    });
  }
  active.set(host, (active.get(host) ?? 0) + 1);
  try {
    return await task();
  } finally {
    active.set(host, active.get(host) - 1);
    waiting.get(host)?.shift()?.();
  }
}

const urls = ["https://example.com/", "https://example.org/", "https://example.net/"];
const results = await Promise.all(
  urls.map((url) =>
    withHostSlot(url, async () => {
      const res = await fetch(url, { dispatcher, signal: AbortSignal.timeout(30000) });
      return [url, res.status, (await res.arrayBuffer()).byteLength];
    }).catch((err) => [url, err.name, 0])
  )
);
console.table(results);
await dispatcher.close();