|
@@ -5,12 +5,12 @@ const db = require('../../plugin/DataBase/db')
|
|
|
const Redis = require('../../plugin/DataBase/Redis')
|
|
const Redis = require('../../plugin/DataBase/Redis')
|
|
|
const Logger = require('../Logger')
|
|
const Logger = require('../Logger')
|
|
|
|
|
|
|
|
-const QG_GET_URL = 'https://share.proxy.qg.net/get'
|
|
|
|
|
|
|
+const QG_POOL_URL = 'https://share.proxy.qg.net/pool'
|
|
|
const QG_RESOURCES_URL = 'https://share.proxy.qg.net/resources'
|
|
const QG_RESOURCES_URL = 'https://share.proxy.qg.net/resources'
|
|
|
const REDIS_CURRENT = 'lepao:qg:current'
|
|
const REDIS_CURRENT = 'lepao:qg:current'
|
|
|
const REDIS_LOCK = 'lepao:qg:fetch_lock'
|
|
const REDIS_LOCK = 'lepao:qg:fetch_lock'
|
|
|
-/** 早于 deadline 提前刷新,尽量减少「边检边过期」(参考 [文档](https://www.qg.net/doc/1850.html)) */
|
|
|
|
|
-const DEADLINE_MARGIN_MS = 18_000
|
|
|
|
|
|
|
+/** 隧道池入口地址缓存时长(服务商后台自动换出口,不依赖本地 deadline 刷新) */
|
|
|
|
|
+const REDIS_SERVER_TTL_SEC = 6 * 60 * 60
|
|
|
/** 仅覆盖单次 /get(含 axios 超时),避免长持锁阻塞其它任务 */
|
|
/** 仅覆盖单次 /get(含 axios 超时),避免长持锁阻塞其它任务 */
|
|
|
const LOCK_TTL_SEC = 45
|
|
const LOCK_TTL_SEC = 45
|
|
|
const LOCK_WAIT_ROUNDS = 40
|
|
const LOCK_WAIT_ROUNDS = 40
|
|
@@ -30,7 +30,8 @@ function getQgConfig() {
|
|
|
return {
|
|
return {
|
|
|
extractKey: String(q.extractKey || '').trim(),
|
|
extractKey: String(q.extractKey || '').trim(),
|
|
|
authUser: String(q.authUser || '').trim(),
|
|
authUser: String(q.authUser || '').trim(),
|
|
|
- authPassword: String(q.authPassword || '').trim()
|
|
|
|
|
|
|
+ authPassword: String(q.authPassword || '').trim(),
|
|
|
|
|
+ tunnelServer: String(q.tunnelServer || q.server || '').trim()
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -44,6 +45,11 @@ function hasProxyAuth() {
|
|
|
return authUser.length > 0 && authPassword.length > 0
|
|
return authUser.length > 0 && authPassword.length > 0
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+function hasTunnelServer() {
|
|
|
|
|
+ const { tunnelServer } = getQgConfig()
|
|
|
|
|
+ return tunnelServer.length > 0
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
async function ensureSettingsRow() {
|
|
async function ensureSettingsRow() {
|
|
|
const now = Date.now()
|
|
const now = Date.now()
|
|
|
await db.query(
|
|
await db.query(
|
|
@@ -88,6 +94,33 @@ function axiosProxyOptsFromServer(server, useAuth) {
|
|
|
return opt
|
|
return opt
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+function isAreaCodeLike(s) {
|
|
|
|
|
+ return /^\d{4,9}$/.test(String(s || '').trim())
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+function buildTunnelProxyOpts(settings) {
|
|
|
|
|
+ const { tunnelServer, authUser, authPassword } = getQgConfig()
|
|
|
|
|
+ if (!tunnelServer) return null
|
|
|
|
|
+ const base = axiosProxyOptsFromServer(tunnelServer, false)
|
|
|
|
|
+ if (!base.proxy) return null
|
|
|
|
|
+
|
|
|
|
|
+ if (authUser && authPassword) {
|
|
|
|
|
+ let password = authPassword
|
|
|
|
|
+ const area = String(settings?.area || '').trim()
|
|
|
|
|
+ const areaEx = String(settings?.area_ex || '').trim()
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 参考隧道代理接入:普通模式指定地区通过 user:password:A<area>@server。
|
|
|
|
|
+ * 仅当 area 为单个纯数字编码且未配置 area_ex 时尝试附加,避免把旧的多地区逗号表达式拼坏。
|
|
|
|
|
+ */
|
|
|
|
|
+ if (area && !areaEx && isAreaCodeLike(area)) {
|
|
|
|
|
+ password = `${password}:A${area}`
|
|
|
|
|
+ }
|
|
|
|
|
+ base.proxy.auth = { username: authUser, password }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return base
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
async function recordLog({ event, server = null, deadline = null, detail = null }) {
|
|
async function recordLog({ event, server = null, deadline = null, detail = null }) {
|
|
|
try {
|
|
try {
|
|
|
const detailStr =
|
|
const detailStr =
|
|
@@ -114,8 +147,10 @@ async function getCachedParsed() {
|
|
|
function cacheStillValid(parsed) {
|
|
function cacheStillValid(parsed) {
|
|
|
if (!parsed || !parsed.server) return false
|
|
if (!parsed || !parsed.server) return false
|
|
|
const ms = parsed.deadlineMs || parseDeadlineMs(parsed.deadline)
|
|
const ms = parsed.deadlineMs || parseDeadlineMs(parsed.deadline)
|
|
|
- if (!ms) return false
|
|
|
|
|
- return Date.now() + DEADLINE_MARGIN_MS < ms
|
|
|
|
|
|
|
+ if (ms) return Date.now() < ms
|
|
|
|
|
+ const fetchedAt = Number(parsed.fetchedAt || 0)
|
|
|
|
|
+ if (!Number.isFinite(fetchedAt) || fetchedAt <= 0) return true
|
|
|
|
|
+ return Date.now() - fetchedAt < REDIS_SERVER_TTL_SEC * 1000
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
async function acquireFetchLock() {
|
|
async function acquireFetchLock() {
|
|
@@ -161,7 +196,7 @@ function buildGetParams(settings) {
|
|
|
|
|
|
|
|
async function extractOnce(settings) {
|
|
async function extractOnce(settings) {
|
|
|
const params = buildGetParams(settings)
|
|
const params = buildGetParams(settings)
|
|
|
- const res = await axios.get(QG_GET_URL, {
|
|
|
|
|
|
|
+ const res = await axios.get(QG_POOL_URL, {
|
|
|
params,
|
|
params,
|
|
|
timeout: 20000,
|
|
timeout: 20000,
|
|
|
proxy: false,
|
|
proxy: false,
|
|
@@ -215,34 +250,49 @@ async function fetchResourceAreas() {
|
|
|
* 是否应在业务层尝试青果(DB 开关 + config 里存在 extractKey)
|
|
* 是否应在业务层尝试青果(DB 开关 + config 里存在 extractKey)
|
|
|
*/
|
|
*/
|
|
|
async function isOutboundProxyEnabled() {
|
|
async function isOutboundProxyEnabled() {
|
|
|
- if (!hasExtractCredentials()) return false
|
|
|
|
|
|
|
+ if (!hasExtractCredentials() && !(hasTunnelServer() && hasProxyAuth())) return false
|
|
|
const row = await loadSettings()
|
|
const row = await loadSettings()
|
|
|
return row && Number(row.proxy_enabled) === 1
|
|
return row && Number(row.proxy_enabled) === 1
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * 对外:得到可合并进 axios 的代理段;未启用则 { proxy: false }
|
|
|
|
|
|
|
+ * 对外:得到可合并进 axios 的代理段;未启用则 { proxy: false }。
|
|
|
|
|
+ * 隧道池模式:服务商后台自动切换出口,本地仅缓存入口 server,不做频繁刷新。
|
|
|
* @param {{ forceRefresh?: boolean }} opt
|
|
* @param {{ forceRefresh?: boolean }} opt
|
|
|
*/
|
|
*/
|
|
|
async function getOutboundAxiosFragment(opt = {}) {
|
|
async function getOutboundAxiosFragment(opt = {}) {
|
|
|
const forceRefresh = opt.forceRefresh === true
|
|
const forceRefresh = opt.forceRefresh === true
|
|
|
warnIfTlsVerifyDisabled()
|
|
warnIfTlsVerifyDisabled()
|
|
|
- if (!hasExtractCredentials()) return { proxy: false }
|
|
|
|
|
|
|
+ if (!hasExtractCredentials() && !(hasTunnelServer() && hasProxyAuth())) return { proxy: false }
|
|
|
|
|
|
|
|
const settings = await loadSettings()
|
|
const settings = await loadSettings()
|
|
|
if (!settings || Number(settings.proxy_enabled) !== 1) return { proxy: false }
|
|
if (!settings || Number(settings.proxy_enabled) !== 1) return { proxy: false }
|
|
|
|
|
|
|
|
- if (!forceRefresh) {
|
|
|
|
|
- const cached = await getCachedParsed()
|
|
|
|
|
- if (cacheStillValid(cached)) {
|
|
|
|
|
- return axiosProxyOptsFromServer(cached.server, hasProxyAuth())
|
|
|
|
|
|
|
+ // 隧道代理模式:直接使用隧道入口地址,不需要 /pool 提取。
|
|
|
|
|
+ const tunnelFrag = buildTunnelProxyOpts(settings)
|
|
|
|
|
+ if (tunnelFrag?.proxy) {
|
|
|
|
|
+ const payload = {
|
|
|
|
|
+ server: `${tunnelFrag.proxy.host}:${tunnelFrag.proxy.port}`,
|
|
|
|
|
+ deadline: '',
|
|
|
|
|
+ deadlineMs: null,
|
|
|
|
|
+ proxyIp: null,
|
|
|
|
|
+ requestId: null,
|
|
|
|
|
+ fetchedAt: Date.now(),
|
|
|
|
|
+ mode: 'tunnel'
|
|
|
}
|
|
}
|
|
|
- if (cached?.server) {
|
|
|
|
|
- const ms = cached.deadlineMs || parseDeadlineMs(cached.deadline)
|
|
|
|
|
- logger.info(
|
|
|
|
|
- `[QgProxy] 缓存代理已失效或临近截止,将重新提取。原节点 server=${cached.server} proxy_ip=${cached.proxyIp ?? '—'} deadline_ms=${ms ?? '—'}`
|
|
|
|
|
- )
|
|
|
|
|
|
|
+ await Redis.set(REDIS_CURRENT, JSON.stringify(payload), { EX: Math.max(60, REDIS_SERVER_TTL_SEC) })
|
|
|
|
|
+ return tunnelFrag
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const cachedFast = await getCachedParsed()
|
|
|
|
|
+ if (cacheStillValid(cachedFast)) {
|
|
|
|
|
+ if (forceRefresh) {
|
|
|
|
|
+ logger.info(`[QgProxy] 隧道池忽略 forceRefresh,复用入口 server=${cachedFast.server}`)
|
|
|
}
|
|
}
|
|
|
|
|
+ return axiosProxyOptsFromServer(cachedFast.server, hasProxyAuth())
|
|
|
|
|
+ }
|
|
|
|
|
+ if (cachedFast?.server) {
|
|
|
|
|
+ logger.info(`[QgProxy] 缓存入口失效,将重新提取。原server=${cachedFast.server}`)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const maxAttempts = 5
|
|
const maxAttempts = 5
|
|
@@ -263,14 +313,14 @@ async function getOutboundAxiosFragment(opt = {}) {
|
|
|
|
|
|
|
|
let shouldBackoff = false
|
|
let shouldBackoff = false
|
|
|
try {
|
|
try {
|
|
|
- if (!forceRefresh) {
|
|
|
|
|
|
|
+ {
|
|
|
const cached = await getCachedParsed()
|
|
const cached = await getCachedParsed()
|
|
|
if (cacheStillValid(cached)) {
|
|
if (cacheStillValid(cached)) {
|
|
|
return axiosProxyOptsFromServer(cached.server, hasProxyAuth())
|
|
return axiosProxyOptsFromServer(cached.server, hasProxyAuth())
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- logger.info(`[QgProxy] 调用青果 /get 拉取新代理... forceRefresh=${forceRefresh}`)
|
|
|
|
|
|
|
+ logger.info(`[QgProxy] 调用青果 /pool 提取代理资源... forceRefresh=${forceRefresh}`)
|
|
|
const { body, item, code } = await extractOnce(settings)
|
|
const { body, item, code } = await extractOnce(settings)
|
|
|
const server = item.server
|
|
const server = item.server
|
|
|
const deadline = item.deadline
|
|
const deadline = item.deadline
|
|
@@ -279,9 +329,7 @@ async function getOutboundAxiosFragment(opt = {}) {
|
|
|
if (!server) throw new Error('青果返回无 server 字段')
|
|
if (!server) throw new Error('青果返回无 server 字段')
|
|
|
|
|
|
|
|
const deadlineMs = parseDeadlineMs(deadline)
|
|
const deadlineMs = parseDeadlineMs(deadline)
|
|
|
- const ttlSec = deadlineMs
|
|
|
|
|
- ? Math.max(15, Math.floor((deadlineMs - Date.now()) / 1000) - 2)
|
|
|
|
|
- : 55
|
|
|
|
|
|
|
+ const ttlSec = REDIS_SERVER_TTL_SEC
|
|
|
|
|
|
|
|
const payload = {
|
|
const payload = {
|
|
|
server,
|
|
server,
|
|
@@ -291,14 +339,14 @@ async function getOutboundAxiosFragment(opt = {}) {
|
|
|
requestId: requestId || null,
|
|
requestId: requestId || null,
|
|
|
fetchedAt: Date.now()
|
|
fetchedAt: Date.now()
|
|
|
}
|
|
}
|
|
|
- await Redis.set(REDIS_CURRENT, JSON.stringify(payload), { EX: Math.min(120, Math.max(15, ttlSec)) })
|
|
|
|
|
|
|
+ await Redis.set(REDIS_CURRENT, JSON.stringify(payload), { EX: Math.max(60, ttlSec) })
|
|
|
|
|
|
|
|
if (attempt > 1) {
|
|
if (attempt > 1) {
|
|
|
logger.info(`[QgProxy] 第 ${attempt} 次 /get 成功`)
|
|
logger.info(`[QgProxy] 第 ${attempt} 次 /get 成功`)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
logger.info(
|
|
logger.info(
|
|
|
- `[QgProxy] 已获取代理节点 server=${server} proxy_ip=${proxyIp ?? '—'} deadline=${deadline || '—'} request_id=${requestId ?? '—'}`
|
|
|
|
|
|
|
+ `[QgProxy] 已获取隧道入口 server=${server} proxy_ip=${proxyIp ?? '—'} deadline=${deadline || '—'} request_id=${requestId ?? '—'}`
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
await recordLog({
|
|
await recordLog({
|
|
@@ -314,7 +362,7 @@ async function getOutboundAxiosFragment(opt = {}) {
|
|
|
shouldBackoff = e.retryable === true && attempt < maxAttempts
|
|
shouldBackoff = e.retryable === true && attempt < maxAttempts
|
|
|
if (shouldBackoff) {
|
|
if (shouldBackoff) {
|
|
|
const backoff = Math.min(2000, 280 * attempt * attempt)
|
|
const backoff = Math.min(2000, 280 * attempt * attempt)
|
|
|
- logger.warn(`[QgProxy] /get 将重试 (${attempt}/${maxAttempts}) ${e.message},等待 ${backoff}ms`)
|
|
|
|
|
|
|
+ logger.warn(`[QgProxy] /pool 将重试 (${attempt}/${maxAttempts}) ${e.message},等待 ${backoff}ms`)
|
|
|
} else {
|
|
} else {
|
|
|
logger.error(`青果拉取异常: ${e.stack || e}`)
|
|
logger.error(`青果拉取异常: ${e.stack || e}`)
|
|
|
throw e
|
|
throw e
|
|
@@ -333,6 +381,19 @@ async function getOutboundAxiosFragment(opt = {}) {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
async function invalidateCurrent(reason, detail) {
|
|
async function invalidateCurrent(reason, detail) {
|
|
|
|
|
+ if (reason === 'request_fail' || reason === 'retry_round_post_fail' || reason === 'extra_round_fail') {
|
|
|
|
|
+ const detailObj =
|
|
|
|
|
+ typeof detail === 'object' && detail !== null ? detail : { message: String(detail || '') }
|
|
|
|
|
+ logger.info(
|
|
|
|
|
+ `[QgProxy] 隧道池模式忽略入口作废 reason=${reason} detail=${JSON.stringify(detailObj)}`
|
|
|
|
|
+ )
|
|
|
|
|
+ await recordLog({
|
|
|
|
|
+ event: 'invalidate',
|
|
|
|
|
+ detail: { reason, ignored_in_tunnel_pool: true, ...detailObj }
|
|
|
|
|
+ })
|
|
|
|
|
+ return
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
let prev = null
|
|
let prev = null
|
|
|
try {
|
|
try {
|
|
|
prev = await getCachedParsed()
|
|
prev = await getCachedParsed()
|
|
@@ -384,6 +445,7 @@ async function getStatusSnapshot() {
|
|
|
distinct_extract: row ? Number(row.distinct_extract) === 1 : true,
|
|
distinct_extract: row ? Number(row.distinct_extract) === 1 : true,
|
|
|
updated_at: row?.updated_at ?? 0,
|
|
updated_at: row?.updated_at ?? 0,
|
|
|
extract_key_configured: hasExtractCredentials(),
|
|
extract_key_configured: hasExtractCredentials(),
|
|
|
|
|
+ tunnel_server_configured: hasTunnelServer(),
|
|
|
proxy_auth_configured: hasProxyAuth(),
|
|
proxy_auth_configured: hasProxyAuth(),
|
|
|
redis_current: cached && cacheStillValid(cached) ? cached : cached,
|
|
redis_current: cached && cacheStillValid(cached) ? cached : cached,
|
|
|
last_fetch_log: lastFetch
|
|
last_fetch_log: lastFetch
|