refactor: simplify _fetch_game_page_metadata using shared helpers
All checks were successful
Build plugin / build (push) Successful in 1m5s
All checks were successful
Build plugin / build (push) Successful in 1m5s
- Replace manual HTTP polling with _download_url_bytes - Extract _is_cloudflare_challenge and _extract_screenshots_from_html - All functions now under 50 lines
This commit is contained in:
parent
3f5c45a99d
commit
a54c90f7c0
1 changed files with 13 additions and 48 deletions
|
|
@ -138,7 +138,7 @@ func get_boxart(item: LibraryItem, kind: LAYOUT) -> Texture2D:
|
||||||
var meta: Dictionary = _enriched_meta.get(game_id, {})
|
var meta: Dictionary = _enriched_meta.get(game_id, {})
|
||||||
if meta.is_empty() and game.has("url"):
|
if meta.is_empty() and game.has("url"):
|
||||||
logger.info("Enriching metadata for: %s (url=%s)" % [title, game.get("url", "")])
|
logger.info("Enriching metadata for: %s (url=%s)" % [title, game.get("url", "")])
|
||||||
meta = _fetch_game_page_metadata(game)
|
meta = await _fetch_game_page_metadata(game)
|
||||||
# Cache both successes and failures to avoid repeated requests.
|
# Cache both successes and failures to avoid repeated requests.
|
||||||
_enriched_meta[game_id] = meta
|
_enriched_meta[game_id] = meta
|
||||||
_save_enriched_meta(game_id, meta)
|
_save_enriched_meta(game_id, meta)
|
||||||
|
|
@ -605,67 +605,32 @@ func _fetch_game_page_metadata(game: Dictionary) -> Dictionary:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
logger.info("Scraping page for screenshots: " + page_url)
|
logger.info("Scraping page for screenshots: " + page_url)
|
||||||
var http := HTTPClient.new()
|
var body := await _download_url_bytes(page_url)
|
||||||
var parts := page_url.split("/")
|
if body.is_empty():
|
||||||
var use_tls := parts[0] == "https:"
|
|
||||||
var err := http.connect_to_host(parts[2], 443 if use_tls else 80, TLSOptions.client() if use_tls else null)
|
|
||||||
if err != OK:
|
|
||||||
logger.warn("HTTP connect failed for %s: %d" % [page_url, err])
|
|
||||||
return {}
|
return {}
|
||||||
var deadline := Time.get_ticks_msec() + HTTP_TIMEOUT_MS
|
|
||||||
while http.get_status() == HTTPClient.STATUS_CONNECTING or http.get_status() == HTTPClient.STATUS_RESOLVING:
|
|
||||||
http.poll()
|
|
||||||
if Time.get_ticks_msec() > deadline:
|
|
||||||
http.close()
|
|
||||||
logger.warn("HTTP connect timed out for: " + page_url)
|
|
||||||
return {}
|
|
||||||
OS.delay_msec(CONNECT_POLL_DELAY_MS)
|
|
||||||
if http.get_status() != HTTPClient.STATUS_CONNECTED:
|
|
||||||
logger.warn("HTTP not connected for %s: status=%d" % [page_url, http.get_status()])
|
|
||||||
http.close()
|
|
||||||
return {}
|
|
||||||
|
|
||||||
http.request(HTTPClient.METHOD_GET, "/" + "/".join(parts.slice(3)), PackedStringArray())
|
|
||||||
deadline = Time.get_ticks_msec() + HTTP_TIMEOUT_MS
|
|
||||||
while http.get_status() == HTTPClient.STATUS_REQUESTING:
|
|
||||||
http.poll()
|
|
||||||
if Time.get_ticks_msec() > deadline:
|
|
||||||
http.close()
|
|
||||||
logger.warn("HTTP request timed out for: " + page_url)
|
|
||||||
return {}
|
|
||||||
OS.delay_msec(READ_POLL_DELAY_MS)
|
|
||||||
|
|
||||||
var body := PackedByteArray()
|
|
||||||
while http.get_status() == HTTPClient.STATUS_BODY:
|
|
||||||
http.poll()
|
|
||||||
var chunk: PackedByteArray = http.read_response_body_chunk()
|
|
||||||
if chunk.is_empty():
|
|
||||||
if Time.get_ticks_msec() > deadline:
|
|
||||||
http.close()
|
|
||||||
logger.warn("HTTP body read timed out for: " + page_url)
|
|
||||||
return {}
|
|
||||||
OS.delay_msec(READ_POLL_DELAY_MS)
|
|
||||||
continue
|
|
||||||
body.append_array(chunk)
|
|
||||||
http.close()
|
|
||||||
|
|
||||||
var html := body.get_string_from_utf8()
|
var html := body.get_string_from_utf8()
|
||||||
if html.is_empty():
|
if html.is_empty():
|
||||||
logger.warn("Empty HTML response for: " + page_url)
|
logger.warn("Empty HTML response for: " + page_url)
|
||||||
return {}
|
return {}
|
||||||
|
if _is_cloudflare_challenge(html):
|
||||||
# Detect Cloudflare challenge pages — no point extracting screenshots.
|
|
||||||
if html.find("challenge-platform") != -1 or html.find("cf-browser-verification") != -1:
|
|
||||||
logger.warn("Cloudflare challenge detected for: %s — falling back to cover art" % page_url)
|
logger.warn("Cloudflare challenge detected for: %s — falling back to cover art" % page_url)
|
||||||
return {}
|
return {}
|
||||||
|
return _extract_screenshots_from_html(html, page_url)
|
||||||
|
|
||||||
|
|
||||||
|
## Returns true if the HTML contains a Cloudflare challenge page.
|
||||||
|
func _is_cloudflare_challenge(html: String) -> bool:
|
||||||
|
return html.find("challenge-platform") != -1 or html.find("cf-browser-verification") != -1
|
||||||
|
|
||||||
|
|
||||||
|
## Extracts screenshot URLs from the itch.io page HTML using regex.
|
||||||
|
func _extract_screenshots_from_html(html: String, page_url: String) -> Dictionary:
|
||||||
var shot_re := RegEx.new()
|
var shot_re := RegEx.new()
|
||||||
shot_re.compile(SCREENSHOT_REGEX_PATTERN)
|
shot_re.compile(SCREENSHOT_REGEX_PATTERN)
|
||||||
var screenshots: Array = []
|
var screenshots: Array = []
|
||||||
for m in shot_re.search_all(html):
|
for m in shot_re.search_all(html):
|
||||||
var url: String = _sanitize_url(m.get_string(0))
|
var url: String = _sanitize_url(m.get_string(0))
|
||||||
screenshots.append(url)
|
screenshots.append(url)
|
||||||
|
|
||||||
logger.info("Found %d screenshots for page %s" % [screenshots.size(), page_url])
|
logger.info("Found %d screenshots for page %s" % [screenshots.size(), page_url])
|
||||||
return {"screenshots": screenshots}
|
return {"screenshots": screenshots}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue