Cloudflare detection, cache failed scrapes, cover art fallback logging

core/boxart_itch.gd

@@ -10,18 +10,13 @@ extends BoxArtProvider
 ## (the itch.io API does not expose them), caches the result in
 ## itch_art_meta.json, and uses HTTPImageFetcher for async image downloads.
 
-const BOXART_DIR := "user://boxart/itch"
-const META_CACHE_FILE := "itch_art_meta.json"
-const HTTP_TIMEOUT_MS := 15000
-const SCREENSHOT_REGEX_PATTERN := "https://img\\.itch\\.zone/aW1hZ2Uv[^\"]*/original/[^\"]*"
-const CONNECT_POLL_DELAY_MS := 50
-const READ_POLL_DELAY_MS := 10
-const LOG_URL_MAX_LEN := 80
+const _boxart_dir := "user://boxart/itch"
+const _meta_cache_file := "itch_art_meta.json"
 
 @export var use_caching: bool = true
 var http_image := HTTPImageFetcher.new()
-var _meta_cache_dir := "itch_art"
-var _enriched_meta: Dictionary = {}
+var _cache_dir := "itch_art"
+var _enriched: Dictionary = {}
 
 var layout_map: Dictionary = {
 	LAYOUT.GRID_PORTRAIT: "-portrait",
@@ -33,14 +28,14 @@ var layout_map: Dictionary = {
 
 func _init() -> void:
 	super()
-	DirAccess.make_dir_recursive_absolute(ProjectSettings.globalize_path(BOXART_DIR))
+	DirAccess.make_dir_recursive_absolute(ProjectSettings.globalize_path(_boxart_dir))
 	provider_id = "itch"
 	logger_name = "BoxArtItch"
 
 
 func _ready() -> void:
 	super()
-	_enriched_meta = _load_enriched_meta()
+	_enriched = _load_enriched_meta()
 	logger.info("itch.io Art Provider loaded")
 	add_child(http_image)
 
@@ -50,7 +45,7 @@ func get_boxart(item: LibraryItem, kind: LAYOUT) -> Texture2D:
 		logger.error("Unsupported boxart layout: {0}".format([kind]))
 		return null
 
-	var game := _find_game_dict(item)
+	var game := _itch_game(item)
 	if game.is_empty():
 		logger.warn("No itch.io game metadata found for: " + item.name)
 		return null
@@ -63,15 +58,16 @@ func get_boxart(item: LibraryItem, kind: LAYOUT) -> Texture2D:
 		return null
 
 	# Enrich on first access: scrape the itch.io page for screenshots.
-	var meta: Dictionary = _enriched_meta.get(game_id, {})
+	var meta: Dictionary = _enriched.get(game_id, {})
 	if meta.is_empty() and game.has("url"):
 		logger.info("Enriching metadata for: %s (url=%s)" % [title, game.get("url", "")])
 		meta = _fetch_game_page_metadata(game)
+		# Cache even empty results to avoid re-scraping on every request.
 		if meta.is_empty():
-			logger.warn("Page scrape returned no metadata for: %s" % title)
-		else:
-			_enriched_meta[game_id] = meta
-			_save_enriched_meta(game_id, meta)
+			meta = {"screenshots": []}
+			logger.info("Page scrape returned no screenshots for: %s (cached as empty)" % title)
+		_enriched[game_id] = meta
+		_save_enriched_meta(game_id, meta)
 	elif not meta.is_empty():
 		logger.info("Using cached metadata for: %s (%d screenshots)" % [title, meta.get("screenshots", []).size()])
 	else:
@@ -89,7 +85,7 @@ func get_boxart(item: LibraryItem, kind: LAYOUT) -> Texture2D:
 	if cover_url.is_empty():
 		logger.warn("No cover URL for: %s (id=%d)" % [title, game_id])
 		return null
-	logger.info("cover_url=%s" % cover_url.left(LOG_URL_MAX_LEN))
+	logger.info("cover_url=%s" % cover_url.left(80))
 
 	var cache_flags := Cache.FLAGS.NONE
 	if use_caching:
@@ -102,7 +98,7 @@ func get_boxart(item: LibraryItem, kind: LAYOUT) -> Texture2D:
 		var tex_b := await http_image.fetch(non_gif_screenshots[1], cache_flags)
 		if tex_a != null and tex_b != null:
 			return _combine_side_by_side(tex_a, tex_b)
-		# Banner combine failed — fall back to single-image path.
+		# Fall through to single-image path if either failed.
 		logger.warn("Banner screenshot fetch failed: tex_a=%s tex_b=%s" % [str(tex_a != null), str(tex_b != null)])
 		if tex_a != null:
 			return tex_a
@@ -114,15 +110,15 @@ func get_boxart(item: LibraryItem, kind: LAYOUT) -> Texture2D:
 		logger.warn("URL for layout %s is empty (cover=%s)" % [str(kind), cover_url])
 		return null
 
-	logger.info("Fetching itch.io box art for: %s layout=%s url=%s" % [item.name, str(kind), url.left(LOG_URL_MAX_LEN)])
+	logger.info("Fetching itch.io box art for: %s layout=%s url=%s" % [item.name, str(kind), url.left(80)])
 	var texture: Texture2D = await http_image.fetch(url, cache_flags)
 	if texture == null:
-		logger.warn("Image download returned null for: %s url=%s" % [item.name, url.left(LOG_URL_MAX_LEN)])
+		logger.warn("Image download returned null for: %s url=%s" % [item.name, url.left(80)])
 	return texture
 
 
 ## Finds the itch.io game dict from the library item's launch items.
-func _find_game_dict(item: LibraryItem) -> Dictionary:
+func _itch_game(item: LibraryItem) -> Dictionary:
 	if item.launch_items.is_empty():
 		logger.warn("Item has no launch items: " + item.name)
 		return {}
@@ -159,6 +155,17 @@ func _url_for_layout(kind: LAYOUT, cover_url: String, screenshots: Array) -> Str
 	return ""
 
 
+func _cover_url(game: Dictionary) -> String:
+	var still: String = game.get("stillCoverUrl", "")
+	if not still.is_empty() and not still.to_lower().ends_with(".gif"):
+		return still
+	# stillCoverUrl is empty or is a GIF — prefer it if non-GIF alternatives
+	# aren't available, but log a warning.
+	if not still.is_empty():
+		logger.debug("stillCoverUrl is a GIF, will try alternatives: " + still)
+	return game.get("coverUrl", "")
+
+
 ## Returns a non-GIF URL suitable for portrait/logo from the game data.
 ## Falls back through: stillCoverUrl → coverUrl (non-GIF) → first non-GIF
 ## screenshot.
@@ -181,11 +188,11 @@ func _best_portrait_url(game: Dictionary, screenshots: Array) -> String:
 
 ## Combines two textures side-by-side into a single banner image.
 ## Both images are scaled to the same height, then placed left and right.
-func _combine_side_by_side(left: Texture2D, right: Texture2D) -> Texture2D:
-	var img_a := left.get_image()
-	var img_b := right.get_image()
+func _combine_side_by_side(a: Texture2D, b: Texture2D) -> Texture2D:
+	var img_a := a.get_image()
+	var img_b := b.get_image()
 	if img_a == null or img_b == null:
-		return left
+		return a
 
 	# Scale both to the same height (use the taller one).
 	var target_h: int = max(img_a.get_height(), img_b.get_height())
@@ -219,28 +226,28 @@ func _fetch_game_page_metadata(game: Dictionary) -> Dictionary:
 	if err != OK:
 		logger.warn("HTTP connect failed for %s: %d" % [page_url, err])
 		return {}
-	var deadline := Time.get_ticks_msec() + HTTP_TIMEOUT_MS
+	var deadline := Time.get_ticks_msec() + 15000
 	while http.get_status() == HTTPClient.STATUS_CONNECTING or http.get_status() == HTTPClient.STATUS_RESOLVING:
 		http.poll()
 		if Time.get_ticks_msec() > deadline:
 			http.close()
 			logger.warn("HTTP connect timed out for: " + page_url)
 			return {}
-		OS.delay_msec(CONNECT_POLL_DELAY_MS)
+		OS.delay_msec(50)
 	if http.get_status() != HTTPClient.STATUS_CONNECTED:
 		logger.warn("HTTP not connected for %s: status=%d" % [page_url, http.get_status()])
 		http.close()
 		return {}
 
 	http.request(HTTPClient.METHOD_GET, "/" + "/".join(parts.slice(3)), PackedStringArray())
-	deadline = Time.get_ticks_msec() + HTTP_TIMEOUT_MS
+	deadline = Time.get_ticks_msec() + 15000
 	while http.get_status() == HTTPClient.STATUS_REQUESTING:
 		http.poll()
 		if Time.get_ticks_msec() > deadline:
 			http.close()
 			logger.warn("HTTP request timed out for: " + page_url)
 			return {}
-		OS.delay_msec(READ_POLL_DELAY_MS)
+		OS.delay_msec(10)
 
 	var body := PackedByteArray()
 	while http.get_status() == HTTPClient.STATUS_BODY:
@@ -251,7 +258,7 @@ func _fetch_game_page_metadata(game: Dictionary) -> Dictionary:
 				http.close()
 				logger.warn("HTTP body read timed out for: " + page_url)
 				return {}
-			OS.delay_msec(READ_POLL_DELAY_MS)
+			OS.delay_msec(10)
 			continue
 		body.append_array(chunk)
 	http.close()
@@ -261,8 +268,12 @@ func _fetch_game_page_metadata(game: Dictionary) -> Dictionary:
 		logger.warn("Empty HTML response for: " + page_url)
 		return {}
 
+	if html.find("challenge-platform") != -1 or html.find("cf-browser-verification") != -1:
+		logger.warn("Cloudflare challenge detected for: " + page_url)
+		return {}
+
 	var shot_re := RegEx.new()
-	shot_re.compile(SCREENSHOT_REGEX_PATTERN)
+	shot_re.compile("https://img\\.itch\\.zone/aW1hZ2Uv[^\"]*/original/[^\"]*")
 	var screenshots: Array = []
 	for m in shot_re.search_all(html):
 		var url: String = m.get_string(0)
@@ -276,7 +287,7 @@ func _fetch_game_page_metadata(game: Dictionary) -> Dictionary:
 
 
 func _load_enriched_meta() -> Dictionary:
-	var raw: Variant = Cache.get_json(_meta_cache_dir, META_CACHE_FILE)
+	var raw: Variant = Cache.get_json(_cache_dir, _meta_cache_file)
 	if typeof(raw) != TYPE_DICTIONARY:
 		return {}
 	var out := {}
@@ -286,8 +297,8 @@ func _load_enriched_meta() -> Dictionary:
 
 
 func _save_enriched_meta(game_id: int, meta: Dictionary) -> void:
-	var raw: Variant = Cache.get_json(_meta_cache_dir, META_CACHE_FILE)
+	var raw: Variant = Cache.get_json(_cache_dir, _meta_cache_file)
 	if typeof(raw) != TYPE_DICTIONARY:
 		raw = {}
 	raw[str(game_id)] = meta
-	Cache.save_json(_meta_cache_dir, META_CACHE_FILE, raw)
+	Cache.save_json(_cache_dir, _meta_cache_file, raw)

plugin.gd

@@ -10,6 +10,6 @@ var icon := preload("res://plugins/itch-artprovider/assets/itch.svg")
 
 func _ready() -> void:
 	logger = Log.get_logger("ItchArt", Log.LEVEL.INFO)
-	var boxart_provider: BoxArtProvider = load("res://plugins/itch-artprovider/core/boxart_itch.tscn").instantiate()
-	add_child(boxart_provider)
+	var boxart: BoxArtProvider = load("res://plugins/itch-artprovider/core/boxart_itch.tscn").instantiate()
+	add_child(boxart)
 	logger.info("itch.io Art Provider loaded")