mtg_scanner_tool.py

import time, re, os, pickle, requests, yaml
from urllib.parse import unquote, quote
from collections import Counter
from playwright.sync_api import sync_playwright


class MTGDeckScanner:
    def __init__(self, cache_dir=None, config_filename="mtg_scanner_config.yaml", min_percentage=0.20):
        # Configuration
        base_cache = os.getenv("CACHE_ROOT", os.path.abspath(os.path.dirname(__file__)))
        self.cache_dir = cache_dir or os.path.join(base_cache, "mtg_scanner_cache")
        self.min_percentage = min_percentage

        os.makedirs(self.cache_dir, exist_ok=True)

        self.config_file = os.path.join(self.cache_dir, config_filename)
        self.urls_cache_file = os.path.join(self.cache_dir, "topdeck_urls_cache.pkl")
        self.scryfall_cache_file = os.path.join(self.cache_dir, "scryfall_data.pkl")
        self.deck_cache_file = os.path.join(self.cache_dir, "topdeck_deck_data.pkl")
        self.excluded_file = os.path.join(self.cache_dir, "excluded_cards.txt")

        self.type_order = [
            "Commander", "Creature", "Artifact", "Enchantment",
            "Instant", "Sorcery", "Planeswalker", "Land"
        ]
        self.type_hierarchy = (
            "Land", "Creature", "Artifact", "Enchantment",
            "Instant", "Sorcery", "Planeswalker"
        )

        self.ui_noise = [
            "Artifact", "Creature", "Instant", "Sorcery",
            "Enchantment", "Land", "Planeswalker"
        ]

        self.commander_targets = self._load_config_urls()
        self.excluded_cards = self._load_exclusions()
        self.game_changers = self._fetch_game_changers()
        self.topdeck_cache = self._load_cache(self.urls_cache_file)
        self.deck_cache = self._load_cache(self.deck_cache_file)
        self.scryfall_cache = self._load_cache(self.scryfall_cache_file)

    def _load_config_urls(self):
        if os.path.exists(self.config_file):
            try:
                with open(self.config_file, 'r') as f:
                    config_data = yaml.safe_load(f)
                    if config_data and "commander_urls" in config_data:
                        targets = []
                        for item in config_data["commander_urls"]:
                            if isinstance(item, dict):
                                targets.append(item)
                            else:
                                targets.append({"url": item, "bracket": None})
                        return targets
            except Exception as e:
                print(f"Error parsing configuration file {self.config_file}: {e}")

        print(f"Warning: {self.config_file} not found or missing 'commander_urls'. Using default empty list.")
        return []

    def _load_exclusions(self):
        exclusions = {}
        if os.path.exists(self.excluded_file):
            with open(self.excluded_file, 'r') as f:
                for line in f:
                    line = line.strip()
                    if not line:
                        continue
                    if '->' in line:
                        target, replacement = line.split('->', 1)
                        exclusions[target.strip().lower()] = replacement.strip()
                    else:
                        exclusions[line.lower()] = None
            return exclusions
        return exclusions

    def _fetch_game_changers(self):
        gc_cache_file = os.path.join(self.cache_dir, "scryfall_game_changers.pkl")

        if os.path.exists(gc_cache_file):
            if time.time() - os.path.getmtime(gc_cache_file) < 7 * 86400:
                return self._load_cache(gc_cache_file)

        print("Fetching latest Game Changers list from Scryfall API...")
        gc_list = set()
        url = "https://api.scryfall.com/cards/search?q=is:gamechanger"

        try:
            while url:
                time.sleep(0.1)
                response = requests.get(url, headers={"User-Agent": "EDH-Builder-Script/1.0"})
                if response.status_code == 200:
                    data = response.json()
                    for card in data.get("data", []):
                        gc_list.add(card["name"].lower())
                    if data.get("has_more"):
                        url = data.get("next_page")
                    else:
                        break
                else:
                    break

            if gc_list:
                self._save_cache(gc_list, gc_cache_file)
                return gc_list
        except Exception:
            pass

        if os.path.exists(gc_cache_file):
            return self._load_cache(gc_cache_file)
        return set()

    def _load_cache(self, filepath):
        if os.path.exists(filepath):
            with open(filepath, 'rb') as f:
                return pickle.load(f)
        return {}

    def _save_cache(self, cache_dict, filepath):
        with open(filepath, 'wb') as f:
            pickle.dump(cache_dict, f)

    def auto_scroll_to_bottom(self, page):
        last_height = page.evaluate("document.body.scrollHeight")
        while True:
            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            page.wait_for_timeout(2000)
            new_height = page.evaluate("document.body.scrollHeight")
            if new_height == last_height:
                break
            last_height = new_height

    def get_topdeck_urls(self, page, commander_url, force_refresh=False):
        if commander_url in self.topdeck_cache and not force_refresh:
            return self.topdeck_cache[commander_url]

        page.goto(commander_url, wait_until="networkidle")
        self.auto_scroll_to_bottom(page)

        html_content = page.content()
        td_regex = re.compile(r"topdeck\.gg/deck/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+")
        matches = td_regex.findall(html_content)
        urls = list(set([f"https://{match}" for match in matches]))

        self.topdeck_cache[commander_url] = urls
        self._save_cache(self.topdeck_cache, self.urls_cache_file)
        return urls

    def scrape_deck_data(self, page, url):
        if url in self.deck_cache:
            return self.deck_cache[url]

        try:
            page.goto(url, wait_until="networkidle")
            page.wait_for_timeout(1500)

            data = page.evaluate('''() => {
                let found = new Set();
                let basicCounts = {
                    mountain: 0, forest: 0, plains: 0, island: 0, swamp: 0, wastes: 0
                };
                const cleanName = (name) => name.replace(/^\\s*\\d+x?\\s+/i, '').trim();

                document.querySelectorAll('img[alt]').forEach(img => {
                    let name = img.getAttribute('alt');
                    if (!name || name.length < 2) return;

                    let lowerName = name.toLowerCase();
                    let basicTypes = [
                        "mountain", "forest", "plains", "island", "swamp", "wastes",
                        "snow-covered mountain", "snow-covered forest",
                        "snow-covered plains", "snow-covered island",
                        "snow-covered swamp", "snow-covered wastes"
                    ];

                    if (basicTypes.includes(lowerName)) {
                        let text = img.parentElement.innerText || "";
                        let match = text.match(/(\\d+)/);
                        let count = match ? parseInt(match[1]) : 1;

                        if (lowerName.includes("mountain")) basicCounts.mountain += count;
                        else if (lowerName.includes("forest")) basicCounts.forest += count;
                        else if (lowerName.includes("plains")) basicCounts.plains += count;
                        else if (lowerName.includes("island")) basicCounts.island += count;
                        else if (lowerName.includes("swamp")) basicCounts.swamp += count;
                        else if (lowerName.includes("wastes")) basicCounts.wastes += count;
                    } else {
                        found.add(cleanName(name));
                    }
                });
                return { cards: Array.from(found), basicCounts };
            }''')

            bad = [
                "topdeck", "logo", "avatar", "profile", "banner", "discord",
                "twitter", "match history", "standings", "deck", "event",
                "buy", "card image"
            ]

            clean_cards = [
                n for n in data['cards']
                if not any(b in n.lower() for b in bad)
                   and not re.match(r'^\d', n)
            ]

            result = (clean_cards, data['basicCounts'])
            self.deck_cache[url] = result
            self._save_cache(self.deck_cache, self.deck_cache_file)
            return result

        except Exception as e:
            print(f"Error reading {url}: {e}")
            return [], {"mountain": 0, "forest": 0, "plains": 0, "island": 0, "swamp": 0, "wastes": 0}

    def get_scryfall_data(self, card_name):
        if card_name in self.scryfall_cache:
            cached_data = self.scryfall_cache[card_name]
            if len(cached_data) == 3:
                real_name, type_line, is_legal = cached_data
                is_gc = real_name.lower() in self.game_changers or card_name.lower() in self.game_changers
                cached_data = (real_name, type_line, is_legal, is_gc)
                self.scryfall_cache[card_name] = cached_data
                self._save_cache(self.scryfall_cache, self.scryfall_cache_file)
            return cached_data

        time.sleep(0.1)
        url = f"https://api.scryfall.com/cards/named?exact={quote(card_name)}"
        try:
            response = requests.get(
                url, headers={"User-Agent": "EDH-Builder-Script/1.0"}
            )
            if response.status_code == 200:
                res_data = response.json()
                type_line = res_data.get("type_line", "")
                if "card_faces" in res_data and not type_line:
                    type_line = res_data["card_faces"][0].get("type_line", "")

                legalities = res_data.get("legalities", {})
                is_legal = legalities.get("commander") == "legal"

                if "Sticker" in type_line or "Attraction" in type_line:
                    is_legal = False

                real_name = res_data.get("name", card_name)
                is_gc = real_name.lower() in self.game_changers or card_name.lower() in self.game_changers

                result = (real_name, type_line, is_legal, is_gc)
                self.scryfall_cache[card_name] = result
                self._save_cache(self.scryfall_cache, self.scryfall_cache_file)
                return result
        except Exception:
            pass

        is_gc = card_name.lower() in self.game_changers
        result = (card_name, "", True, is_gc)
        self.scryfall_cache[card_name] = result
        self._save_cache(self.scryfall_cache, self.scryfall_cache_file)
        return result

    def categorize_card(self, type_line):
        if not type_line:
            return "Creature"
        return next(
            (t for t in self.type_hierarchy if t.lower() in type_line.lower()),
            "Artifact"
        )

    def analyze_commander(self, page, commander_url, bracket=None):
        name_match = re.search(r"commander/([^?]+)", commander_url)
        commander_name = (
            unquote(name_match.group(1)) if name_match else "Unknown Commander"
        )

        bracket_text = f" (Bracket {bracket})" if bracket else ""
        print(f"\n{'=' * 60}\nGathering Consensus Data for: {commander_name}{bracket_text}\n{'=' * 60}")

        urls = self.get_topdeck_urls(page, commander_url)
        if not urls:
            print("No lists found!")
            return

        print(f"Scraping {len(urls)} tournament decklists...")

        raw_card_counter = Counter()
        card_counter = Counter()
        total_basics = {
            "mountain": 0, "forest": 0, "plains": 0, "island": 0, "swamp": 0, "wastes": 0
        }
        valid_deck_count = 0

        for i, url in enumerate(urls, 1):
            print(f"Processing decklist {i}/{len(urls)}...", end="\r")
            cards, b_counts = self.scrape_deck_data(page, url)

            if not cards and sum(b_counts.values()) == 0:
                continue

            valid_deck_count += 1
            raw_card_counter.update(cards)

            processed_cards = set()
            for c in cards:
                c_lower = c.lower()
                if c_lower in self.excluded_cards:
                    replacement = self.excluded_cards[c_lower]
                    if replacement:
                        processed_cards.add(replacement)
                else:
                    processed_cards.add(c)

            card_counter.update(processed_cards)

            for k in total_basics:
                total_basics[k] += b_counts.get(k, 0)

        if valid_deck_count == 0:
            print(f"\nNo valid decklists could be processed for {commander_name}.")
            return

        total_lands_all_decks = sum(total_basics.values())
        total_instants_all_decks = 0

        total_fetchable_targets = {
            "Plains": 0, "Island": 0, "Swamp": 0, "Mountain": 0, "Forest": 0
        }

        for land, total in total_basics.items():
            if land.capitalize() in total_fetchable_targets:
                total_fetchable_targets[land.capitalize()] += total

        raw_unique_cards = [
            card for card in raw_card_counter.keys()
            if card not in self.ui_noise and card.lower() != commander_name.lower()
        ]

        print(f"\nAnalyzing {len(raw_unique_cards)} unique cards for averages...")
        for idx, card in enumerate(raw_unique_cards, 1):
            print(f"Checking Scryfall typelines {idx}/{len(raw_unique_cards)}...", end="\r")
            scry_data = self.get_scryfall_data(card)
            type_line, is_legal = scry_data[1], scry_data[2]

            if not is_legal:
                continue

            cat = self.categorize_card(type_line)
            if cat == "Land":
                total_lands_all_decks += raw_card_counter[card]

                for land_type in total_fetchable_targets.keys():
                    if re.search(rf'\b{land_type}\b', type_line, re.IGNORECASE):
                        total_fetchable_targets[land_type] += raw_card_counter[card]

            elif cat == "Instant":
                total_instants_all_decks += raw_card_counter[card]

        print(" " * 80, end="\r")

        dynamic_min_lands = round(total_lands_all_decks / valid_deck_count)
        dynamic_min_instants = round(total_instants_all_decks / valid_deck_count)
        dynamic_max_instants = dynamic_min_instants + 2

        print(
            f"Targeting: {dynamic_min_lands} Lands | "
            f"{dynamic_min_instants}-{dynamic_max_instants} Instants"
        )

        avg_basics = {}
        total_avg_basics = 0
        for land, total in total_basics.items():
            avg = round(total / valid_deck_count)
            if avg > 0:
                avg_basics[land.capitalize()] = avg
                total_avg_basics += avg

        avg_fetchable = {}
        for land_type, total in total_fetchable_targets.items():
            avg = round(total / valid_deck_count)
            if avg > 0:
                avg_fetchable[land_type] = avg

        print(f"Targeting Average Fetchable Subtypes: {avg_fetchable}")

        if str(bracket) in ['1', '2']:
            max_gc = 0
        elif str(bracket) == '3':
            max_gc = 3
        else:
            max_gc = 100

        target_total_lands = dynamic_min_lands
        target_non_basics = max(0, target_total_lands - total_avg_basics)
        target_spells = 99 - target_total_lands

        high_consensus_pool = []
        gc_count = 0
        added_non_basics = 0
        added_spells = 0

        for card, count in card_counter.most_common():
            if (card in self.ui_noise
                    or card.lower() == commander_name.lower()
                    or card.lower() in self.excluded_cards):
                continue

            if (count / valid_deck_count) >= self.min_percentage:
                scry_data = self.get_scryfall_data(card)
                is_legal, is_gc = scry_data[2], scry_data[3]

                if is_legal:
                    if is_gc and gc_count >= max_gc:
                        print(f"   -> Bracket {bracket} Game Changer cap reached. Excluded '{card}'.")
                        continue

                    cat = self.categorize_card(scry_data[1])

                    if cat == "Land":
                        if added_non_basics < target_non_basics:
                            high_consensus_pool.append(card)
                            added_non_basics += 1
                            if is_gc: gc_count += 1
                    else:
                        if added_spells < target_spells:
                            high_consensus_pool.append(card)
                            added_spells += 1
                            if is_gc: gc_count += 1

            if added_non_basics >= target_non_basics and added_spells >= target_spells:
                break

        deck_list = {t: [] for t in self.type_order}
        deck_list["Commander"] = [f"1 {commander_name}"]

        current_count = 1
        for idx, card in enumerate(high_consensus_pool, 1):
            scry_data = self.get_scryfall_data(card)
            real_name, type_line = scry_data[0], scry_data[1]
            category = self.categorize_card(type_line)

            # Add an asterisk marker for Game Changers when analyzing Bracket 3 decks
            display_name = real_name
            if str(bracket) == '3' and scry_data[3]:
                display_name = f"{real_name} *"

            deck_list[category].append(f"1 {display_name}")
            current_count += 1

        current_instants = len(deck_list["Instant"])
        if current_instants > dynamic_max_instants:
            to_cut = current_instants - dynamic_max_instants
            cards_removed = 0
            for card in reversed(high_consensus_pool):
                if cards_removed >= to_cut:
                    break

                matched_item = next(
                    (item for item in deck_list["Instant"]
                     if item.startswith("1 ") and item[2:].rstrip(" *") == card), None
                )
                if matched_item:
                    deck_list["Instant"].remove(matched_item)
                    current_count -= 1
                    cards_removed += 1
                    print(f"   -> Cut '{card}' (Instant) to meet maximum cap of {dynamic_max_instants}.")

            refill_needed = cards_removed
            for card, count in card_counter.most_common():
                if refill_needed == 0: break
                if card in self.ui_noise or card.lower() == commander_name.lower() or card.lower() in self.excluded_cards:
                    continue
                if card not in high_consensus_pool:
                    scry_data = self.get_scryfall_data(card)
                    cat = self.categorize_card(scry_data[1])
                    if scry_data[2] and cat not in ["Land", "Instant"]:
                        if scry_data[3] and gc_count >= max_gc: continue

                        display_name = scry_data[0]
                        if str(bracket) == '3' and scry_data[3]:
                            display_name = f"{scry_data[0]} *"

                        deck_list[cat].append(f"1 {display_name}")
                        high_consensus_pool.append(card)
                        current_count += 1
                        refill_needed -= 1
                        if scry_data[3]: gc_count += 1

        current_instants = len(deck_list["Instant"])
        if current_instants < dynamic_min_instants:
            instant_deficit = dynamic_min_instants - current_instants
            next_best_instants = []
            for card, count in card_counter.most_common():
                if (card in self.ui_noise
                        or card.lower() == commander_name.lower()
                        or card.lower() in self.excluded_cards):
                    continue
                if card not in high_consensus_pool:
                    scry_data = self.get_scryfall_data(card)
                    type_line, is_legal = scry_data[1], scry_data[2]
                    if is_legal and self.categorize_card(type_line) == "Instant":
                        next_best_instants.append(card)
                        if len(next_best_instants) == instant_deficit:
                            break

            instants_to_add = len(next_best_instants)
            cards_removed = 0
            for card in reversed(high_consensus_pool):
                if cards_removed >= instants_to_add:
                    break

                for category in self.type_order:
                    if category in ["Land", "Commander", "Instant"]:
                        continue

                    matched_item = next(
                        (item for item in deck_list[category]
                         if item.startswith("1 ") and item[2:].rstrip(" *") == card), None
                    )
                    if matched_item:
                        deck_list[category].remove(matched_item)
                        current_count -= 1
                        cards_removed += 1
                        print(f"   -> Cut '{card}' for required average Instants.")
                        break

            for card in next_best_instants:
                scry_data = self.get_scryfall_data(card)
                real_name = scry_data[0]

                display_name = real_name
                if str(bracket) == '3' and scry_data[3]:
                    display_name = f"{real_name} *"

                deck_list["Instant"].append(f"1 {display_name}")
                current_count += 1
                print(f"   -> Added '{card}' (Instant) to meet dynamic average.")

        current_lands = len(deck_list["Land"])
        slots_remaining = 100 - current_count
        if (current_lands + slots_remaining) < dynamic_min_lands:
            deficit = dynamic_min_lands - (current_lands + slots_remaining)
            cards_removed = 0
            for card in reversed(high_consensus_pool):
                if cards_removed >= deficit:
                    break
                for category in self.type_order:
                    if category in ["Land", "Commander", "Instant"]:
                        continue
                    matched_item = next(
                        (item for item in deck_list[category]
                         if item.startswith("1 ") and item[2:].rstrip(" *") == card), None
                    )
                    if matched_item:
                        deck_list[category].remove(matched_item)
                        current_count -= 1
                        slots_remaining += 1
                        cards_removed += 1
                        print(f"   -> Cut '{card}' for required average Lands.")
                        break

        # =====================================================================
        # MOVED UP: BASIC LAND INJECTION
        # (Inject basics FIRST so the safety net can accurately see they exist)
        # =====================================================================
        sorted_basics = sorted(
            total_basics.items(), key=lambda x: x[1], reverse=True
        )
        basic_types_to_use = [k.capitalize() for k, v in sorted_basics if v > 0]

        if not basic_types_to_use:
            basic_types_to_use = ["Forest", "Island", "Swamp", "Mountain", "Plains", "Wastes"]

        for basic_name in basic_types_to_use:
            if avg_basics.get(basic_name, 0) > 0 and slots_remaining > 0:
                allocated = min(avg_basics[basic_name], slots_remaining)

                found_existing = False
                for i, entry in enumerate(deck_list["Land"]):
                    if entry.endswith(f" {basic_name}"):
                        old_count = int(entry.split(' ', 1)[0])
                        deck_list["Land"][i] = f"{old_count + allocated} {basic_name}"
                        found_existing = True
                        break
                if not found_existing:
                    deck_list["Land"].append(f"{allocated} {basic_name}")

                slots_remaining -= allocated

        if slots_remaining > 0:
            distribution = {b: 0 for b in basic_types_to_use}
            idx = 0
            while slots_remaining > 0:
                distribution[basic_types_to_use[idx % len(basic_types_to_use)]] += 1
                slots_remaining -= 1
                idx += 1

            for basic_name, count in distribution.items():
                if count > 0:
                    found = False
                    for i, entry in enumerate(deck_list["Land"]):
                        if entry.endswith(f" {basic_name}"):
                            old_count = int(entry.split(' ', 1)[0])
                            deck_list["Land"][i] = f"{old_count + count} {basic_name}"
                            found = True
                            break
                    if not found:
                        deck_list["Land"].append(f"{count} {basic_name}")


        # =====================================================================
        # FETCHABLE TARGET SAFETY NET
        # =====================================================================
        current_fetchables = {"Plains": 0, "Island": 0, "Swamp": 0, "Mountain": 0, "Forest": 0}
        fetch_land_names = {
            "arid mesa", "bloodstained mire", "flooded strand", "marsh flats",
            "misty rainforest", "polluted delta", "scalding tarn", "verdant catacombs",
            "windswept heath", "wooded foothills"
        }

        for land_entry in deck_list["Land"]:
            parts = land_entry.split(' ', 1)
            count = int(parts[0])
            land_name = parts[1]

            scry_data = self.get_scryfall_data(land_name)
            type_line = scry_data[1]

            for ft in current_fetchables.keys():
                if re.search(rf'\b{ft}\b', type_line, re.IGNORECASE):
                    current_fetchables[ft] += count

        deficits = {k: v - current_fetchables.get(k, 0) for k, v in avg_fetchable.items() if v > current_fetchables.get(k, 0)}

        if deficits:
            replaceable_lands = []
            for land_entry in deck_list["Land"]:
                parts = land_entry.split(' ', 1)
                count = int(parts[0])
                land_name = parts[1]
                scry_data = self.get_scryfall_data(land_name)
                type_line = scry_data[1]

                is_fetch = land_name.lower() in fetch_land_names
                has_subtype = any(re.search(rf'\b{ft}\b', type_line, re.IGNORECASE) for ft in current_fetchables.keys())

                if not is_fetch and not has_subtype:
                    for _ in range(count):
                        replaceable_lands.append(land_entry)

            replaceable_lands.reverse()

            for ft, def_val in list(deficits.items()):
                while def_val > 0 and replaceable_lands:
                    to_cut = replaceable_lands.pop(0)
                    for i, entry in enumerate(deck_list["Land"]):
                        if entry == to_cut:
                            old_count = int(entry.split(' ', 1)[0])
                            if old_count > 1:
                                deck_list["Land"][i] = f"{old_count - 1} {to_cut.split(' ', 1)[1]}"
                            else:
                                deck_list["Land"].pop(i)
                            break

                    basic_added = False
                    for i, entry in enumerate(deck_list["Land"]):
                        if entry.endswith(f" {ft}"):
                            old_count = int(entry.split(' ', 1)[0])
                            deck_list["Land"][i] = f"{old_count + 1} {ft}"
                            basic_added = True
                            break
                    if not basic_added:
                        deck_list["Land"].append(f"1 {ft}")

                    print(f"   -> Cut '{to_cut.split(' ', 1)[1]}' (Utility Land) for Basic '{ft}' to meet fetchable subtype targets.")
                    def_val -= 1


        # =====================================================================
        # FINAL CONSOLIDATION PASS
        # =====================================================================
        final_land_counts = Counter()
        for land_entry in deck_list["Land"]:
            parts = land_entry.split(' ', 1)
            count = int(parts[0])
            land_name = parts[1]
            final_land_counts[land_name] += count

        deck_list["Land"] = [f"{count} {name}" for name, count in final_land_counts.items()]

        print(f"\n\n### {commander_name} - Meta Optimized Decklist")
        for category in self.type_order:
            cards_in_cat = deck_list[category]
            if cards_in_cat:
                cat_total = sum(int(c.split(' ', 1)[0]) for c in cards_in_cat)
                print(f"\n### {category} ({cat_total})")
                for card_entry in sorted(cards_in_cat, key=lambda x: x.split(' ', 1)[1]):
                    print(card_entry)

    def run(self):
        if not self.commander_targets:
            print(f"No URLs configured to scrape. Ensure your configuration file exists at: {self.config_file}")
            return

        with sync_playwright() as p:
            browser = p.chromium.launch(headless=True)
            page = browser.new_page()

            for target in self.commander_targets:
                self.analyze_commander(page, target["url"], target.get("bracket"))

            browser.close()


if __name__ == "__main__":
    scanner = MTGDeckScanner()
    scanner.run()