package ai.intelliswarm.swarmai.process;

import ai.intelliswarm.swarmai.agent.Agent;
import ai.intelliswarm.swarmai.budget.BudgetSnapshot;
import ai.intelliswarm.swarmai.budget.BudgetTracker;
import ai.intelliswarm.swarmai.cache.ScanResultCache;
import ai.intelliswarm.swarmai.event.SwarmEvent;
import ai.intelliswarm.swarmai.memory.Memory;
import ai.intelliswarm.swarmai.skill.*;
import ai.intelliswarm.swarmai.swarm.SwarmOutput;
import ai.intelliswarm.swarmai.task.Task;
import ai.intelliswarm.swarmai.task.output.TaskOutput;
import ai.intelliswarm.swarmai.tool.base.BaseTool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationEventPublisher;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * Self-Improving Process — extends the iterative review loop with dynamic skill generation.
 *
 * The loop:
 * 1. Execute tasks with current tools
 * 2. Reviewer evaluates → ReviewResult with QUALITY_ISSUES and CAPABILITY_GAPS
 * 3. If CAPABILITY_GAPS found: generate new skills → validate → register → rebuild agents
 * 4. If only QUALITY_ISSUES: inject feedback (standard iterative behavior)
 * 5. Re-execute with expanded toolkit. Repeat until APPROVED or max iterations.
 */
public class SelfImprovingProcess implements Process {

    private static final Logger logger = LoggerFactory.getLogger(SelfImprovingProcess.class);

    /** Hard safety cap — never iterate more than this regardless of convergence. */
    /** Hard safety cap — never iterate more than this regardless of configuration. */
    private static final int ABSOLUTE_MAX_ITERATIONS = 20;

    private final List<Agent> originalAgents;
    private final Agent reviewerAgent;
    private final ApplicationEventPublisher eventPublisher;
    private final int maxIterations;
    private final String qualityCriteria;

    private static final Path DEFAULT_SKILLS_DIR = Paths.get("output", "skills");

    /** Similarity threshold for skill deduplication (0.0 - 1.0). */
    private static final double SKILL_SIMILARITY_THRESHOLD = 0.35;

    /** Maximum generated skills to load per agent to prevent context bloat. */
    private static final int MAX_SKILLS_PER_AGENT = 20;

    private final SkillRegistry skillRegistry;
    private final SkillValidator skillValidator;
    private final SkillGapAnalyzer gapAnalyzer;
    private final Memory memory;
    private final ai.intelliswarm.swarmai.rl.PolicyEngine policyEngine;
    private List<Agent> currentAgents;
    private int skillsReused = 0;
    private ScanResultCache scanCache;
    /**
     * Optional override for the skill-generation backend. When null, a default
     * {@link SkillGenerator} (LLM-driven) is constructed per iteration. Set to
     * a {@link ai.intelliswarm.swarmai.skill.CodingAgentSkillGenerator} to drive
     * skill creation through an external coding agent (Codex, Claude Code).
     */
    private ai.intelliswarm.swarmai.skill.SkillEngine skillEngineOverride;

    public SelfImprovingProcess(List<Agent> agents, Agent reviewerAgent,
                                 ApplicationEventPublisher eventPublisher,
                                 int maxIterations, String qualityCriteria) {
        this(agents, reviewerAgent, eventPublisher, maxIterations, qualityCriteria, (Memory) null);
    }

    public SelfImprovingProcess(List<Agent> agents, Agent reviewerAgent,
                                 ApplicationEventPublisher eventPublisher,
                                 int maxIterations, String qualityCriteria,
                                 Memory memory) {
        this(agents, reviewerAgent, eventPublisher, maxIterations, qualityCriteria, memory, null);
    }

    /**
     * Full constructor with PolicyEngine for RL-driven decision making.
     */
    public SelfImprovingProcess(List<Agent> agents, Agent reviewerAgent,
                                 ApplicationEventPublisher eventPublisher,
                                 int maxIterations, String qualityCriteria,
                                 Memory memory,
                                 ai.intelliswarm.swarmai.rl.PolicyEngine policyEngine) {
        this.originalAgents = new ArrayList<>(agents);
        this.reviewerAgent = reviewerAgent;
        this.eventPublisher = eventPublisher;
        this.maxIterations = maxIterations;
        this.qualityCriteria = qualityCriteria;
        this.memory = memory;
        this.policyEngine = policyEngine;
        this.skillRegistry = new SkillRegistry();
        this.skillValidator = new SkillValidator();
        this.gapAnalyzer = new SkillGapAnalyzer();
        this.currentAgents = new ArrayList<>(agents);

        // Load previously generated skills from disk
        loadPersistedSkills();
    }

    /**
     * Constructor that accepts a shared SkillRegistry instead of creating a new one.
     * Used by SwarmCoordinator to share skills across parallel sub-processes.
     * Does NOT load persisted skills — the shared registry is already populated.
     */
    public SelfImprovingProcess(List<Agent> agents, Agent reviewerAgent,
                                 ApplicationEventPublisher eventPublisher,
                                 int maxIterations, String qualityCriteria,
                                 SkillRegistry sharedRegistry) {
        this.originalAgents = new ArrayList<>(agents);
        this.currentAgents = new ArrayList<>(agents);
        this.reviewerAgent = reviewerAgent;
        this.eventPublisher = eventPublisher;
        this.maxIterations = maxIterations;
        this.qualityCriteria = qualityCriteria;
        this.skillRegistry = sharedRegistry;
        this.skillValidator = new SkillValidator();
        this.gapAnalyzer = new SkillGapAnalyzer();
        this.memory = null;
        this.policyEngine = null;
        // Do NOT call loadPersistedSkills() — shared registry is already populated
    }

    /**
     * Plug in an alternate skill-generation backend.
     *
     * <p>Defaults to {@code null} — meaning the process constructs a fresh
     * {@link SkillGenerator} (LLM-driven) per iteration. Pass a
     * {@link ai.intelliswarm.swarmai.skill.CodingAgentSkillGenerator} (or any
     * {@link ai.intelliswarm.swarmai.skill.SkillEngine}) to delegate skill
     * creation to an external coding agent like Codex or Claude Code.
     *
     * <p>Set this <em>before</em> calling {@code execute()}; the override is
     * read once at the start of each generation pass.
     *
     * @param engine the engine to use, or {@code null} to revert to default
     * @return this, for chaining
     */
    public SelfImprovingProcess setSkillEngine(ai.intelliswarm.swarmai.skill.SkillEngine engine) {
        this.skillEngineOverride = engine;
        return this;
    }

    /**
     * Load previously generated and validated skills from disk.
     * Skills are saved in output/skills/ as JSON files.
     * Wires up cross-skill references so composed skills can call each other.
     */
    private void loadPersistedSkills() {
        int loaded = skillRegistry.load(DEFAULT_SKILLS_DIR);
        if (loaded > 0) {
            logger.info("Loaded {} persisted skills (deduplicated to {})",
                loaded, skillRegistry.size());

            // Build complete tools map: agent tools + all loaded skills
            Map<String, BaseTool> allToolsMap = currentAgents.stream()
                .flatMap(a -> a.getTools().stream())
                .collect(Collectors.toMap(BaseTool::getFunctionName, t -> t, (a, b) -> a));

            // Add loaded skills to the tools map so they can reference each other
            for (GeneratedSkill skill : skillRegistry.getActiveSkills()) {
                allToolsMap.put(skill.getFunctionName(), skill);
            }

            // Inject the complete tools map into each skill and rebuild agents
            for (GeneratedSkill skill : skillRegistry.getActiveSkills()) {
                skill.setAvailableTools(allToolsMap);
                rebuildAgentsWithSkill(skill);
            }
        }
    }

    @Override
    public ProcessType getType() {
        return ProcessType.SELF_IMPROVING;
    }

    @Override
    public boolean isAsync() {
        return false;
    }

    @Override
    public void validateTasks(List<Task> tasks) {
        if (tasks == null || tasks.isEmpty()) {
            throw new IllegalArgumentException("At least one task is required");
        }
        if (reviewerAgent == null) {
            throw new IllegalArgumentException("Reviewer agent is required for self-improving process");
        }
    }

    @Override
    public SwarmOutput execute(List<Task> tasks, Map<String, Object> inputs, String swarmId) {
        validateTasks(tasks);

        // Load scan result cache
        scanCache = new ScanResultCache();
        scanCache.loadAll();
        String cachedScanContext = scanCache.toContextString();
        if (cachedScanContext != null) {
            logger.info("Loaded {} cached scan results", scanCache.getAll().size());
        }

        int effectiveMaxIterations = maxIterations > 0 ? Math.min(maxIterations, ABSOLUTE_MAX_ITERATIONS) : ABSOLUTE_MAX_ITERATIONS;
        logger.info("Self-Improving Process: Starting (max {} iterations, auto-stop on convergence)", effectiveMaxIterations);

        publishEvent(SwarmEvent.Type.PROCESS_STARTED,
            "Self-improving process execution started", swarmId, Map.of());

        // Order tasks by dependencies
        List<Task> orderedTasks = orderTasks(tasks);

        // Interpolate inputs
        for (Task task : orderedTasks) {
            if (inputs != null) {
                String interpolated = interpolateInputs(task.getDescription(), inputs);
                // Task descriptions are already set via builder
            }
        }

        List<TaskOutput> allOutputs = new ArrayList<>();
        List<String> commandLedger = new ArrayList<>();
        int iteration = 0;
        boolean approved = false;
        String reviewFeedback = null;
        String toolErrorSummary = null;
        int skillsGenerated = 0;

        // Convergence tracking (single-element arrays used as mutable carriers for extracted methods)
        int[] previousOutputLengthRef = {0};
        Set<String> previousGaps = new HashSet<>();
        int[] staleIterationsRef = {0};

        // Cross-iteration gap memory — prevents the same gap from being processed repeatedly
        List<String> processedGapDescriptions = new ArrayList<>();
        int[] gapsSkippedAsDuplicateRef = {0};

        while (iteration < effectiveMaxIterations && !approved) {
            if (Thread.currentThread().isInterrupted()) {
                logger.info("Self-Improving Process: Stopping — thread interrupted before iteration {}", iteration + 1);
                break;
            }

            iteration++;

            // Check budget-based stopping (leave 10% headroom for final output)
            BudgetTracker budgetTracker = inputs != null && inputs.get("__budgetTracker") instanceof BudgetTracker bt ? bt : null;
            String budgetSwarmId = inputs != null && inputs.get("__budgetSwarmId") instanceof String s ? s : swarmId;
            if (budgetTracker != null) {
                BudgetSnapshot snap = budgetTracker.getSnapshot(budgetSwarmId);
                if (snap != null && (snap.tokenUtilizationPercent() > 90.0 || snap.costUtilizationPercent() > 90.0)) {
                    logger.info("Self-Improving Process: Stopping — budget utilization at {}% tokens / {}% cost",
                        String.format("%.1f", snap.tokenUtilizationPercent()),
                        String.format("%.1f", snap.costUtilizationPercent()));
                    break;
                }
            }

            logger.info("Self-Improving Process: Iteration {}/{}", iteration, effectiveMaxIterations);
            publishEvent(SwarmEvent.Type.ITERATION_STARTED,
                "Iteration " + iteration + "/" + maxIterations + " started", swarmId,
                Map.of("iteration", iteration, "skillsAvailable", skillRegistry.size()));

            // Reset tasks for re-execution (except first iteration)
            if (iteration > 1) {
                for (Task task : orderedTasks) {
                    task.reset();
                }
                evolveTaskDescription(orderedTasks, reviewFeedback, toolErrorSummary, commandLedger);
            }

            // 1. Execute all tasks with current agents
            List<TaskOutput> contextOutputs = new ArrayList<>();
            if (iteration == 1 && cachedScanContext != null) {
                contextOutputs.add(TaskOutput.builder().taskId("scan-cache").rawOutput(cachedScanContext).build());
            }
            List<TaskOutput> iterationOutputs = executeIterationTasks(orderedTasks, inputs, contextOutputs, swarmId, iteration);
            allOutputs = new ArrayList<>(iterationOutputs);

            // Build command ledger and cache scan results
            commandLedger.addAll(extractExecutedCommands(iterationOutputs));
            if (scanCache != null) {
                cacheNewScanResults(iterationOutputs);
            }

            // 2a. Extract tool errors for self-adjustment feedback
            toolErrorSummary = extractToolErrors(iterationOutputs);
            if (toolErrorSummary != null) {
                logger.info("Iteration {}: Tool errors detected — will feed back for self-adjustment", iteration);
            }

            // 2b. Tool-evidence quality check and reviewer evaluation
            String outputToReview = iterationOutputs.stream()
                .map(TaskOutput::getRawOutput)
                .collect(Collectors.joining("\n\n---\n\n"));
            String toolEvidenceWarning = checkToolEvidence(outputToReview);
            if (toolEvidenceWarning != null) {
                logger.warn("Iteration {}: Tool evidence check: {}", iteration, toolEvidenceWarning);
                outputToReview = "TOOL USAGE WARNING: " + toolEvidenceWarning +
                    "\nThe reviewer should flag this as a QUALITY ISSUE, not a CAPABILITY GAP.\n\n" +
                    outputToReview;
            }

            // 3. Reviewer evaluates and parse structured review
            ReviewResult review = reviewIterationOutput(outputToReview, iteration, tasks, inputs, swarmId);
            String reviewText = review.rawFeedback();

            if (review.approved()) {
                approved = true;
                logger.info("Iteration {}: APPROVED by reviewer", iteration);
                publishEvent(SwarmEvent.Type.ITERATION_REVIEW_PASSED,
                    "Iteration " + iteration + " approved", swarmId, Map.of());

            } else if (review.hasCapabilityGaps()) {
                // 4. Process capability gaps — generate/reuse skills, rebuild agents
                int newSkills = processCapabilityGaps(review, processedGapDescriptions,
                    gapsSkippedAsDuplicateRef, inputs, swarmId, iteration);

                if (newSkills == -1) {
                    // All gaps were reclassified as tool errors — treat as quality issues
                    reviewFeedback = reviewText +
                        "\n\nNOTE: The reviewer flagged tool errors as capability gaps, but these are quality issues. " +
                        "The agents should use REAL URLs (not placeholder domains) and try different data sources.";
                    logger.info("Iteration {}: All {} gaps reclassified as quality issues — no skills to generate",
                        iteration, review.capabilityGaps().size());
                    publishEvent(SwarmEvent.Type.ITERATION_REVIEW_FAILED,
                        "Iteration " + iteration + " needs refinement (quality issues, gaps reclassified)", swarmId, Map.of());
                    continue; // skip to next iteration with quality feedback
                } else {
                    skillsGenerated += newSkills;
                    reviewFeedback = reviewText;
                    publishEvent(SwarmEvent.Type.ITERATION_REVIEW_FAILED,
                        "Iteration " + iteration + " needs refinement (capability gaps)", swarmId,
                        Map.of("gaps", review.capabilityGaps().size(),
                               "skillsGenerated", skillsGenerated,
                               "skillsReused", skillsReused));
                }

            } else {
                // 5. QUALITY ISSUES only — standard feedback
                reviewFeedback = reviewText;
                logger.info("Iteration {}: NEEDS_REFINEMENT (quality issues only)", iteration);
                publishEvent(SwarmEvent.Type.ITERATION_REVIEW_FAILED,
                    "Iteration " + iteration + " needs refinement", swarmId, Map.of());
            }

            // 6. Convergence detection
            if (!approved && detectConvergence(allOutputs, review, toolErrorSummary,
                    previousOutputLengthRef, previousGaps, staleIterationsRef)) {
                break;
            }

            // Memory flush: persist iteration outcome for future runs
            saveToMemory("iteration-outcome", String.format(
                "Iteration %d/%d: %s (skills generated: %d, skills reused: %d, registry size: %d)",
                iteration, effectiveMaxIterations, approved ? "APPROVED" : "NEEDS_REFINEMENT",
                skillsGenerated, skillsReused, skillRegistry.size()),
                Map.of("iteration", iteration, "approved", approved,
                       "skillsGenerated", skillsGenerated, "skillsReused", skillsReused));

            publishEvent(SwarmEvent.Type.ITERATION_COMPLETED,
                "Iteration " + iteration + " completed (approved: " + approved + ")", swarmId,
                Map.of("approved", approved, "iteration", iteration));
        }

        // 7. Determine stop reason and build final output
        String stopReason;
        if (approved) {
            stopReason = "APPROVED by reviewer";
        } else if (staleIterationsRef[0] >= 2) {
            stopReason = "auto-stopped (convergence — no meaningful progress)";
        } else {
            stopReason = "max iterations (" + effectiveMaxIterations + ") reached";
        }
        logger.info("Self-Improving Process: Stopped — {}", stopReason);

        int promoted = autoPromoteSkills(swarmId);
        return buildFinalOutput(allOutputs, iteration, approved, stopReason,
            skillsGenerated, gapsSkippedAsDuplicateRef[0], promoted, swarmId);
    }

    /**
     * Evolve the first task's description to incorporate feedback and command history from prior iterations.
     */
    private void evolveTaskDescription(List<Task> orderedTasks, String reviewFeedback,
                                        String toolErrorSummary, List<String> commandLedger) {
        if (reviewFeedback == null || orderedTasks.isEmpty()) return;

        Task analysisTask = orderedTasks.get(0);

        // Build structured summary from command ledger instead of raw output
        String ledgerText = commandLedger.isEmpty() ? "(no commands executed yet)" :
            commandLedger.stream()
                .map(cmd -> "- " + cmd)
                .collect(Collectors.joining("\n"));

        String evolvedDescription = String.format(
            "CONTINUE from where the previous iteration left off. Do NOT repeat completed work.\n\n" +
            "COMMANDS ALREADY EXECUTED (DO NOT REPEAT THESE):\n%s\n\n" +
            "REVIEWER FEEDBACK (what to do NOW):\n%s\n\n" +
            "%s" +
            "YOUR TASK: Address the reviewer's feedback by taking NEW actions. " +
            "Use different commands, scan different targets, or try exploitation tools (hydra, nikto, etc.). " +
            "Do NOT re-run commands that already succeeded.",
            ledgerText.length() > 4000 ? ledgerText.substring(0, 4000) + "\n[... truncated ...]" : ledgerText,
            reviewFeedback,
            toolErrorSummary != null ?
                "TOOL ERRORS (adapt your approach):\n" + toolErrorSummary + "\n\n" : ""
        );

        // Inject reviewer's specific next commands
        ReviewResult lastReview = ReviewResult.parse(reviewFeedback);
        if (lastReview != null && lastReview.hasNextCommands()) {
            evolvedDescription += "\n\nMANDATORY COMMANDS TO EXECUTE (run these FIRST before anything else):\n";
            int cmdNum = 1;
            for (String cmd : lastReview.nextCommands()) {
                evolvedDescription += cmdNum++ + ". " + cmd + "\n";
            }
            evolvedDescription += "\nYou MUST call shell_command with each of these commands. Do NOT skip them.\n";
        }

        analysisTask.setDescription(evolvedDescription);
    }

    // ==================== Extracted Methods (execute decomposition) ====================

    /**
     * Execute all tasks for a single iteration with the current agents.
     * Returns the outputs from this iteration and populates contextOutputs for downstream use.
     */
    private List<TaskOutput> executeIterationTasks(List<Task> orderedTasks, Map<String, Object> inputs,
                                                    List<TaskOutput> contextOutputs, String swarmId,
                                                    int iteration) {
        List<TaskOutput> iterationOutputs = new ArrayList<>();

        for (Task task : orderedTasks) {
            // Find the current agent for this task (may have been rebuilt with new tools)
            Agent taskAgent = findCurrentAgent(task.getAgent());
            // Execute with context from prior tasks
            try {
                publishEvent(SwarmEvent.Type.TASK_STARTED,
                    "Starting task: " + task.getId() + " (iteration " + iteration + ")", swarmId, Map.of());

                TaskOutput output = taskAgent.executeTask(task, contextOutputs);
                iterationOutputs.add(output);
                contextOutputs.add(output);

                // Save to file if task has outputFile configured
                if (task.getOutputFile() != null) {
                    try {
                        java.nio.file.Path path = java.nio.file.Path.of(task.getOutputFile());
                        if (path.getParent() != null) {
                            java.nio.file.Files.createDirectories(path.getParent());
                        }
                        java.nio.file.Files.writeString(path, output.getRawOutput() != null ? output.getRawOutput() : "",
                            java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.TRUNCATE_EXISTING);
                        logger.info("Task output saved to: {}", task.getOutputFile());
                    } catch (Exception fileErr) {
                        logger.warn("Failed to save output to {}: {}", task.getOutputFile(), fileErr.getMessage());
                    }
                }

                logger.info("Task completed (iteration {}): {} ({} chars, {} ms)",
                    iteration, truncate(task.getDescription(), 50),
                    output.getRawOutput().length(), output.getExecutionTimeMs());

                // Record budget usage
                BudgetTracker bt = inputs != null && inputs.get("__budgetTracker") instanceof BudgetTracker b ? b : null;
                String bsId = inputs != null && inputs.get("__budgetSwarmId") instanceof String s ? s : swarmId;
                recordBudgetUsage(bt, bsId, output, taskAgent.getModelName());

                publishEvent(SwarmEvent.Type.TASK_COMPLETED,
                    "Completed task: " + task.getId() + " (iteration " + iteration + ")", swarmId, Map.of());

            } catch (Exception e) {
                logger.error("Task failed (iteration {}): {}", iteration, e.getMessage());
                TaskOutput errorOutput = TaskOutput.builder()
                    .taskId(task.getId())
                    .rawOutput("Error: " + e.getMessage())
                    .build();
                iterationOutputs.add(errorOutput);
                contextOutputs.add(errorOutput);
            }
        }

        return iterationOutputs;
    }

    /**
     * Have the reviewer agent evaluate the iteration output.
     * Returns the parsed ReviewResult.
     */
    private ReviewResult reviewIterationOutput(String outputToReview, int iteration,
                                                List<Task> tasks, Map<String, Object> inputs,
                                                String swarmId) {
        Task reviewTask = createReviewTask(outputToReview, iteration, tasks);
        TaskOutput reviewOutput = reviewerAgent.executeTask(reviewTask, Collections.emptyList());

        // Record reviewer budget usage
        BudgetTracker bt2 = inputs != null && inputs.get("__budgetTracker") instanceof BudgetTracker b ? b : null;
        String bsId2 = inputs != null && inputs.get("__budgetSwarmId") instanceof String s ? s : swarmId;
        recordBudgetUsage(bt2, bsId2, reviewOutput, reviewerAgent.getModelName());

        // Parse structured review
        return ReviewResult.parse(reviewOutput.getRawOutput());
    }

    /**
     * Process capability gaps from a review: classify, deduplicate, generate skills, validate, rebuild agents.
     * Returns the number of new skills generated in this pass.
     */
    private int processCapabilityGaps(ReviewResult review, List<String> processedGapDescriptions,
                                       int[] gapsSkippedAsDuplicateRef,
                                       Map<String, Object> inputs, String swarmId,
                                       int iteration) {
        int skillsGenerated = 0;

        // Pre-filter: reclassify tool-error gaps as quality issues
        List<String> genuineGaps = new ArrayList<>();
        List<String> reclassifiedAsQuality = new ArrayList<>();

        for (String gap : review.capabilityGaps()) {
            String gapLower = gap.toLowerCase();
            boolean isToolError = gapLower.contains("i/o error") || gapLower.contains("io error") ||
                gapLower.contains("connection refused") || gapLower.contains("connection timed out") ||
                gapLower.contains("unknownhostexception") || gapLower.contains("status=404") ||
                gapLower.contains("status=403") || gapLower.contains("http error") ||
                (gapLower.contains("failed") && (gapLower.contains("api") || gapLower.contains("request")) &&
                 (gapLower.contains("error") || gapLower.contains("i/o")));

            if (isToolError) {
                reclassifiedAsQuality.add(gap);
            } else {
                genuineGaps.add(gap);
            }
        }

        if (!reclassifiedAsQuality.isEmpty()) {
            logger.info("Iteration {}: Reclassified {}/{} gaps as quality issues (tool errors, not missing capabilities)",
                iteration, reclassifiedAsQuality.size(), review.capabilityGaps().size());
            for (String reclassified : reclassifiedAsQuality) {
                logger.info("  RECLASSIFIED (tool error): {}", truncate(reclassified, 100));
            }
        }

        if (genuineGaps.isEmpty()) {
            // All gaps were tool errors — signal caller to treat as quality issues
            return -1; // sentinel: all gaps reclassified
        }

        logger.info("Iteration {}: {} genuine capability gaps (out of {} total)",
            iteration, genuineGaps.size(), review.capabilityGaps().size());

        // Collect tools once for all gaps in this iteration
        List<String> existingNames = currentAgents.stream()
            .flatMap(a -> a.getTools().stream())
            .map(BaseTool::getFunctionName)
            .distinct()
            .collect(Collectors.toList());

        Map<String, BaseTool> existingToolsMap = currentAgents.stream()
            .flatMap(a -> a.getTools().stream())
            .collect(Collectors.toMap(BaseTool::getFunctionName, t -> t, (a, b) -> a));

        // Resolve the skill engine. Override (e.g. CodingAgentSkillGenerator) wins;
        // otherwise build a fresh LLM-driven SkillGenerator and warm its tool-format
        // hints. discoverToolFormats only matters to the LLM path — coding agents
        // discover tools themselves.
        ai.intelliswarm.swarmai.skill.SkillEngine generator;
        if (skillEngineOverride != null) {
            generator = skillEngineOverride;
        } else {
            SkillGenerator llmGenerator = new SkillGenerator(reviewerAgent.getChatClient());
            llmGenerator.discoverToolFormats(existingToolsMap);
            generator = llmGenerator;
        }

        for (String gap : genuineGaps) {
            // --- Semantic Deduplication: check for existing similar skills ---
            GeneratedSkill reusedSkill = findReusableSkill(gap, swarmId);
            if (reusedSkill != null) {
                // Skill already exists — reuse it if not already in agent toolkit
                if (!existingNames.contains(reusedSkill.getFunctionName())) {
                    reusedSkill.setAvailableTools(existingToolsMap);
                    rebuildAgentsWithSkill(reusedSkill);
                    existingNames.add(reusedSkill.getFunctionName());
                }
                skillsReused++;
                saveToMemory("skill-reuse", String.format(
                    "Reused skill '%s' for gap: %s (effectiveness: %.0f%%)",
                    reusedSkill.getName(), truncate(gap, 100), reusedSkill.getEffectiveness() * 100),
                    Map.of("skillName", reusedSkill.getName(), "gap", gap));
                continue;
            }

            // --- Gap Quality Analysis: should we generate a skill at all? ---
            List<BaseTool> allExistingTools = existingToolsMap.values().stream()
                .collect(java.util.stream.Collectors.toList());

            // Use PolicyEngine if available, otherwise fall back to SkillGapAnalyzer directly
            SkillGapAnalyzer.GapAnalysis gapAnalysis;
            if (policyEngine != null) {
                ai.intelliswarm.swarmai.rl.SkillGenerationContext genCtx =
                    gapAnalyzer.buildContext(gap, allExistingTools, skillRegistry);
                ai.intelliswarm.swarmai.rl.SkillDecision decision =
                    policyEngine.shouldGenerateSkill(genCtx);

                // Convert SkillDecision back to GapAnalysis for downstream compatibility
                gapAnalysis = new SkillGapAnalyzer.GapAnalysis(
                    gap, decision.recommendation(), decision.confidence(),
                    List.of(decision.reasoning()),
                    new SkillGapAnalyzer.CoverageResult(1.0 - genCtx.noveltyScore(), List.of()),
                    genCtx.recommendedType());
            } else {
                gapAnalysis = gapAnalyzer.analyze(gap, allExistingTools, skillRegistry);
            }

            if (!gapAnalysis.shouldGenerate()) {
                logger.info("Gap analysis REJECTED skill generation for: {} (recommendation={}, score={:.2f}, reasons={})",
                    truncate(gap, 60), gapAnalysis.recommendation(), gapAnalysis.score(), gapAnalysis.reasons());

                publishEvent(SwarmEvent.Type.SKILL_GENERATION_SKIPPED,
                    "Skill generation skipped: " + truncate(gap, 80), swarmId,
                    Map.of("gap", gap, "recommendation", gapAnalysis.recommendation().name(),
                           "reasons", gapAnalysis.reasons()));

                saveToMemory("skill-skipped", String.format(
                    "Skipped skill generation for gap: %s (reason: %s, score: %.2f)",
                    truncate(gap, 100), gapAnalysis.recommendation(), gapAnalysis.score()),
                    Map.of("gap", gap, "recommendation", gapAnalysis.recommendation().name()));
                continue;
            }

            logger.info("Gap analysis APPROVED skill generation: {} (type={}, score={:.2f})",
                truncate(gap, 60), gapAnalysis.recommendedType(), gapAnalysis.score());

            // --- Cross-iteration dedup: skip if a very similar gap was already processed ---
            boolean isDuplicateGap = false;
            Set<String> gapTokens = tokenizeForDedup(gap);
            for (String processed : processedGapDescriptions) {
                Set<String> processedTokens = tokenizeForDedup(processed);
                double overlap = jaccardSimilaritySimple(gapTokens, processedTokens);
                if (overlap > 0.35) {
                    isDuplicateGap = true;
                    gapsSkippedAsDuplicateRef[0]++;
                    logger.info("Gap SKIPPED (duplicate of previously processed gap, overlap={:.0f}%): {}",
                        overlap * 100, truncate(gap, 80));
                    break;
                }
            }
            if (isDuplicateGap) continue;

            // Track this gap as processed (whether generation succeeds or fails)
            processedGapDescriptions.add(gap);

            // Generate and validate skill for this gap
            Optional<GeneratedSkill> validatedSkill = generateAndValidateSkill(
                gap, generator, existingToolsMap, existingNames, swarmId, iteration);

            if (validatedSkill.isPresent()) {
                GeneratedSkill skill = validatedSkill.get();
                rebuildAgentsWithSkill(skill);
                existingNames.add(skill.getFunctionName());
                skillsGenerated++;
            }
        }

        return skillsGenerated;
    }

    /**
     * Generate a single skill for a capability gap, validate it, and if invalid attempt one refinement.
     * Returns the validated skill wrapped in Optional, or empty if generation/validation fails.
     */
    private Optional<GeneratedSkill> generateAndValidateSkill(String gap,
                                                               ai.intelliswarm.swarmai.skill.SkillEngine generator,
                                                               Map<String, BaseTool> existingToolsMap,
                                                               List<String> existingNames,
                                                               String swarmId, int iteration) {
        // Query memory for hints about similar past gaps
        String memoryHint = queryMemoryForGap(gap);

        logger.info("Generating skill for gap: {}", gap);

        GeneratedSkill skill = generator.generate(
            memoryHint != null ? gap + "\n\nHINT FROM PAST RUNS:\n" + memoryHint : gap,
            existingNames);

        if (skill == null) {
            return Optional.empty();
        }

        publishEvent(SwarmEvent.Type.SKILL_GENERATED,
            "Generated skill: " + skill.getName(), swarmId,
            Map.of("skillName", skill.getName(), "gap", gap));

        // Inject existing tools so validation can succeed
        skill.setAvailableTools(existingToolsMap);

        // Validate
        SkillValidator.ValidationResult validation = skillValidator.validate(skill);

        if (validation.passed()) {
            skill.setStatus(SkillStatus.VALIDATED);
            skill.setAvailableTools(existingToolsMap);
            skillRegistry.register(skill);

            // Log integration test results if available
            String integrationSummary = "";
            if (validation.hasIntegrationTestResults()) {
                integrationSummary = String.format(", integration tests: %d/%d passed",
                    validation.integrationTestsPassed(),
                    validation.integrationTestsPassed() + validation.integrationTestsFailed());
            }

            publishEvent(SwarmEvent.Type.SKILL_VALIDATED,
                "Skill validated: " + skill.getName() + integrationSummary, swarmId,
                Map.of("skillName", skill.getName(), "skillId", skill.getId(),
                    "integrationTestsPassed", validation.integrationTestsPassed(),
                    "integrationTestsFailed", validation.integrationTestsFailed()));
            publishEvent(SwarmEvent.Type.SKILL_REGISTERED,
                "Skill registered: " + skill.getName(), swarmId,
                Map.of("skillName", skill.getName(), "skillId", skill.getId()));

            logger.info("New skill validated and registered: {} ({}{})",
                skill.getName(), skill.getId(), integrationSummary);

            // Persist to memory for future runs
            saveToMemory("skill-generated", String.format(
                "Generated skill '%s' for gap: %s%s",
                skill.getName(), truncate(gap, 150), integrationSummary),
                Map.of("skillName", skill.getName(), "gap", gap,
                    "iteration", iteration));

            return Optional.of(skill);
        }

        // Validation failed — attempt refinement
        String allErrors = validation.errorsAsString();
        if (validation.hasIntegrationTestResults() && validation.integrationTestsFailed() > 0) {
            allErrors += "; Integration test failures: " +
                String.join("; ", validation.integrationTestResults().failureMessages());
        }

        logger.warn("Skill '{}' failed validation: {}",
            skill.getName(), allErrors);

        publishEvent(SwarmEvent.Type.SKILL_VALIDATION_FAILED,
            "Skill validation failed: " + skill.getName(), swarmId,
            Map.of("skillName", skill.getName(), "errors", allErrors));

        // Try to refine once — include integration test failures in feedback
        GeneratedSkill refined = generator.refine(skill, allErrors);
        if (refined != null) {
            refined.setAvailableTools(existingToolsMap);
            SkillValidator.ValidationResult retryValidation = skillValidator.validate(refined);
            if (retryValidation.passed()) {
                refined.setStatus(SkillStatus.VALIDATED);
                refined.setAvailableTools(existingToolsMap);
                skillRegistry.register(refined);

                String retryIntSummary = retryValidation.hasIntegrationTestResults()
                    ? String.format(", integration tests: %d/%d passed",
                        retryValidation.integrationTestsPassed(),
                        retryValidation.integrationTestsPassed() + retryValidation.integrationTestsFailed())
                    : "";

                publishEvent(SwarmEvent.Type.SKILL_VALIDATED,
                    "Refined skill validated: " + refined.getName() + retryIntSummary, swarmId,
                    Map.of("skillName", refined.getName(), "refined", true));
                publishEvent(SwarmEvent.Type.SKILL_REGISTERED,
                    "Refined skill registered: " + refined.getName(), swarmId,
                    Map.of("skillName", refined.getName(), "skillId", refined.getId()));

                logger.info("Refined skill validated: {}{}", refined.getName(), retryIntSummary);

                saveToMemory("skill-refined", String.format(
                    "Skill '%s' required refinement for gap: %s (original errors: %s)",
                    refined.getName(), truncate(gap, 100), truncate(allErrors, 100)),
                    Map.of("skillName", refined.getName(), "gap", gap));

                return Optional.of(refined);
            } else {
                logger.warn("Refined skill also failed. Skipping gap: {}", gap);
                saveToMemory("skill-failed", String.format(
                    "Failed to generate skill for gap: %s (errors: %s)",
                    truncate(gap, 150), truncate(retryValidation.errorsAsString(), 100)),
                    Map.of("gap", gap, "iteration", iteration));
            }
        }

        return Optional.empty();
    }

    /**
     * Detect convergence: check if the process is making meaningful progress.
     * Updates staleIterations counter in-place via the passed array (single-element carrier).
     * Returns true if the process should stop due to convergence.
     */
    private ai.intelliswarm.swarmai.rl.PolicyEngine getEffectivePolicy() {
        return policyEngine != null ? policyEngine : new ai.intelliswarm.swarmai.rl.HeuristicPolicy();
    }

    private boolean detectConvergence(List<TaskOutput> allOutputs, ReviewResult review,
                                       String toolErrorSummary, int[] previousOutputLengthRef,
                                       Set<String> previousGaps, int[] staleIterationsRef) {
        // Compute convergence signals
        int currentOutputLength = allOutputs.stream()
            .mapToInt(o -> o.getRawOutput() != null ? o.getRawOutput().length() : 0).sum();

        double outputGrowthRate = previousOutputLengthRef[0] > 0
            ? (double) currentOutputLength / previousOutputLengthRef[0]
            : 2.0; // first iteration always counts as growth

        Set<String> currentGaps = new HashSet<>();
        if (review != null && review.hasCapabilityGaps()) {
            currentGaps.addAll(review.capabilityGaps());
        }

        double gapRepetitionRate = (!currentGaps.isEmpty() && !previousGaps.isEmpty())
            ? (double) countIntersection(currentGaps, previousGaps) / currentGaps.size()
            : 0.0;

        int newSkillsThisIteration = 0; // will be set by caller context if available
        boolean agentAdapted = toolErrorSummary != null &&
            (previousOutputLengthRef[0] > 0 && !currentGaps.equals(previousGaps));
        if (agentAdapted) newSkillsThisIteration = 1; // approximate

        // Delegate to PolicyEngine
        ai.intelliswarm.swarmai.rl.ConvergenceContext ctx = new ai.intelliswarm.swarmai.rl.ConvergenceContext(
                outputGrowthRate, gapRepetitionRate, newSkillsThisIteration,
                skillsReused > 0 ? 1.0 : 0.0, // approximate reuse rate
                staleIterationsRef[0] + 1, // current iteration
                1.0 // no budget info available here
        );

        boolean shouldStop = getEffectivePolicy().shouldStopIteration(ctx);

        // Update tracking state for next call
        previousOutputLengthRef[0] = currentOutputLength;
        previousGaps.clear();
        previousGaps.addAll(currentGaps);

        // Update stale counter (for logging and metadata)
        boolean outputGrew = outputGrowthRate > 1.1;
        boolean sameGaps = !currentGaps.isEmpty() && currentGaps.equals(previousGaps);
        if ((!outputGrew && sameGaps) && !agentAdapted) {
            staleIterationsRef[0]++;
        } else {
            staleIterationsRef[0] = 0;
        }

        if (shouldStop) {
            logger.info("Self-Improving Process: Auto-stopping — PolicyEngine recommends stop " +
                "(outputGrowth={:.2f}, gapRepetition={:.2f})", outputGrowthRate, gapRepetitionRate);
        }

        return shouldStop;
    }

    private int countIntersection(Set<String> a, Set<String> b) {
        return (int) a.stream().filter(b::contains).count();
    }

    /**
     * Build the final SwarmOutput with metadata, persist skills and scan cache.
     */
    private SwarmOutput buildFinalOutput(List<TaskOutput> allOutputs, int iteration, boolean approved,
                                          String stopReason, int skillsGenerated, int gapsSkippedAsDuplicate,
                                          int promoted, String swarmId) {
        SwarmOutput.Builder outputBuilder = SwarmOutput.builder()
            .swarmId(swarmId)
            .successful(approved || !allOutputs.isEmpty())
            .taskOutputs(allOutputs)
            .metadata("skillsGenerated", skillsGenerated)
            .metadata("skillsReused", skillsReused)
            .metadata("gapsSkippedAsDuplicate", gapsSkippedAsDuplicate)
            .metadata("skillsPromoted", promoted)
            .metadata("totalIterations", iteration)
            .metadata("stopReason", stopReason)
            .metadata("registryStats", skillRegistry.getStats());

        if (!allOutputs.isEmpty()) {
            outputBuilder.finalOutput(allOutputs.get(allOutputs.size() - 1).getRawOutput());
        }

        SwarmOutput output = outputBuilder.build();

        // Persist generated skills to disk for reuse in future runs
        if (skillsGenerated > 0 || promoted > 0) {
            try {
                skillRegistry.save(DEFAULT_SKILLS_DIR);
                logger.info("Persisted {} skills to {}", skillRegistry.size(), DEFAULT_SKILLS_DIR);
            } catch (Exception e) {
                logger.warn("Failed to persist skills: {}", e.getMessage());
            }
        }

        // Persist scan cache to disk for reuse in future runs
        if (scanCache != null) {
            try {
                scanCache.saveAll();
            } catch (Exception e) {
                logger.warn("Failed to save scan cache: {}", e.getMessage());
            }
        }

        logger.info("Self-Improving Process complete: {} iterations, {} skills generated, {} reused, {} promoted, approved={}",
            iteration, skillsGenerated, skillsReused, promoted, approved);

        return output;
    }

    /**
     * Rebuild all current agents with a new skill added to their toolkit.
     */
    private void rebuildAgentsWithSkill(GeneratedSkill skill) {
        skill.setStatus(SkillStatus.ACTIVE);
        List<Agent> rebuilt = new ArrayList<>();
        for (Agent agent : currentAgents) {
            rebuilt.add(agent.withAdditionalTools(List.of(skill)));
        }
        currentAgents = rebuilt;
        logger.info("Rebuilt {} agents with new skill: {}", rebuilt.size(), skill.getName());
    }

    /**
     * Find the current version of an agent (may have been rebuilt with new tools).
     */
    private Agent findCurrentAgent(Agent originalAgent) {
        for (Agent current : currentAgents) {
            if (current.getRole().equals(originalAgent.getRole())) {
                return current;
            }
        }
        return originalAgent;
    }

    // ==================== Semantic Skill Deduplication ====================

    /**
     * Search the registry for an existing skill that matches the gap description.
     * Returns the best match above the similarity threshold, or null if none found.
     */
    private GeneratedSkill findReusableSkill(String gap, String swarmId) {
        List<SkillRegistry.SimilarSkill> similar = skillRegistry.findSimilar(gap, SKILL_SIMILARITY_THRESHOLD);

        if (!similar.isEmpty()) {
            SkillRegistry.SimilarSkill best = similar.get(0);
            logger.info("Reusing existing skill '{}' for gap '{}' (similarity: {})",
                best.skill().getName(), truncate(gap, 60),
                String.format("%.2f", best.similarity()));

            publishEvent(SwarmEvent.Type.SKILL_REUSED,
                "Reused skill: " + best.skill().getName() + " (similarity: " +
                    String.format("%.0f%%", best.similarity() * 100) + ")",
                swarmId,
                Map.of("skillName", best.skill().getName(),
                       "similarity", best.similarity(),
                       "gap", gap));

            return best.skill();
        }

        return null;
    }

    // ==================== Memory Integration ====================

    /**
     * Save a learning to memory for cross-run persistence.
     */
    private void saveToMemory(String category, String content, Map<String, Object> metadata) {
        if (memory == null) return;
        try {
            Map<String, Object> enrichedMeta = new HashMap<>(metadata);
            enrichedMeta.put("category", category);
            enrichedMeta.put("processType", "SELF_IMPROVING");
            memory.save("self-improving-process", content, enrichedMeta);
        } catch (Exception e) {
            logger.debug("Failed to save to memory: {}", e.getMessage());
        }
    }

    /**
     * Query memory for past learnings relevant to a capability gap.
     * Returns a hint string if useful context is found, null otherwise.
     */
    private String queryMemoryForGap(String gap) {
        if (memory == null) return null;
        try {
            List<String> memories = memory.search(gap, 3);
            if (memories != null && !memories.isEmpty()) {
                String hint = memories.stream()
                    .map(m -> "- " + m)
                    .collect(Collectors.joining("\n"));
                logger.info("Found {} memory hints for gap: {}", memories.size(), truncate(gap, 50));
                return hint;
            }
        } catch (Exception e) {
            logger.debug("Memory query failed: {}", e.getMessage());
        }
        return null;
    }

    // ==================== Skill Lifecycle ====================

    /**
     * Auto-promote skills that meet the effectiveness threshold.
     * Returns the number of skills promoted.
     */
    private int autoPromoteSkills(String swarmId) {
        int promoted = 0;
        for (GeneratedSkill skill : skillRegistry.getActiveSkills()) {
            if (skill.getStatus() == SkillStatus.ACTIVE && skill.meetsPromotionThreshold()) {
                skillRegistry.promote(skill.getId());
                promoted++;
                logger.info("Auto-promoted skill: {} (usage={}, effectiveness={}%)",
                    skill.getName(), skill.getUsageCount(),
                    String.format("%.0f", skill.getEffectiveness() * 100));

                publishEvent(SwarmEvent.Type.SKILL_PROMOTED,
                    "Skill promoted: " + skill.getName(), swarmId,
                    Map.of("skillName", skill.getName(),
                           "usageCount", skill.getUsageCount(),
                           "effectiveness", skill.getEffectiveness()));

                saveToMemory("skill-promoted", String.format(
                    "Promoted skill '%s' (usage: %d, effectiveness: %.0f%%)",
                    skill.getName(), skill.getUsageCount(), skill.getEffectiveness() * 100),
                    Map.of("skillName", skill.getName()));
            }
        }
        return promoted;
    }

    /**
     * Create the review task with enhanced gap-detection prompting.
     */
    private Task createReviewTask(String outputToReview, int iteration, List<Task> tasks) {
        String taskObjectives = tasks.stream()
            .map(t -> "- " + truncate(t.getDescription(), 100))
            .collect(Collectors.joining("\n"));

        String description = String.format(
            "You are reviewing the output of iteration %d.\n\n" +
            "TASK OBJECTIVES:\n%s\n\n" +
            "%s\n\n" +
            "OUTPUT TO REVIEW:\n%s\n\n" +
            "RESPOND IN THIS EXACT FORMAT:\n\n" +
            "VERDICT: APPROVED or NEEDS_REFINEMENT\n\n" +
            "QUALITY_ISSUES:\n" +
            "- [list any quality problems with the output]\n\n" +
            "CAPABILITY_GAPS:\n" +
            "- NO_TOOL: [describe a missing tool capability that would improve the output]\n" +
            "- INSUFFICIENT_TOOL: [describe a tool that exists but doesn't work well enough]\n\n" +
            "NEXT_COMMANDS:\n" +
            "- [If exploitation or deeper scanning is needed, list SPECIFIC shell commands to run next iteration]\n" +
            "- [Include full command with arguments, e.g., 'hydra -l admin -P /usr/share/nmap/nselib/data/passwords.lst 192.168.1.1 http-get /']\n" +
            "- [If no further commands needed, omit this section]\n\n" +
            "If the output fully meets all objectives, use VERDICT: APPROVED and omit the other sections.\n" +
            "If there are quality issues but no missing tools, include QUALITY_ISSUES but omit CAPABILITY_GAPS.\n" +
            "Only include CAPABILITY_GAPS if a NEW TOOL would genuinely help — not just prompt improvements.",
            iteration, taskObjectives,
            qualityCriteria != null ? "QUALITY CRITERIA:\n" + qualityCriteria : "",
            outputToReview.length() > 8000 ? outputToReview.substring(0, 8000) + "\n[... truncated ...]" : outputToReview
        );

        return Task.builder()
            .id("review-iteration-" + iteration)
            .description(description)
            .expectedOutput("Structured review with VERDICT, QUALITY_ISSUES, and optional CAPABILITY_GAPS")
            .agent(reviewerAgent)
            .maxExecutionTime(120000)
            .build();
    }

    private List<Task> orderTasks(List<Task> tasks) {
        // Simple dependency ordering (same as SequentialProcess)
        List<Task> ordered = new ArrayList<>();
        Set<String> completed = new HashSet<>();
        List<Task> remaining = new ArrayList<>(tasks);

        while (!remaining.isEmpty()) {
            boolean progress = false;
            Iterator<Task> it = remaining.iterator();
            while (it.hasNext()) {
                Task task = it.next();
                if (task.isReady(completed)) {
                    ordered.add(task);
                    completed.add(task.getId());
                    it.remove();
                    progress = true;
                }
            }
            if (!progress) {
                ordered.addAll(remaining);
                break;
            }
        }

        return ordered;
    }

    private void publishEvent(SwarmEvent.Type type, String message, String swarmId, Map<String, Object> metadata) {
        if (eventPublisher != null) {
            Map<String, Object> meta = new HashMap<>(metadata);
            meta.put("processType", "SELF_IMPROVING");
            eventPublisher.publishEvent(new SwarmEvent(this, type, message, swarmId, meta));
        }
    }

    private String truncate(String text, int maxLen) {
        if (text == null) return "";
        return text.length() > maxLen ? text.substring(0, maxLen) + "..." : text;
    }

    /**
     * Tokenize a gap description for cross-iteration deduplication.
     * Filters stop words and short tokens to focus on semantic content.
     */
    private Set<String> tokenizeForDedup(String text) {
        if (text == null) return Set.of();
        Set<String> stopWords = Set.of(
            "the", "and", "for", "that", "this", "with", "from", "are", "was", "tool",
            "would", "could", "should", "need", "provide", "existing", "currently",
            "more", "also", "new", "which", "using", "used", "will", "can", "not",
            "data", "output", "report", "analysis", "information", "sources", "multiple");
        return java.util.Arrays.stream(text.toLowerCase().split("[^a-z0-9]+"))
            .filter(t -> t.length() > 3)
            .filter(t -> !stopWords.contains(t))
            .collect(Collectors.toSet());
    }

    /**
     * Check whether the output contains evidence of actual tool usage.
     * If the output appears to be purely LLM knowledge with no tool data,
     * return a warning message. Otherwise return null (evidence found).
     */
    private String checkToolEvidence(String output) {
        if (output == null || output.isBlank()) return "Output is empty";

        String lower = output.toLowerCase();

        // Positive signals: evidence that tools were actually called
        String[] toolEvidenceMarkers = {
            "http://", "https://",           // URLs from http_request/web_scrape
            "api response", "api returned",   // API data
            "json", "status code",            // HTTP response data
            "search results",                 // web_search output
            "scraped", "fetched",             // web_scrape output
            "command output", "exit code",    // shell_command output
            "**command:**", "**stdout:**",    // shell_command formatted output
            "nmap", "scan report",            // nmap output
            "open port", "filtered",          // port scan results
            "mac address", "host is up",      // network scan results
            "calculated", "calculation",      // calculator output
            "file content", "csv data",       // file tool output
            "retrieved:", "source:",          // data attribution
            "[from tool]", "[from api]",      // explicit source markers
            "wikipedia", "github.com",        // known-good API sources
            "hn.algolia", "duckduckgo"        // known-good API sources
        };

        int evidenceCount = 0;
        for (String marker : toolEvidenceMarkers) {
            if (lower.contains(marker)) evidenceCount++;
        }

        // Negative signals: LLM-knowledge-only patterns
        String[] knowledgeOnlyMarkers = {
            "based on general knowledge",
            "from my training data",
            "as of my last update",
            "i don't have access to real-time",
            "no results were found",
            "data not available",
            "could not be retrieved"
        };

        int knowledgeOnlyCount = 0;
        for (String marker : knowledgeOnlyMarkers) {
            if (lower.contains(marker)) knowledgeOnlyCount++;
        }

        if (evidenceCount == 0 && output.length() > 500) {
            return "No tool evidence found in " + output.length() + " chars of output. " +
                "The agents appear to have answered from LLM knowledge without calling any tools. " +
                "They should use http_request with real URLs (Wikipedia API, GitHub API, HN Algolia) to gather data.";
        }

        if (knowledgeOnlyCount >= 2 && evidenceCount < 2) {
            return "Output contains " + knowledgeOnlyCount + " 'no data' markers and only " + evidenceCount +
                " tool evidence markers. The agents may not have used tools effectively.";
        }

        return null; // evidence looks OK
    }

    /**
     * Extract nmap scan results from iteration outputs and cache them for future runs.
     * Parses ShellCommandTool output for nmap commands and their scan reports.
     */
    private void cacheNewScanResults(List<TaskOutput> outputs) {
        Pattern hostPattern = Pattern.compile("Nmap scan report for (\\d+\\.\\d+\\.\\d+\\.\\d+)");
        Pattern cmdPattern = Pattern.compile("\\*\\*Command:\\*\\*\\s*`([^`]*nmap[^`]*)`");

        for (TaskOutput output : outputs) {
            String text = output.getRawOutput();
            if (text == null) continue;

            // Find nmap commands and their output
            Matcher cmdMatcher = cmdPattern.matcher(text);
            while (cmdMatcher.find()) {
                String command = cmdMatcher.group(1);
                // Find host IPs from the scan output
                Matcher hostMatcher = hostPattern.matcher(text);
                while (hostMatcher.find()) {
                    String hostIp = hostMatcher.group(1);
                    scanCache.save(hostIp, command, text);
                }
            }
        }
    }

    /**
     * Extract tool execution errors from iteration outputs for self-adjustment feedback.
     * Scans output text for timeout, error, and failure patterns so the next iteration
     * can adapt its approach (e.g., scan individual hosts instead of full subnet).
     */
    private String extractToolErrors(List<TaskOutput> outputs) {
        List<String> errors = new ArrayList<>();
        String[] errorPatterns = {
            "timed out", "Command timed out", "Error: Command",
            "Error: Failed to execute", "Error: Command is required",
            "execution failed", "no hosts were scanned",
            "invalid", "connection refused", "host is down"
        };

        for (TaskOutput output : outputs) {
            String text = output.getRawOutput();
            if (text == null) continue;

            // Extract lines containing error patterns
            for (String line : text.split("\n")) {
                String lineLower = line.toLowerCase();
                for (String pattern : errorPatterns) {
                    if (lineLower.contains(pattern.toLowerCase())) {
                        String trimmed = line.trim();
                        if (trimmed.length() > 200) trimmed = trimmed.substring(0, 200) + "...";
                        if (!errors.contains(trimmed)) {
                            errors.add(trimmed);
                        }
                        break;
                    }
                }
            }
        }

        if (errors.isEmpty()) return null;

        StringBuilder summary = new StringBuilder();
        for (int i = 0; i < Math.min(errors.size(), 10); i++) {
            summary.append("- ").append(errors.get(i)).append("\n");
        }
        if (errors.size() > 10) {
            summary.append("- ... and ").append(errors.size() - 10).append(" more errors\n");
        }
        return summary.toString();
    }

    /**
     * Extract executed shell commands from iteration outputs to build a structured ledger.
     * Parses the ShellCommandTool output format: **Command:** `...` / **Exit Code:** N
     */
    private List<String> extractExecutedCommands(List<TaskOutput> outputs) {
        List<String> commands = new ArrayList<>();
        java.util.regex.Pattern cmdPattern = java.util.regex.Pattern.compile(
            "\\*\\*Command:\\*\\*\\s*`([^`]+)`");
        java.util.regex.Pattern exitPattern = java.util.regex.Pattern.compile(
            "\\*\\*Exit Code:\\*\\*\\s*(\\d+)");

        for (TaskOutput output : outputs) {
            String text = output.getRawOutput();
            if (text == null) continue;

            // Split by command blocks
            String[] blocks = text.split("(?=\\*\\*Command:\\*\\*)");
            for (String block : blocks) {
                java.util.regex.Matcher cmdMatcher = cmdPattern.matcher(block);
                if (cmdMatcher.find()) {
                    String cmd = cmdMatcher.group(1);
                    String status = "OK";

                    java.util.regex.Matcher exitMatcher = exitPattern.matcher(block);
                    if (exitMatcher.find() && !"0".equals(exitMatcher.group(1))) {
                        status = "EXIT " + exitMatcher.group(1);
                    }

                    if (block.contains("timed out")) status = "TIMEOUT";
                    if (block.contains("Error:")) status = "ERROR";

                    // Brief result summary (first 80 chars of stdout)
                    String brief = "";
                    int stdoutIdx = block.indexOf("**stdout:**");
                    if (stdoutIdx >= 0) {
                        String after = block.substring(stdoutIdx + 11).trim();
                        // Remove code fences
                        after = after.replaceAll("```\\w*\\n?", "").trim();
                        brief = after.length() > 80 ? after.substring(0, 80) + "..." : after;
                        brief = brief.replace("\n", " ").trim();
                    }

                    commands.add(cmd + " [" + status + "]" + (brief.isEmpty() ? "" : " — " + brief));
                }
            }
        }
        return commands;
    }

    private double jaccardSimilaritySimple(Set<String> a, Set<String> b) {
        if (a.isEmpty() || b.isEmpty()) return 0.0;
        Set<String> intersection = new HashSet<>(a);
        intersection.retainAll(b);
        Set<String> union = new HashSet<>(a);
        union.addAll(b);
        return (double) intersection.size() / union.size();
    }

    // Expose registry and memory for external access
    public SkillRegistry getSkillRegistry() {
        return skillRegistry;
    }

    public Memory getMemory() {
        return memory;
    }
}
