/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

/**
 * Columnar parser - builds columnar data structures
 *
 * OPTIMIZED: Single-pass extraction for maximum performance
 * Instead of multiple passes through entities, we extract everything in ONE loop.
 */

import type { EntityRef } from './types.js';
import { SpatialHierarchyBuilder } from './spatial-hierarchy-builder.js';
import { EntityExtractor } from './entity-extractor.js';
import { extractLengthUnitScale } from './unit-extractor.js';
import { getAttributeNames, getInheritanceChain } from './ifc-schema.js';
import { parsePropertyValue } from './on-demand-extractors.js';
import { buildCompactEntityIndexAsync } from './compact-entity-index.js';
import { yieldToEventLoop } from './yield-to-event-loop.js';
import {
    StringTable,
    EntityTableBuilder,
    PropertyTableBuilder,
    QuantityTableBuilder,
    RelationshipGraphBuilder,
    RelationshipType,
    QuantityType,
} from '@ifc-lite/data';
import type { SpatialHierarchy, QuantityTable, PropertyValue, PropertySet, QuantitySet, IfcStoreBase, IfcEntity, IfcAttributeValue } from '@ifc-lite/data';
import { BufferEntitySource } from './entity-source.js';
import { batchExtractGlobalIdAndName } from './columnar-parser-attributes.js';
import {
    GEOMETRY_TYPES,
    REL_TYPE_MAP,
    QUANTITY_TYPE_MAP,
    SPATIAL_TYPES,
    HIERARCHY_REL_TYPES,
    PROPERTY_REL_TYPES,
    ASSOCIATION_REL_TYPES,
    SKIP_DISPLAY_ATTRS,
    PROPERTY_ENTITY_TYPES,
    PROPERTY_CONTAINER_TYPES,
    isIfcTypeLikeEntity,
} from './columnar-parser-indexes.js';
import { extractRelFast, extractPropertyRelFast } from './columnar-parser-relationships.js';
import { safeUtf8Decode } from '@ifc-lite/data';

import type { SpatialIndex, EntityByIdIndex } from './columnar-parser-indexes.js';

// Re-export interfaces/types from extracted modules for public API compatibility
export type { SpatialIndex, EntityByIdIndex } from './columnar-parser-indexes.js';

export interface IfcDataStore extends IfcStoreBase {
    parseTime: number;

    source: Uint8Array;
    entityIndex: { byId: EntityByIdIndex; byType: Map<string, number[]> };
    deferredEntityIndex?: EntityByIdIndex;

    strings: StringTable;
    entities: ReturnType<EntityTableBuilder['build']>;
    properties: ReturnType<PropertyTableBuilder['build']>;
    quantities: QuantityTable;
    relationships: ReturnType<RelationshipGraphBuilder['build']>;

    /**
     * On-demand property lookup: entityId -> array of property set expressIds
     * Used for fast single-entity property access without pre-building property tables.
     * Use extractPropertiesOnDemand() with this map for instant property retrieval.
     */
    onDemandPropertyMap?: Map<number, number[]>;

    /**
     * On-demand quantity lookup: entityId -> array of quantity set expressIds
     * Used for fast single-entity quantity access without pre-building quantity tables.
     * Use extractQuantitiesOnDemand() with this map for instant quantity retrieval.
     */
    onDemandQuantityMap?: Map<number, number[]>;

    /**
     * On-demand classification lookup: entityId -> array of IfcClassificationReference expressIds
     * Built from IfcRelAssociatesClassification relationships during parsing.
     */
    onDemandClassificationMap?: Map<number, number[]>;

    /**
     * On-demand material lookup: entityId -> relatingMaterial expressId
     * Built from IfcRelAssociatesMaterial relationships during parsing.
     * Value is the expressId of IfcMaterial, IfcMaterialLayerSet, IfcMaterialProfileSet, or IfcMaterialConstituentSet.
     */
    onDemandMaterialMap?: Map<number, number>;

    /**
     * On-demand document lookup: entityId -> array of IfcDocumentReference/IfcDocumentInformation expressIds
     * Built from IfcRelAssociatesDocument relationships during parsing.
     */
    onDemandDocumentMap?: Map<number, number[]>;

    /**
     * Project-level length unit scale to convert raw IFC numeric measure
     * values into base SI metres. `1.0` for metres, `0.001` for milli,
     * `0.0254` for inches, etc. Surfaced on the store so consumers
     * (notably the IDS validator, where IDS literals are always in
     * base SI units) can convert without re-parsing the unit graph.
     */
    lengthUnitScale?: number;
}


function detectSchemaVersion(buffer: Uint8Array): IfcDataStore['schemaVersion'] {
    const headerEnd = Math.min(buffer.length, 2000);
    const headerText = safeUtf8Decode(buffer, 0, headerEnd).toUpperCase();

    if (headerText.includes('IFC5')) return 'IFC5';
    if (headerText.includes('IFC4X3')) return 'IFC4X3';
    if (headerText.includes('IFC4')) return 'IFC4';
    if (headerText.includes('IFC2X3')) return 'IFC2X3';

    return 'IFC4'; // Default fallback
}

export class ColumnarParser {
    /**
     * Parse IFC file into columnar data store
     *
     * Uses fast semicolon-based scanning with on-demand property extraction.
     * Properties are parsed lazily when accessed, not upfront.
     * This provides instant UI responsiveness even for very large files.
     */
    async parseLite(
        buffer: ArrayBuffer | SharedArrayBuffer,
        entityRefs: EntityRef[],
        options: {
            onProgress?: (progress: { phase: string; percent: number }) => void;
            onDiagnostic?: (message: string) => void;
            yieldIntervalMs?: number;
            deferPropertyAtomIndex?: boolean;
            onSpatialReady?: (partialStore: IfcDataStore) => void;
        } = {}
    ): Promise<IfcDataStore> {
        const startTime = performance.now();
        const uint8Buffer = new Uint8Array(buffer);
        const totalEntities = entityRefs.length;

        // Phase timing for performance telemetry
        let phaseStart = startTime;
        const emitDiagnostic = (message: string) => {
            options.onDiagnostic?.(message);
        };
        const logPhase = (name: string) => {
            const now = performance.now();
            const elapsed = Math.round(now - phaseStart);
            console.log(`[parseLite] ${name}: ${elapsed}ms`);
            emitDiagnostic(`${name}: ${elapsed}ms`);
            phaseStart = now;
        };

        options.onProgress?.({ phase: 'building', percent: 0 });

        // Detect schema version from FILE_SCHEMA header
        const schemaVersion = detectSchemaVersion(uint8Buffer);

        // Initialize builders (entity table capacity set after categorization below)
        const strings = new StringTable();
        const propertyTableBuilder = new PropertyTableBuilder(strings);
        const quantityTableBuilder = new QuantityTableBuilder(strings);
        const relationshipGraphBuilder = new RelationshipGraphBuilder();

        logPhase('init builders');

        // Single pass: build byType index AND categorize entities simultaneously.
        // Uses a type-name cache to avoid calling .toUpperCase() on 4.4M refs
        // (only ~776 unique type names in IFC4).
        const byType = new Map<string, number[]>();
        const deferPropertyAtomIndex = options.deferPropertyAtomIndex === true;
        const typeUpperCache = new Map<string, string>();
        const getTypeUpper = (type: string) => {
            let upper = typeUpperCache.get(type);
            if (upper === undefined) {
                upper = type.toUpperCase();
                typeUpperCache.set(type, upper);
            }
            return upper;
        };

        // Non-product helper entities that on-demand extraction / StepExporter
        // need addressable in `byId`. These are not IfcProduct subtypes so the
        // schema-driven IFCPRODUCT subtype check below cannot capture them.
        // Without them, findPreferredGeometricRepresentationContextId() and
        // findLengthUnitReference() fail because the entities are missing from
        // the compact entity index.
        const RELEVANT_NON_PRODUCT_HELPERS = new Set([
            'IFCGEOMETRICREPRESENTATIONCONTEXT', 'IFCGEOMETRICREPRESENTATIONSUBCONTEXT',
            'IFCUNITASSIGNMENT', 'IFCSIUNIT', 'IFCCONVERSIONBASEDUNIT',
            'IFCDERIVEDUNIT', 'IFCDERIVEDUNITELEMENT', 'IFCMEASUREWITHUNIT',
            'IFCDIMENSIONALEXPONENTS',
            'IFCMAPCONVERSION', 'IFCPROJECTEDCRS',
            'IFCMATERIALLAYER', 'IFCMATERIALLAYERSET', 'IFCMATERIALLAYERSETUSAGE',
            'IFCMATERIALCONSTITUENTSET', 'IFCMATERIALCONSTITUENT',
            'IFCMATERIALPROFILESET', 'IFCMATERIALPROFILE', 'IFCMATERIAL',
            'IFCCLASSIFICATION', 'IFCCLASSIFICATIONREFERENCE',
            'IFCDOCUMENTINFORMATION', 'IFCDOCUMENTREFERENCE',
        ]);

        // Schema-driven inclusion: every IfcProduct subtype belongs in the
        // EntityTable. The previous hardcoded enumeration of IFC4 building-
        // element leaves (IFCWALL, IFCSLAB, …) and IFC4x3 infrastructure
        // leaves (IFCREFERENT, IFCSIGNAL, IFCALIGNMENT, IFCPAVEMENT, …) drifted
        // with every schema bump — new entities silently became CAT_SKIP and
        // disappeared from the hierarchy panel. The generated schema registry
        // already knows the full inheritance chain, so use it.
        const RELEVANT_PRODUCT_ROOTS = new Set(['IFCPRODUCT']);

        // Category constants for the lookup cache
        const CAT_SKIP = 0, CAT_SPATIAL = 1, CAT_GEOMETRY = 2, CAT_HIERARCHY_REL = 3,
              CAT_PROPERTY_REL = 4, CAT_PROPERTY_ENTITY = 5, CAT_ASSOCIATION_REL = 6,
              CAT_TYPE_OBJECT = 7, CAT_RELEVANT = 8;


        /** Returns true if `upper` (already uppercased) is a subtype of any type in `set`. */
        function isSubtypeOfAny(upper: string, set: Set<string>): boolean {
            const chain = getInheritanceChain(upper);
            return chain.some(ancestor => set.has(ancestor.toUpperCase()));
        }

        // Cache: type name → category (avoids 4.4M .toUpperCase() calls)
        const typeCategoryCache = new Map<string, number>();
        function getCategory(type: string): number {
            let cat = typeCategoryCache.get(type);
            if (cat !== undefined) return cat;
            const upper = getTypeUpper(type);
            if (SPATIAL_TYPES.has(upper) || isSubtypeOfAny(upper, SPATIAL_TYPES)) cat = CAT_SPATIAL;
            else if (GEOMETRY_TYPES.has(upper) || isSubtypeOfAny(upper, GEOMETRY_TYPES)) cat = CAT_GEOMETRY;
            else if (HIERARCHY_REL_TYPES.has(upper)) cat = CAT_HIERARCHY_REL;
            else if (PROPERTY_REL_TYPES.has(upper)) cat = CAT_PROPERTY_REL;
            else if (PROPERTY_ENTITY_TYPES.has(upper)) cat = CAT_PROPERTY_ENTITY;
            else if (ASSOCIATION_REL_TYPES.has(upper)) cat = CAT_ASSOCIATION_REL;
            else if (isIfcTypeLikeEntity(upper)) cat = CAT_TYPE_OBJECT;
            else if (
                RELEVANT_NON_PRODUCT_HELPERS.has(upper)
                || isSubtypeOfAny(upper, RELEVANT_PRODUCT_ROOTS)
                || upper.startsWith('IFCREL')
            ) cat = CAT_RELEVANT;
            else cat = CAT_SKIP;
            typeCategoryCache.set(type, cat);
            return cat;
        }

        // Time-based yielding: yield to the main thread every ~80ms so geometry
        // streaming callbacks can fire. This limits main-thread blocking to short
        // bursts that don't starve geometry, while adding minimal overhead (~15 yields
        // × ~1ms each ≈ 15ms total over the full parse).
        const YIELD_INTERVAL_MS = Math.max(16, options.yieldIntervalMs ?? 80);
        let lastYieldTime = performance.now();
        const yieldIfNeeded = async () => {
            const now = performance.now();
            if (now - lastYieldTime >= YIELD_INTERVAL_MS) {
                await yieldToEventLoop();
                lastYieldTime = performance.now();
            }
        };

        emitDiagnostic(`parseLite start: totalEntities=${totalEntities} yieldInterval=${YIELD_INTERVAL_MS}ms`);

        const spatialRefs: EntityRef[] = [];
        const geometryRefs: EntityRef[] = [];
        const relationshipRefs: EntityRef[] = [];
        const propertyRelRefs: EntityRef[] = [];
        const propertyContainerRefs: EntityRef[] = [];
        const propertyAtomRefs: EntityRef[] = [];
        const associationRelRefs: EntityRef[] = [];
        const typeObjectRefs: EntityRef[] = [];
        const otherRelevantRefs: EntityRef[] = [];

        for (let i = 0; i < entityRefs.length; i++) {
            if ((i & 0x3FF) === 0) await yieldIfNeeded();
            const ref = entityRefs[i];
            // Categorize (cached — .toUpperCase() called once per unique type)
            const cat = getCategory(ref.type);
            const typeUpper = cat === CAT_PROPERTY_ENTITY ? getTypeUpper(ref.type) : '';
            // ALL entities must be indexed in byType for on-demand extraction
            // (e.g. IfcGeometricRepresentationContext, IfcSiUnit, IfcMaterialLayer).
            // Only property atoms are optionally deferred for huge-file lazy loading.
            const includeInPrimaryIndex =
                !deferPropertyAtomIndex || cat !== CAT_PROPERTY_ENTITY || PROPERTY_CONTAINER_TYPES.has(typeUpper);
            if (includeInPrimaryIndex) {
                // STEP convention is uppercase entity type names and every
                // downstream consumer (schedule-extractor, property readers,
                // test helpers) keys on uppercase. The tokenizer preserves
                // original case though, so if a STEP writer ever emits
                // mixed-case or lowercase types the index would miss on
                // canonical lookups. Normalise once here — `getTypeUpper`
                // is already cached by type name so the cost is ~0.
                const typeKey = getTypeUpper(ref.type);
                let typeList = byType.get(typeKey);
                if (!typeList) { typeList = []; byType.set(typeKey, typeList); }
                typeList.push(ref.expressId);
            }
            if (cat === CAT_SPATIAL) spatialRefs.push(ref);
            else if (cat === CAT_GEOMETRY) geometryRefs.push(ref);
            else if (cat === CAT_HIERARCHY_REL) relationshipRefs.push(ref);
            else if (cat === CAT_PROPERTY_REL) propertyRelRefs.push(ref);
            else if (cat === CAT_PROPERTY_ENTITY) {
                if (PROPERTY_CONTAINER_TYPES.has(typeUpper)) propertyContainerRefs.push(ref);
                else propertyAtomRefs.push(ref);
            }
            else if (cat === CAT_ASSOCIATION_REL) associationRelRefs.push(ref);
            else if (cat === CAT_TYPE_OBJECT) typeObjectRefs.push(ref);
            else if (cat === CAT_RELEVANT) otherRelevantRefs.push(ref);
        }

        logPhase(`categorize ${totalEntities} → spatial:${spatialRefs.length} geom:${geometryRefs.length} rel:${relationshipRefs.length} propRel:${propertyRelRefs.length} propContainers:${propertyContainerRefs.length} propAtoms:${propertyAtomRefs.length} assocRel:${associationRelRefs.length} type:${typeObjectRefs.length} other:${otherRelevantRefs.length}`);

        // Pre-scan association rels to discover relatingRef target IDs (e.g.
        // IfcClassificationReference, IfcMaterial, IfcDocumentReference).  These
        // entities are typically categorised as CAT_SKIP and would otherwise be
        // missing from the compact index, making on-demand extraction fail.
        const associationTargetIds = new Set<number>();
        for (const ref of associationRelRefs) {
            const result = extractPropertyRelFast(uint8Buffer, ref.byteOffset, ref.byteLength);
            if (result) associationTargetIds.add(result.relatingDef);
        }

        // Collect EntityRefs for association targets that aren't already categorised.
        // Single O(n) pass over entityRefs filtered to the (small) target ID set.
        const alreadyIndexedIds = new Set<number>();
        for (const arr of [spatialRefs, geometryRefs, relationshipRefs, propertyRelRefs,
            propertyContainerRefs, associationRelRefs, typeObjectRefs, otherRelevantRefs,
            ...(deferPropertyAtomIndex ? [] : [propertyAtomRefs])]) {
            for (const r of arr) alreadyIndexedIds.add(r.expressId);
        }
        const extraAssocRefs: EntityRef[] = [];
        for (const ref of entityRefs) {
            if (associationTargetIds.has(ref.expressId) && !alreadyIndexedIds.has(ref.expressId)) {
                extraAssocRefs.push(ref);
            }
        }
        logPhase(`association target pre-scan: ${associationTargetIds.size} targets, ${extraAssocRefs.length} extra refs`);

        // ALL entity refs must be indexed in byId so that on-demand extraction
        // can look up any entity by expressId (e.g. IfcUnitAssignment,
        // IfcGeometricRepresentationContext, IfcSiUnit, IfcLocalPlacement, etc.).
        // Only property atoms are optionally deferred for huge-file lazy loading.
        const indexedRefs = deferPropertyAtomIndex
            ? entityRefs.filter(ref => {
                const cat = getCategory(ref.type);
                return cat !== CAT_PROPERTY_ENTITY || PROPERTY_CONTAINER_TYPES.has(getTypeUpper(ref.type));
              })
            : entityRefs;
        emitDiagnostic(
            `index input: indexedRefs=${indexedRefs.length} deferredPropertyAtoms=${deferPropertyAtomIndex ? propertyAtomRefs.length : 0} extraAssocTargets=${extraAssocRefs.length}`
        );

        // Build compact entity index from only the refs that survive lite parsing.
        // This avoids spending huge-file startup time indexing millions of skipped
        // representation/helper entities that the viewer never queries.
        const compactByIdIndex = await buildCompactEntityIndexAsync(indexedRefs);
        logPhase('compact entity index');

        // Create entity table builder with EXACT capacity (not totalEntities which
        // includes millions of geometry-representation entities we don't store).
        // For a 14M entity file, this reduces allocation from ~546MB to ~20MB.
        const relevantCount = spatialRefs.length + geometryRefs.length + typeObjectRefs.length
            + relationshipRefs.length + otherRelevantRefs.length;
        const entityTableBuilder = new EntityTableBuilder(relevantCount, strings);

        const entityIndex = {
            byId: compactByIdIndex as EntityByIdIndex,
            byType,
        };

        // === TARGETED PARSING using batch byte-level extraction ===
        // Uses 2 TextDecoder.decode() calls total for ALL entity GlobalIds/Names
        // (instead of per-entity calls), and pure byte scanning for relationships.
        options.onProgress?.({ phase: 'parsing entities', percent: 10 });

        const extractor = new EntityExtractor(uint8Buffer);

        // Spatial entities: small count, use extractEntity for full accuracy
        const parsedEntityData = new Map<number, { globalId: string; name: string }>();
        for (const ref of spatialRefs) {
            const entity = extractor.extractEntity(ref);
            if (entity) {
                const attrs = entity.attributes || [];
                parsedEntityData.set(ref.expressId, {
                    globalId: typeof attrs[0] === 'string' ? attrs[0] : '',
                    name: typeof attrs[2] === 'string' ? attrs[2] : '',
                });
            }
        }
        logPhase('spatial entities');

        await yieldIfNeeded();

        // Geometry + type object entities: batch extract GlobalId+Name with 2 TextDecoder calls
        options.onProgress?.({ phase: 'parsing geometry names', percent: 12 });
        const geomData = await batchExtractGlobalIdAndName(uint8Buffer, geometryRefs, yieldIfNeeded);
        for (const [id, data] of geomData) parsedEntityData.set(id, data);

        await yieldIfNeeded();

        const typeData = await batchExtractGlobalIdAndName(uint8Buffer, typeObjectRefs, yieldIfNeeded);
        for (const [id, data] of typeData) parsedEntityData.set(id, data);
        logPhase('batch geom GlobalId+Name');

        await yieldIfNeeded();

        // Relationships: byte-level scanning (numbers only, no TextDecoder)
        options.onProgress?.({ phase: 'parsing relationships', percent: 20 });

        for (let i = 0; i < relationshipRefs.length; i++) {
            if ((i & 0x3FF) === 0) await yieldIfNeeded();
            const ref = relationshipRefs[i];
            const typeUpper = getTypeUpper(ref.type);
            const rel = extractRelFast(uint8Buffer, ref.byteOffset, ref.byteLength, typeUpper);
            if (rel) {
                const relType = REL_TYPE_MAP[typeUpper];
                if (relType) {
                    for (const targetId of rel.relatedObjects) {
                        relationshipGraphBuilder.addEdge(rel.relatingObject, targetId, relType, ref.expressId);
                    }
                }
            }
        }

        logPhase('byte-level relationships');

        // === BUILD ENTITY TABLE from categorized arrays ===
        // Instead of iterating ALL 4.4M entityRefs, iterate only categorized arrays
        // (~100K-200K total). This eliminates a 200-300ms loop over 4.4M items.
        options.onProgress?.({ phase: 'building entities', percent: 30 });

        // Helper to add entities with pre-parsed data
        const addEntityBatch = (refs: EntityRef[], hasGeometry: boolean, isType: boolean) => {
            for (const ref of refs) {
                const entityData = parsedEntityData.get(ref.expressId);
                entityTableBuilder.add(
                    ref.expressId,
                    ref.type,
                    entityData?.globalId || '',
                    entityData?.name || '',
                    '', // description
                    '', // objectType
                    hasGeometry,
                    isType
                );
            }
        };

        addEntityBatch(spatialRefs, false, false);
        addEntityBatch(geometryRefs, true, false);
        addEntityBatch(typeObjectRefs, false, true);
        addEntityBatch(relationshipRefs, false, false);
        addEntityBatch(otherRelevantRefs, false, false);
        logPhase('add entity batches');

        const entityTable = entityTableBuilder.build();
        logPhase('entity table build()');

        // Empty property/quantity tables - use on-demand extraction instead
        const propertyTable = propertyTableBuilder.build();
        const quantityTable = quantityTableBuilder.build();

        // Build intermediate relationship graph (spatial/hierarchy edges only).
        // Property/association edges are added later; final graph is rebuilt at the end.
        const hierarchyRelGraph = relationshipGraphBuilder.build();
        logPhase('hierarchy rel graph build()');

        await yieldIfNeeded();

        // === EXTRACT LENGTH UNIT SCALE ===
        options.onProgress?.({ phase: 'extracting units', percent: 85 });
        const lengthUnitScale = extractLengthUnitScale(uint8Buffer, entityIndex);

        // === BUILD SPATIAL HIERARCHY ===
        options.onProgress?.({ phase: 'building hierarchy', percent: 90 });

        let spatialHierarchy: SpatialHierarchy | undefined;
        try {
            const hierarchyBuilder = new SpatialHierarchyBuilder();
            spatialHierarchy = hierarchyBuilder.build(
                entityTable,
                hierarchyRelGraph,
                strings,
                uint8Buffer,
                entityIndex,
                lengthUnitScale
            );
            logPhase('spatial hierarchy');
        } catch (error) {
            console.warn('[ColumnarParser] Failed to build spatial hierarchy:', error);
        }

        // === EMIT SPATIAL HIERARCHY EARLY ===
        // The hierarchy panel can render immediately while property/association
        // parsing continues. This lets the panel appear at the same time as
        // geometry streaming completes.
        const entitySource = new BufferEntitySource(uint8Buffer, entityIndex);
        const earlyStore: IfcDataStore = {
            fileSize: buffer.byteLength,
            schemaVersion,
            entityCount: totalEntities,
            parseTime: performance.now() - startTime,
            source: uint8Buffer,
            entityIndex,
            strings,
            entities: entityTable,
            properties: propertyTable,
            quantities: quantityTable,
            relationships: hierarchyRelGraph,
            spatialHierarchy,
            lengthUnitScale,
            getEntity(expressId) { return entitySource.getEntity(expressId); },
            getEntitiesByType(typeName) { return entitySource.getEntitiesByType(typeName); },
            getProperties(expressId) { return this.properties.getForEntity(expressId); },
            getQuantities(expressId) { return this.quantities.getForEntity(expressId); },
        };
        options.onSpatialReady?.(earlyStore);

        await yieldIfNeeded(); // Let geometry process after hierarchy emission

        // === DEFERRED: Parse property and association relationships ===
        // These are NOT needed for the spatial hierarchy panel.
        options.onProgress?.({ phase: 'parsing property refs', percent: 92 });

        const onDemandPropertyMap = new Map<number, number[]>();
        const onDemandQuantityMap = new Map<number, number[]>();

        // Pre-build Sets of property set / quantity set IDs from already-categorized refs.
        // This replaces 252K binary searches on the 14M compact entity index with O(1) Set lookups.
        const propertySetIds = new Set<number>();
        const quantitySetIds = new Set<number>();
        for (const ref of propertyContainerRefs) {
            const tu = getTypeUpper(ref.type);
            if (tu === 'IFCPROPERTYSET') propertySetIds.add(ref.expressId);
            else if (tu === 'IFCELEMENTQUANTITY') quantitySetIds.add(ref.expressId);
        }

        // Property rels: byte-level scanning + addEdge (now fast with SoA builder).
        let totalPropRelObjects = 0;
        for (let pi = 0; pi < propertyRelRefs.length; pi++) {
            if ((pi & 0x3FF) === 0) await yieldIfNeeded();
            const ref = propertyRelRefs[pi];
            const result = extractPropertyRelFast(uint8Buffer, ref.byteOffset, ref.byteLength);
            if (result) {
                const { relatedObjects, relatingDef } = result;
                totalPropRelObjects += relatedObjects.length;

                for (const objId of relatedObjects) {
                    relationshipGraphBuilder.addEdge(relatingDef, objId, RelationshipType.DefinesByProperties, ref.expressId);
                }

                // Build on-demand property/quantity maps using pre-built Sets (O(1) vs binary search)
                const isPropSet = propertySetIds.has(relatingDef);
                const isQtySet = !isPropSet && quantitySetIds.has(relatingDef);

                if (isPropSet || isQtySet) {
                    const targetMap = isPropSet ? onDemandPropertyMap : onDemandQuantityMap;
                    for (const objId of relatedObjects) {
                        let list = targetMap.get(objId);
                        if (!list) { list = []; targetMap.set(objId, list); }
                        list.push(relatingDef);
                    }
                }
            }
        }
        await yieldIfNeeded();

        // Association rels: byte-level scanning, no addEdge (same reasoning as property rels)
        options.onProgress?.({ phase: 'parsing associations', percent: 95 });

        const onDemandClassificationMap = new Map<number, number[]>();
        const onDemandMaterialMap = new Map<number, number>();
        const onDemandDocumentMap = new Map<number, number[]>();

        for (let i = 0; i < associationRelRefs.length; i++) {
            if ((i & 0x3FF) === 0) await yieldIfNeeded();
            const ref = associationRelRefs[i];
            const result = extractPropertyRelFast(uint8Buffer, ref.byteOffset, ref.byteLength);
            if (result) {
                const { relatedObjects, relatingDef: relatingRef } = result;
                const typeUpper = getTypeUpper(ref.type);

                if (typeUpper === 'IFCRELASSOCIATESCLASSIFICATION') {
                    for (const objId of relatedObjects) {
                        let list = onDemandClassificationMap.get(objId);
                        if (!list) { list = []; onDemandClassificationMap.set(objId, list); }
                        list.push(relatingRef);
                        relationshipGraphBuilder.addEdge(relatingRef, objId, RelationshipType.AssociatesClassification, ref.expressId);
                    }
                } else if (typeUpper === 'IFCRELASSOCIATESMATERIAL') {
                    for (const objId of relatedObjects) {
                        onDemandMaterialMap.set(objId, relatingRef);
                        relationshipGraphBuilder.addEdge(relatingRef, objId, RelationshipType.AssociatesMaterial, ref.expressId);
                    }
                } else if (typeUpper === 'IFCRELASSOCIATESDOCUMENT') {
                    for (const objId of relatedObjects) {
                        let list = onDemandDocumentMap.get(objId);
                        if (!list) { list = []; onDemandDocumentMap.set(objId, list); }
                        list.push(relatingRef);
                        relationshipGraphBuilder.addEdge(relatingRef, objId, RelationshipType.AssociatesDocument, ref.expressId);
                    }
                }
            }
        }

        logPhase('property+association rels');

        // Rebuild relationship graph with ALL edges (hierarchy + property + association)
        const fullRelationshipGraph = relationshipGraphBuilder.build();
        logPhase('relationship graph build()');

        let deferredEntityIndex: EntityByIdIndex | undefined;
        if (deferPropertyAtomIndex && propertyAtomRefs.length > 0) {
            options.onProgress?.({ phase: 'indexing property atoms', percent: 98 });
            deferredEntityIndex = await buildCompactEntityIndexAsync(
                propertyAtomRefs,
                undefined,
                1024,
                2,
            );
            logPhase('deferred property atom index');
        }

        const parseTime = performance.now() - startTime;
        options.onProgress?.({ phase: 'complete', percent: 100 });

        const finalStore: IfcDataStore = {
            ...earlyStore,
            parseTime,
            relationships: fullRelationshipGraph,
            deferredEntityIndex,
            onDemandPropertyMap,
            onDemandQuantityMap,
            onDemandClassificationMap,
            onDemandMaterialMap,
            onDemandDocumentMap,
            lengthUnitScale,
            getEntity(expressId) { return entitySource.getEntity(expressId); },
            getEntitiesByType(typeName) { return entitySource.getEntitiesByType(typeName); },
            getProperties(expressId) {
                if (onDemandPropertyMap.size > 0) return extractPropertiesOnDemand(this as IfcDataStore, expressId) as PropertySet[];
                return this.properties.getForEntity(expressId);
            },
            getQuantities(expressId) {
                if (onDemandQuantityMap.size > 0) return extractQuantitiesOnDemand(this as IfcDataStore, expressId) as QuantitySet[];
                return this.quantities.getForEntity(expressId);
            },
        };
        return finalStore;
    }

    /**
     * Extract properties for a single entity ON-DEMAND
     * Parses only what's needed from the source buffer - instant results.
     */
    extractPropertiesOnDemand(
        store: IfcDataStore,
        entityId: number
    ): Array<{ name: string; globalId?: string; properties: Array<{ name: string; type: number; value: PropertyValue; values?: string[]; dataType?: string }> }> {
        // Use on-demand extraction if map is available (preferred for single-entity access)
        if (!store.onDemandPropertyMap || !store.source?.length) {
            // Fallback to pre-computed property table (e.g., server-parsed data)
            return store.properties.getForEntity(entityId);
        }

        const psetIds = store.onDemandPropertyMap.get(entityId);
        if (!psetIds || psetIds.length === 0) {
            return [];
        }

        const extractor = new EntityExtractor(store.source);
        const result: Array<{ name: string; globalId?: string; properties: Array<{ name: string; type: number; value: PropertyValue; values?: string[]; dataType?: string }> }> = [];

        for (const psetId of psetIds) {
            const psetRef = getEntityRefFromStore(store, psetId);
            if (!psetRef) continue;

            const psetEntity = extractor.extractEntity(psetRef);
            if (!psetEntity) continue;

            const psetAttrs = psetEntity.attributes || [];
            const psetGlobalId = typeof psetAttrs[0] === 'string' ? psetAttrs[0] : undefined;
            const psetName = typeof psetAttrs[2] === 'string' ? psetAttrs[2] : `PropertySet #${psetId}`;
            const hasProperties = psetAttrs[4];

            const properties: Array<{ name: string; type: number; value: PropertyValue; values?: string[]; dataType?: string }> = [];

            if (Array.isArray(hasProperties)) {
                for (const propRef of hasProperties) {
                    if (typeof propRef !== 'number') continue;

                    const propEntityRef = getEntityRefFromStore(store, propRef);
                    if (!propEntityRef) continue;

                    const propEntity = extractor.extractEntity(propEntityRef);
                    if (!propEntity) continue;

                    const propAttrs = propEntity.attributes || [];
                    const propName = typeof propAttrs[0] === 'string' ? propAttrs[0] : '';
                    if (!propName) continue;

                    const parsed = parsePropertyValue(propEntity);
                    const entry: { name: string; type: number; value: PropertyValue; values?: string[]; dataType?: string } = {
                        name: propName,
                        type: parsed.type,
                        value: parsed.value,
                    };
                    if (parsed.values) entry.values = parsed.values;
                    if (parsed.dataType) entry.dataType = parsed.dataType;
                    properties.push(entry);
                }
            }

            if (properties.length > 0 || psetName) {
                result.push({ name: psetName, globalId: psetGlobalId, properties });
            }
        }

        return result;
    }

    /**
     * Extract quantities for a single entity ON-DEMAND
     * Parses only what's needed from the source buffer - instant results.
     */
    extractQuantitiesOnDemand(
        store: IfcDataStore,
        entityId: number
    ): Array<{ name: string; quantities: Array<{ name: string; type: number; value: number }> }> {
        // Use on-demand extraction if map is available (preferred for single-entity access)
        if (!store.onDemandQuantityMap || !store.source?.length) {
            // Fallback to pre-computed quantity table (e.g., server-parsed data)
            return store.quantities.getForEntity(entityId);
        }

        const qsetIds = store.onDemandQuantityMap.get(entityId);
        if (!qsetIds || qsetIds.length === 0) {
            return [];
        }

        const extractor = new EntityExtractor(store.source);
        const result: Array<{ name: string; quantities: Array<{ name: string; type: number; value: number }> }> = [];

        for (const qsetId of qsetIds) {
            const qsetRef = getEntityRefFromStore(store, qsetId);
            if (!qsetRef) continue;

            const qsetEntity = extractor.extractEntity(qsetRef);
            if (!qsetEntity) continue;

            const qsetAttrs = qsetEntity.attributes || [];
            const qsetName = typeof qsetAttrs[2] === 'string' ? qsetAttrs[2] : `QuantitySet #${qsetId}`;
            const hasQuantities = qsetAttrs[5];

            const quantities: Array<{ name: string; type: number; value: number }> = [];

            if (Array.isArray(hasQuantities)) {
                for (const qtyRef of hasQuantities) {
                    if (typeof qtyRef !== 'number') continue;

                    const qtyEntityRef = getEntityRefFromStore(store, qtyRef);
                    if (!qtyEntityRef) continue;

                    const qtyEntity = extractor.extractEntity(qtyEntityRef);
                    if (!qtyEntity) continue;

                    const qtyAttrs = qtyEntity.attributes || [];
                    const qtyName = typeof qtyAttrs[0] === 'string' ? qtyAttrs[0] : '';
                    if (!qtyName) continue;

                    // Get quantity type from entity type
                    const qtyTypeUpper = qtyEntity.type.toUpperCase();
                    const qtyType = QUANTITY_TYPE_MAP[qtyTypeUpper] ?? QuantityType.Count;

                    // Value is at index 3 for most quantity types
                    const value = typeof qtyAttrs[3] === 'number' ? qtyAttrs[3] : 0;

                    quantities.push({ name: qtyName, type: qtyType, value });
                }
            }

            if (quantities.length > 0 || qsetName) {
                result.push({ name: qsetName, quantities });
            }
        }

        return result;
    }
}

/**
 * Standalone on-demand property extractor
 * Can be used outside ColumnarParser class
 */
export function extractPropertiesOnDemand(
    store: IfcDataStore,
    entityId: number
): Array<{ name: string; globalId?: string; properties: Array<{ name: string; type: number; value: PropertyValue; values?: string[] }> }> {
    const parser = new ColumnarParser();
    return parser.extractPropertiesOnDemand(store, entityId);
}

/**
 * Standalone on-demand quantity extractor
 * Can be used outside ColumnarParser class
 */
export function extractQuantitiesOnDemand(
    store: IfcDataStore,
    entityId: number
): Array<{ name: string; quantities: Array<{ name: string; type: number; value: number }> }> {
    const parser = new ColumnarParser();
    return parser.extractQuantitiesOnDemand(store, entityId);
}

function getEntityRefFromStore(store: IfcDataStore, expressId: number): EntityRef | undefined {
    return store.entityIndex.byId.get(expressId) ?? store.deferredEntityIndex?.get(expressId);
}

/**
 * Extract entity attributes on-demand from source buffer.
 * Returns globalId, name, description, objectType, tag mapped by schema name
 * (see {@link extractRootAttributesFromEntity}), so the result stays correct
 * for entity types whose attribute order differs from the IfcElement layout.
 * This is used for entities that weren't fully parsed during initial load.
 */
export function extractEntityAttributesOnDemand(
    store: IfcDataStore,
    entityId: number
): { globalId: string; name: string; description: string; objectType: string; tag: string } {
    const ref = store.entityIndex.byId.get(entityId);
    if (!ref) {
        return { globalId: '', name: '', description: '', objectType: '', tag: '' };
    }

    const extractor = new EntityExtractor(store.source);
    const entity = extractor.extractEntity(ref);
    if (!entity) {
        return { globalId: '', name: '', description: '', objectType: '', tag: '' };
    }

    return extractRootAttributesFromEntity(entity);
}

/**
 * Extract ALL named entity attributes on-demand from source buffer.
 * Uses the IFC schema to map attribute indices to names.
 * Returns only string/enum attributes, skipping references and structural attributes.
 */
export function extractAllEntityAttributes(
    store: IfcDataStore,
    entityId: number
): Array<{ name: string; value: string | number | boolean }> {
    const ref = store.entityIndex.byId.get(entityId);
    if (!ref) return [];

    const extractor = new EntityExtractor(store.source);
    const entity = extractor.extractEntity(ref);
    if (!entity) return [];

    const attrs = entity.attributes || [];
    // Use properly-cased type name from entity table (IfcTypeEnumToString)
    // instead of ref.type which is UPPERCASE from STEP (e.g., IFCWALLSTANDARDCASE)
    // and breaks multi-word type normalization in getAttributeNames.
    // For resource-level entities (IfcTask, IfcTaskTime, IfcMaterial,
    // IfcClassification, ...) the entity table returns 'Unknown';
    // fall back to ref.type so the schema-driven attribute-name
    // resolution still works for those types.
    const tableName = store.entities.getTypeName(entityId);
    const typeName = tableName && tableName !== 'Unknown' ? tableName : ref.type;
    const attrNames = getAttributeNames(typeName);

    const result: Array<{ name: string; value: string | number | boolean }> = [];
    const len = Math.min(attrs.length, attrNames.length);
    for (let i = 0; i < len; i++) {
        const attrName = attrNames[i];
        if (SKIP_DISPLAY_ATTRS.has(attrName)) continue;

        const raw = attrs[i];
        // STEP `$` (unset) and `*` (derived) deserialize as null /
        // undefined and must be skipped. Strings are emitted with
        // `.ENUM.` markers stripped. Empty strings are preserved so
        // IDS optional-attribute checks can distinguish "slot truly
        // absent" from "slot explicitly empty". Numbers and booleans
        // pass through unchanged so e.g. `CountValue = 0` reads as
        // present.
        if (typeof raw === 'string') {
            // STEP logical-unknown markers (`.U.`, `.X.`) read as
            // "no value" per IDS spec — they fail any attribute
            // check, including a bare existence check, so don't
            // surface them as if the slot were populated.
            if (raw === '.U.' || raw === '.X.') continue;
            // Bare boolean tokens (`.T.` / `.F.`) on a schema-typed
            // IfcBoolean attribute slot — resolve to JS boolean so
            // IDS checks comparing against `true` / `false` literals
            // pass without case-sensitive string contortions.
            if (raw === '.T.') {
                result.push({ name: attrName, value: true });
                continue;
            }
            if (raw === '.F.') {
                result.push({ name: attrName, value: false });
                continue;
            }
            const display = raw.startsWith('.') && raw.endsWith('.')
                ? raw.slice(1, -1)
                : raw;
            result.push({ name: attrName, value: display });
        } else if (typeof raw === 'number' || typeof raw === 'boolean') {
            result.push({ name: attrName, value: raw });
        } else if (Array.isArray(raw) && raw.length === 2) {
            // Typed STEP values like IFCREAL(0.0), IFCBOOLEAN(.T.) —
            // return the underlying primitive so attribute existence
            // and value checks can compare it directly.
            const inner = raw[1];
            const tag = String(raw[0]).toUpperCase();
            if (tag.includes('BOOLEAN')) {
                result.push({ name: attrName, value: inner === '.T.' || inner === true });
            } else if (tag.includes('LOGICAL')) {
                if (inner === '.U.' || inner === '.X.') {
                    // UNKNOWN logical → don't surface (treated as absent)
                    continue;
                }
                result.push({ name: attrName, value: inner === '.T.' || inner === true });
            } else if (typeof inner === 'number' || typeof inner === 'boolean') {
                result.push({ name: attrName, value: inner });
            } else if (typeof inner === 'string' && inner) {
                const display = inner.startsWith('.') && inner.endsWith('.')
                    ? inner.slice(1, -1)
                    : inner;
                result.push({ name: attrName, value: display });
            }
        }
    }

    return result;
}

/**
 * Returns named raw attribute pairs for an entity, filtered to display-relevant attributes.
 * Skips structural/reference attributes using the IFC schema. Used by query layer for coercion.
 */
export function getRawNamedAttributes(
    entity: IfcEntity
): Array<{ name: string; raw: IfcAttributeValue }> {
    const attrs = entity.attributes || [];
    const attrNames = getAttributeNames(entity.type);

    const result: Array<{ name: string; raw: IfcAttributeValue }> = [];
    const len = Math.min(attrs.length, attrNames.length);
    for (let i = 0; i < len; i++) {
        const attrName = attrNames[i];
        if (SKIP_DISPLAY_ATTRS.has(attrName)) continue;
        result.push({ name: attrName, raw: attrs[i] });
    }
    return result;
}

interface RootAttrIndices {
    known: boolean;
    globalId: number;
    name: number;
    description: number;
    objectType: number;
    tag: number;
}

// getAttributeNames() walks the schema registry (an O(types) scan for the
// UPPERCASE STEP names entities carry), so memoise the per-type index lookup.
// There are only a few hundred distinct types but potentially millions of
// entities, keeping the on-demand path cheap even when called per entity.
const rootAttrIndexCache = new Map<string, RootAttrIndices>();

function getRootAttrIndices(type: string): RootAttrIndices {
    let idx = rootAttrIndexCache.get(type);
    if (!idx) {
        const names = getAttributeNames(type);
        idx = {
            known: names.length > 0,
            globalId: names.indexOf('GlobalId'),
            name: names.indexOf('Name'),
            description: names.indexOf('Description'),
            objectType: names.indexOf('ObjectType'),
            tag: names.indexOf('Tag'),
        };
        rootAttrIndexCache.set(type, idx);
    }
    return idx;
}

/**
 * Resolve the common IfcRoot-family display attributes (GlobalId, Name,
 * Description, ObjectType, Tag) from an entity's raw attribute array.
 *
 * These are mapped by schema-derived attribute *name*, not fixed index. The
 * fixed indices `[0],[2],[3],[4],[7]` only hold for the IfcElement layout: for
 * a spatial element `attrs[7]` is LongName (not Tag), and for a resource entity
 * like IfcMaterial `attrs[0]` is Name (not GlobalId). Name-mapping keeps all of
 * these correct for every entity type, returning '' for attributes the type
 * does not declare.
 *
 * For types the schema registry does not recognise (e.g. an IFC4x3 infra leaf
 * outside the codegen pin, or a vendor extension) we fall back to the canonical
 * IfcRoot/IfcElement positions so we never regress vs. the old fixed-index path.
 */
export function extractRootAttributesFromEntity(
    entity: IfcEntity
): { globalId: string; name: string; description: string; objectType: string; tag: string } {
    const attrs = entity.attributes || [];
    const idx = getRootAttrIndices(entity.type);
    const pick = (schemaIndex: number, fallbackIndex: number): string => {
        const i = idx.known ? schemaIndex : fallbackIndex;
        const raw = i >= 0 ? attrs[i] : undefined;
        return typeof raw === 'string' ? raw : '';
    };
    return {
        globalId: pick(idx.globalId, 0),
        name: pick(idx.name, 2),
        description: pick(idx.description, 3),
        objectType: pick(idx.objectType, 4),
        tag: pick(idx.tag, 7),
    };
}

// Re-export on-demand extraction functions from focused module
export {
    extractClassificationsOnDemand,
    extractMaterialsOnDemand,
    extractMaterialPropertiesOnDemand,
    extractMaterialPropertiesForMaterialId,
    resolveMaterialDefId,
    collectMaterialLeaves,
    buildMaterialUsageIndex,
    getMaterialDisplay,
    extractTypePropertiesOnDemand,
    extractTypeEntityOwnProperties,
    extractDocumentsOnDemand,
    extractRelationshipsOnDemand,
    extractGeoreferencingOnDemand,
    parsePropertyValue,
    extractPsetsFromIds,
} from './on-demand-extractors.js';

export type {
    ClassificationInfo,
    MaterialInfo,
    MaterialLayerInfo,
    MaterialProfileInfo,
    MaterialConstituentInfo,
    MaterialPsetGroup,
    MaterialLeaf,
    MaterialUsage,
    TypePropertyInfo,
    DocumentInfo,
    EntityRelationships,
    GeorefInfo,
} from './on-demand-extractors.js';
