osmpbfparser.cpp

#include "osmpbfparser.h"

#include <QFile>
#include <QVector>
#include <QHash>
#include <QDebug>
#include <QThreadPool>
#include <QRunnable>
#include <QElapsedTimer>
#include <QFuture>
#include <QtConcurrent>

#include <zlib.h>
#include <cstdint>
#include <cstring>
#include <utility>
#include <atomic>

namespace Pb {

enum WireType { VARINT = 0, FIXED64 = 1, LENDELIM = 2, FIXED32 = 5 };

struct Tag {
    uint32_t fieldNumber;
    WireType wireType;
};

static inline bool readVarint(const uint8_t *&p, const uint8_t *end, uint64_t &out) {
    out = 0;
    int shift = 0;
    while (p < end) {
        uint8_t b = *p++;
        out |= (uint64_t)(b & 0x7F) << shift;
        if (!(b & 0x80)) return true;
        shift += 7;
        if (shift >= 64) return false;
    }
    return false;
}

static inline bool readTag(const uint8_t *&p, const uint8_t *end, Tag &tag) {
    uint64_t v;
    if (!readVarint(p, end, v)) return false;
    tag.fieldNumber = (uint32_t)(v >> 3);
    tag.wireType = (WireType)(v & 7);
    return true;
}

static inline bool skipValue(const uint8_t *&p, const uint8_t *end, WireType wt) {
    switch (wt) {
    case VARINT: { uint64_t v; return readVarint(p, end, v); }
    case FIXED64: if (end - p < 8) return false; p += 8; return true;
    case LENDELIM: {
        uint64_t len;
        if (!readVarint(p, end, len)) return false;
        if ((uint64_t)(end - p) < len) return false;
        p += len;
        return true;
    }
    case FIXED32: if (end - p < 4) return false; p += 4; return true;
    default: return false;
    }
}

static inline bool readBytes(const uint8_t *&p, const uint8_t *end, QByteArray &out) {
    uint64_t len;
    if (!readVarint(p, end, len)) return false;
    if ((uint64_t)(end - p) < len) return false;
    out = QByteArray((const char*)p, (int)len);
    p += len;
    return true;
}

static inline int64_t zigZagDecode64(uint64_t n) {
    return (int64_t)(n >> 1) ^ -(int64_t)(n & 1);
}

}

static QByteArray zlibDecompress(const QByteArray &in, int expectedSize) {
    if (in.isEmpty()) return QByteArray();
    QByteArray out;
    out.resize(expectedSize > 0 ? expectedSize : qMax(4096, in.size() * 4));
    z_stream strm;
    memset(&strm, 0, sizeof(strm));
    if (inflateInit(&strm) != Z_OK) return QByteArray();
    strm.next_in  = (Bytef*)in.constData();
    strm.avail_in = (uInt)in.size();
    for (;;) {
        strm.next_out  = (Bytef*)out.data() + strm.total_out;
        strm.avail_out = (uInt)(out.size() - strm.total_out);
        if (strm.avail_out == 0) {
            out.resize(out.size() * 2);
            continue;
        }
        int ret = inflate(&strm, Z_NO_FLUSH);
        if (ret == Z_STREAM_END) break;
        if (ret != Z_OK) { inflateEnd(&strm); return QByteArray(); }
    }
    inflateEnd(&strm);
    out.resize((int)strm.total_out);
    return out;
}

struct BlobHeader {
    QByteArray typeBytes;
    int32_t datasize;
};

static bool parseBlobHeader(const QByteArray &data, BlobHeader &hdr) {
    const uint8_t *p = (const uint8_t*)data.constData();
    const uint8_t *end = p + data.size();
    hdr.datasize = 0;
    while (p < end) {
        Pb::Tag tag;
        if (!Pb::readTag(p, end, tag)) return false;
        switch (tag.fieldNumber) {
        case 1: { QByteArray b; if (!Pb::readBytes(p, end, b)) return false; hdr.typeBytes = b; break; }
        case 2: Pb::skipValue(p, end, tag.wireType); break;
        case 3: { uint64_t v; if (!Pb::readVarint(p, end, v)) return false; hdr.datasize = (int32_t)v; break; }
        default: if (!Pb::skipValue(p, end, tag.wireType)) return false;
        }
    }
    return true;
}

struct Blob {
    QByteArray raw;
    int32_t raw_size;
    QByteArray zlib_data;
};

static bool parseBlob(const QByteArray &data, Blob &blob) {
    const uint8_t *p = (const uint8_t*)data.constData();
    const uint8_t *end = p + data.size();
    blob.raw_size = 0;
    while (p < end) {
        Pb::Tag tag;
        if (!Pb::readTag(p, end, tag)) return false;
        switch (tag.fieldNumber) {
        case 1: if (!Pb::readBytes(p, end, blob.raw)) return false; break;
        case 2: { uint64_t v; if (!Pb::readVarint(p, end, v)) return false; blob.raw_size = (int32_t)v; break; }
        case 3: if (!Pb::readBytes(p, end, blob.zlib_data)) return false; break;
        default: if (!Pb::skipValue(p, end, tag.wireType)) return false;
        }
    }
    return true;
}

static QByteArray decompressBlob(const Blob &blob) {
    if (!blob.raw.isEmpty()) return blob.raw;
    if (!blob.zlib_data.isEmpty()) {
        return zlibDecompress(blob.zlib_data, blob.raw_size > 0 ? blob.raw_size : blob.zlib_data.size() * 4);
    }
    return QByteArray();
}

static QByteArrayList parseStringTable(const QByteArray &data) {
    QByteArrayList table;
    const uint8_t *p = (const uint8_t*)data.constData();
    const uint8_t *end = p + data.size();
    while (p < end) {
        Pb::Tag tag;
        if (!Pb::readTag(p, end, tag)) break;
        if (tag.fieldNumber == 1 && tag.wireType == Pb::LENDELIM) {
            QByteArray ba;
            if (!Pb::readBytes(p, end, ba)) break;
            table.append(ba);
        } else {
            if (!Pb::skipValue(p, end, tag.wireType)) break;
        }
    }
    return table;
}

static const int64_t DEFAULT_GRANULARITY = 100;
static const int64_t LATLON_OFFSET = 0;

static QVector<double> parsePackedDeltaDegrees(const QByteArray &buf, int64_t granularity, int64_t offset) {
    QVector<double> result;
    const uint8_t *bp = (const uint8_t*)buf.constData();
    const uint8_t *bend = bp + buf.size();
    int64_t cur = 0;
    while (bp < bend) {
        uint64_t v;
        if (!Pb::readVarint(bp, bend, v)) break;
        cur += Pb::zigZagDecode64(v);
        result.append((offset + granularity * cur) * 1e-9);
    }
    return result;
}

static QVector<qint64> parsePackedDeltaIds(const QByteArray &buf) {
    QVector<qint64> result;
    const uint8_t *bp = (const uint8_t*)buf.constData();
    const uint8_t *bend = bp + buf.size();
    qint64 cur = 0;
    while (bp < bend) {
        uint64_t v;
        if (!Pb::readVarint(bp, bend, v)) break;
        cur += Pb::zigZagDecode64(v);
        result.append(cur);
    }
    return result;
}

static bool parsePrimitiveBlockToTemp(const QByteArray &blockData, ParsedBlock &result)
{
    const uint8_t *p = (const uint8_t*)blockData.constData();
    const uint8_t *end = p + blockData.size();

    QByteArray stringTableData;
    int64_t granularity = DEFAULT_GRANULARITY;
    int64_t latOffset = LATLON_OFFSET;
    int64_t lonOffset = LATLON_OFFSET;
    QList<QByteArray> groupDatas;

    while (p < end) {
        Pb::Tag tag;
        if (!Pb::readTag(p, end, tag)) return false;
        switch (tag.fieldNumber) {
        case 1: { QByteArray b; if (!Pb::readBytes(p, end, b)) return false; stringTableData = b; break; }
        case 2: { QByteArray b; if (!Pb::readBytes(p, end, b)) return false; groupDatas.append(b); break; }
        case 17: { uint64_t v; if (!Pb::readVarint(p, end, v)) return false; granularity = (int32_t)v; break; }
        case 19: { uint64_t v; if (!Pb::readVarint(p, end, v)) return false; latOffset = (int64_t)v; break; }
        case 20: { uint64_t v; if (!Pb::readVarint(p, end, v)) return false; lonOffset = (int64_t)v; break; }
        case 18: { uint64_t v; if (!Pb::readVarint(p, end, v)) return false; break; }
        default: if (!Pb::skipValue(p, end, tag.wireType)) return false;
        }
    }

    QByteArrayList stringTable = parseStringTable(stringTableData);
    auto addTempTag = [&](const QString &keyStr, const QString &val, int &head) {
        result.tags.push_back({keyStr, val, head});
        head = result.tags.size() - 1;
    };
    auto addTempNd = [&](qint64 nodeId, int &head) {
        result.nds.push_back({nodeId, head});
        head = result.nds.size() - 1;
    };
    auto addTempMember = [&](int type, qint64 ref, const QString &role, int &head) {
        result.members.push_back({type, ref, role, head});
        head = result.members.size() - 1;
    };

    for (const QByteArray &gdata : std::as_const(groupDatas)) {
        const uint8_t *gp = (const uint8_t*)gdata.constData();
        const uint8_t *gend = gp + gdata.size();

        QList<QByteArray> nodeMsgs, denseMsgs, wayMsgs, relMsgs;

        while (gp < gend) {
            Pb::Tag gtag;
            if (!Pb::readTag(gp, gend, gtag)) return false;
            switch (gtag.fieldNumber) {
            case 1: { QByteArray b; if (!Pb::readBytes(gp, gend, b)) return false; nodeMsgs.append(b); break; }
            case 2: { QByteArray b; if (!Pb::readBytes(gp, gend, b)) return false; denseMsgs.append(b); break; }
            case 3: { QByteArray b; if (!Pb::readBytes(gp, gend, b)) return false; wayMsgs.append(b); break; }
            case 4: { QByteArray b; if (!Pb::readBytes(gp, gend, b)) return false; relMsgs.append(b); break; }
            case 5: Pb::skipValue(gp, gend, gtag.wireType); break;
            default: if (!Pb::skipValue(gp, gend, gtag.wireType)) return false;
            }
        }

        for (const QByteArray &nd : std::as_const(nodeMsgs)) {
            const uint8_t *np = (const uint8_t*)nd.constData();
            const uint8_t *nend = np + nd.size();
            qint64 id = 0; double lat = 0, lon = 0;
            QVector<uint32_t> keys, vals;
            while (np < nend) {
                Pb::Tag t;
                if (!Pb::readTag(np, nend, t)) return false;
                switch (t.fieldNumber) {
                case 1: { uint64_t v; if (!Pb::readVarint(np, nend, v)) return false; id = Pb::zigZagDecode64(v); break; }
                case 8: case 9: {
                    QByteArray b;
                    if (!Pb::readBytes(np, nend, b)) return false;
                    const uint8_t *bp = (const uint8_t*)b.constData();
                    const uint8_t *bend = bp + b.size();
                    auto &out = (t.fieldNumber == 8) ? keys : vals;
                    while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; out.append((uint32_t)v); }
                    break;
                }
                case 19: { uint64_t v; if (!Pb::readVarint(np, nend, v)) return false; lat = (latOffset + granularity * Pb::zigZagDecode64(v)) * 1e-9; break; }
                case 20: { uint64_t v; if (!Pb::readVarint(np, nend, v)) return false; lon = (lonOffset + granularity * Pb::zigZagDecode64(v)) * 1e-9; break; }
                default: if (!Pb::skipValue(np, nend, t.wireType)) return false;
                }
            }
            OsmNode node(id, lat, lon);
            int tagHead = -1;
            int tagCount = qMin(keys.size(), vals.size());
            for (int i = tagCount - 1; i >= 0; i--) {
                if (keys[i] >= (uint32_t)stringTable.size() || vals[i] >= (uint32_t)stringTable.size()) continue;
                QString keyStr = QString::fromUtf8(stringTable[keys[i]]);
                QString val = QString::fromUtf8(stringTable[vals[i]]);
                addTempTag(keyStr, val, tagHead);
            }
            node.setTagHead(tagHead);
            result.nodes.append(node);
            if (lat < result.latMin) result.latMin = lat;
            if (lat > result.latMax) result.latMax = lat;
            if (lon < result.lonMin) result.lonMin = lon;
            if (lon > result.lonMax) result.lonMax = lon;
        }

        for (const QByteArray &dn : std::as_const(denseMsgs)) {
            const uint8_t *dp = (const uint8_t*)dn.constData();
            const uint8_t *dend = dp + dn.size();
            QByteArray idBuf, latBuf, lonBuf, kvBuf;
            while (dp < dend) {
                Pb::Tag t;
                if (!Pb::readTag(dp, dend, t)) return false;
                switch (t.fieldNumber) {
                case 1: if (!Pb::readBytes(dp, dend, idBuf)) return false; break;
                case 8: if (!Pb::readBytes(dp, dend, latBuf)) return false; break;
                case 9: if (!Pb::readBytes(dp, dend, lonBuf)) return false; break;
                case 10: if (!Pb::readBytes(dp, dend, kvBuf)) return false; break;
                default: if (!Pb::skipValue(dp, dend, t.wireType)) return false;
                }
            }
            QVector<qint64> ids = parsePackedDeltaIds(idBuf);
            QVector<double> lats = parsePackedDeltaDegrees(latBuf, granularity, latOffset);
            QVector<double> lons = parsePackedDeltaDegrees(lonBuf, granularity, lonOffset);
            int count = qMin(ids.size(), qMin(lats.size(), lons.size()));
            result.nodes.reserve(result.nodes.size() + count);

            QVector<int> nodeTagHeads(count, -1);
            if (!kvBuf.isEmpty()) {
                const uint8_t *kvp = (const uint8_t*)kvBuf.constData();
                const uint8_t *kvend = kvp + kvBuf.size();
                int nodeIdx = 0;
                int currentTagHead = -1;
                while (kvp < kvend && nodeIdx < count) {
                    uint64_t v;
                    if (!Pb::readVarint(kvp, kvend, v)) break;
                    uint32_t k = (uint32_t)v;
                    if (k == 0) {
                        nodeTagHeads[nodeIdx] = currentTagHead;
                        currentTagHead = -1;
                        nodeIdx++;
                        continue;
                    }
                    if (!Pb::readVarint(kvp, kvend, v)) break;
                    uint32_t vi = (uint32_t)v;
                    if (k < (uint32_t)stringTable.size() && vi < (uint32_t)stringTable.size()) {
                        QString keyStr = QString::fromUtf8(stringTable[k]);
                        QString valStr = QString::fromUtf8(stringTable[vi]);
                        addTempTag(keyStr, valStr, currentTagHead);
                    }
                }
                while (nodeIdx < count) {
                    nodeTagHeads[nodeIdx] = currentTagHead;
                    currentTagHead = -1;
                    nodeIdx++;
                }
            }

            for (int i = 0; i < count; i++) {
                OsmNode node(ids[i], lats[i], lons[i]);
                node.setTagHead(nodeTagHeads[i]);
                result.nodes.append(node);
                if (lats[i] < result.latMin) result.latMin = lats[i];
                if (lats[i] > result.latMax) result.latMax = lats[i];
                if (lons[i] < result.lonMin) result.lonMin = lons[i];
                if (lons[i] > result.lonMax) result.lonMax = lons[i];
            }
        }

        for (const QByteArray &wd : std::as_const(wayMsgs)) {
            const uint8_t *wp = (const uint8_t*)wd.constData();
            const uint8_t *wend = wp + wd.size();
            qint64 id = 0;
            QByteArray keysBuf, valsBuf, refsBuf;
            while (wp < wend) {
                Pb::Tag t;
                if (!Pb::readTag(wp, wend, t)) return false;
                switch (t.fieldNumber) {
                case 1: { uint64_t v; if (!Pb::readVarint(wp, wend, v)) return false; id = Pb::zigZagDecode64(v); break; }
                case 2: if (!Pb::readBytes(wp, wend, keysBuf)) return false; break;
                case 3: if (!Pb::readBytes(wp, wend, valsBuf)) return false; break;
                case 8: if (!Pb::readBytes(wp, wend, refsBuf)) return false; break;
                default: if (!Pb::skipValue(wp, wend, t.wireType)) return false;
                }
            }

            OsmWay way(id);
            QVector<qint64> refs = parsePackedDeltaIds(refsBuf);
            int ndHead = -1;
            for (int i = refs.size() - 1; i >= 0; i--) {
                addTempNd(refs[i], ndHead);
            }
            way.setNdHead(ndHead);

            QVector<uint32_t> keys, vals;
            {
                const uint8_t *bp = (const uint8_t*)keysBuf.constData();
                const uint8_t *bend = bp + keysBuf.size();
                while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; keys.append((uint32_t)v); }
            }
            {
                const uint8_t *bp = (const uint8_t*)valsBuf.constData();
                const uint8_t *bend = bp + valsBuf.size();
                while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; vals.append((uint32_t)v); }
            }

            int tagHead = -1;
            int tagCount = qMin(keys.size(), vals.size());
            for (int i = tagCount - 1; i >= 0; i--) {
                if (keys[i] >= (uint32_t)stringTable.size() || vals[i] >= (uint32_t)stringTable.size()) continue;
                QString keyStr = QString::fromUtf8(stringTable[keys[i]]);
                QString v = QString::fromUtf8(stringTable[vals[i]]);
                addTempTag(keyStr, v, tagHead);
            }
            way.setTagHead(tagHead);
            result.ways.append(way);
        }

        for (const QByteArray &rd : std::as_const(relMsgs)) {
            const uint8_t *rp = (const uint8_t*)rd.constData();
            const uint8_t *rend = rp + rd.size();
            qint64 id = 0;
            QByteArray keysBuf, valsBuf, rolesBuf, memidsBuf, typesBuf;
            while (rp < rend) {
                Pb::Tag t;
                if (!Pb::readTag(rp, rend, t)) return false;
                switch (t.fieldNumber) {
                case 1: { uint64_t v; if (!Pb::readVarint(rp, rend, v)) return false; id = Pb::zigZagDecode64(v); break; }
                case 2: if (!Pb::readBytes(rp, rend, keysBuf)) return false; break;
                case 3: if (!Pb::readBytes(rp, rend, valsBuf)) return false; break;
                case 8: if (!Pb::readBytes(rp, rend, rolesBuf)) return false; break;
                case 9: if (!Pb::readBytes(rp, rend, memidsBuf)) return false; break;
                case 10: if (!Pb::readBytes(rp, rend, typesBuf)) return false; break;
                default: if (!Pb::skipValue(rp, rend, t.wireType)) return false;
                }
            }

            OsmRelation rel(id);
            QVector<qint64> memids = parsePackedDeltaIds(memidsBuf);

            QVector<uint32_t> keys, vals, roles_si, types;
            {
                const uint8_t *bp = (const uint8_t*)keysBuf.constData();
                const uint8_t *bend = bp + keysBuf.size();
                while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; keys.append((uint32_t)v); }
            }
            {
                const uint8_t *bp = (const uint8_t*)valsBuf.constData();
                const uint8_t *bend = bp + valsBuf.size();
                while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; vals.append((uint32_t)v); }
            }
            {
                const uint8_t *bp = (const uint8_t*)rolesBuf.constData();
                const uint8_t *bend = bp + rolesBuf.size();
                while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; roles_si.append((uint32_t)v); }
            }
            {
                const uint8_t *bp = (const uint8_t*)typesBuf.constData();
                const uint8_t *bend = bp + typesBuf.size();
                while (bp < bend) { uint64_t v; if (!Pb::readVarint(bp, bend, v)) return false; types.append((uint32_t)v); }
            }

            int memberHead = -1;
            int memCount = qMin(memids.size(), qMin(types.size(), roles_si.size()));
            for (int i = memCount - 1; i >= 0; i--) {
                if (roles_si[i] >= (uint32_t)stringTable.size()) continue;
                QString roleStr = QString::fromUtf8(stringTable[roles_si[i]]);
                int mtype = OsmRelationMember::NODE;
                if (types[i] == 1) mtype = OsmRelationMember::WAY;
                else if (types[i] == 2) mtype = OsmRelationMember::RELATION;
                addTempMember(mtype, memids[i], roleStr, memberHead);
            }
            rel.setMemberHead(memberHead);

            int tagHead = -1;
            int tagCount = qMin(keys.size(), vals.size());
            for (int i = tagCount - 1; i >= 0; i--) {
                if (keys[i] >= (uint32_t)stringTable.size() || vals[i] >= (uint32_t)stringTable.size()) continue;
                QString keyStr = QString::fromUtf8(stringTable[keys[i]]);
                QString v = QString::fromUtf8(stringTable[vals[i]]);
                addTempTag(keyStr, v, tagHead);
            }
            rel.setTagHead(tagHead);
            result.relations.append(rel);
        }
    }
    return true;
}

static bool decompressAndParseBlock(const QByteArray &blobBytes, ParsedBlock &result) {
    Blob blob;
    if (!parseBlob(blobBytes, blob)) return false;
    QByteArray uncompressed = decompressBlob(blob);
    if (uncompressed.isEmpty()) return false;
    return parsePrimitiveBlockToTemp(uncompressed, result);
}

class BlockParseTask : public QRunnable {
public:
    QByteArray blobBytes;
    ParsedBlock *outResult;
    std::atomic<bool> *failedFlag;
    std::atomic<int> *completedCount;
    BlockParseTask(const QByteArray &data, ParsedBlock *out, std::atomic<bool> *failed, std::atomic<int> *cnt)
        : blobBytes(data), outResult(out), failedFlag(failed), completedCount(cnt) {
        setAutoDelete(true);
    }
    void run() override {
        bool ok = decompressAndParseBlock(blobBytes, *outResult);
        if (!ok) failedFlag->store(true);
        completedCount->fetch_add(1);
    }
};

static void mergeAllBlocksFast(OsmModel &model, const QVector<ParsedBlock*> &blocks) {
    int totalTags = 0, totalNds = 0, totalMembers = 0, totalNodes = 0, totalWays = 0, totalRelations = 0;
    for (const auto *b : blocks) {
        totalTags += b->tags.size();
        totalNds += b->nds.size();
        totalMembers += b->members.size();
        totalNodes += b->nodes.size();
        totalWays += b->ways.size();
        totalRelations += b->relations.size();
    }

    model.reserveTags(model.tagCount() + totalTags);
    model.reserveNds(model.nds().size() + totalNds);
    model.reserveNodes(model.nodeCount() + totalNodes);
    model.reserveWays(model.wayCount() + totalWays);
    model.reserveRelations(model.relationCount() + totalRelations);
    model.reserveMembers(model.members().size() + totalMembers);

    QHash<QString, int> keyMap = model.keyMap();
    for (const auto *b : blocks) {
        for (const auto &t : b->tags) {
            if (!keyMap.contains(t.key)) {
                keyMap.insert(t.key, model.registerKey(t.key));
            }
        }
    }

    OsmBounds bounds = model.bounds();

    for (const auto *b : blocks) {
        const int tagOffset = model.tagCount();
        const int ndOffset = model.nds().size();
        const int memberOffset = model.members().size();

        for (const auto &t : b->tags) {
            int keyId = keyMap.value(t.key);
            int next = (t.next >= 0) ? (t.next + tagOffset) : -1;
            model.addTag(keyId, t.value, next);
        }

        for (const auto &n : b->nds) {
            int next = (n.next >= 0) ? (n.next + ndOffset) : -1;
            model.addNd(n.nodeId, next);
        }

        for (const auto &m : b->members) {
            int next = (m.next >= 0) ? (m.next + memberOffset) : -1;
            model.addMember((OsmRelationMember::MemberType)m.type, m.ref, m.role, next);
        }

        for (int i = 0; i < b->nodes.size(); i++) {
            OsmNode node = b->nodes[i];
            if (node.tagHead() >= 0) node.setTagHead(node.tagHead() + tagOffset);
            model.addNode(node);
        }

        for (int i = 0; i < b->ways.size(); i++) {
            OsmWay way = b->ways[i];
            if (way.tagHead() >= 0) way.setTagHead(way.tagHead() + tagOffset);
            if (way.ndHead() >= 0) way.setNdHead(way.ndHead() + ndOffset);
            model.addWay(way);
        }

        for (int i = 0; i < b->relations.size(); i++) {
            OsmRelation rel = b->relations[i];
            if (rel.tagHead() >= 0) rel.setTagHead(rel.tagHead() + tagOffset);
            if (rel.memberHead() >= 0) rel.setMemberHead(rel.memberHead() + memberOffset);
            model.addRelation(rel);
        }

        bounds.extend(b->latMin, b->lonMin);
        bounds.extend(b->latMax, b->lonMax);
    }
}

OsmPbfParser::OsmPbfParser(QObject *parent) : OsmAbstractParser(parent) {}

bool OsmPbfParser::parse(const QString &fileName, OsmModel &model, OsmStyle &/*style*/) {
    QElapsedTimer totalTimer;
    totalTimer.start();

    QFile file(fileName);
    if (!file.open(QIODevice::ReadOnly)) {
        emitProgress(0, QStringLiteral("Cannot open PBF file: %1").arg(fileName));
        emit parseFinished(false, tr("Cannot open file"));
        return false;
    }
    emitProgress(0, QStringLiteral("Scanning PBF file..."));

    model.clear();
    model.registerKey(QStringLiteral("highway"));
    model.registerKey(QStringLiteral("railway"));
    model.registerKey(QStringLiteral("waterway"));
    model.registerKey(QStringLiteral("building"));
    model.registerKey(QStringLiteral("landuse"));
    model.registerKey(QStringLiteral("natural"));
    model.registerKey(QStringLiteral("leisure"));
    model.registerKey(QStringLiteral("boundary"));
    model.registerKey(QStringLiteral("barrier"));
    model.registerKey(QStringLiteral("aeroway"));
    model.registerKey(QStringLiteral("power"));
    model.registerKey(QStringLiteral("man_made"));
    model.registerKey(QStringLiteral("historic"));
    model.registerKey(QStringLiteral("tourism"));
    model.registerKey(QStringLiteral("amenity"));
    model.registerKey(QStringLiteral("shop"));
    model.registerKey(QStringLiteral("sport"));
    model.registerKey(QStringLiteral("military"));
    model.registerKey(QStringLiteral("place"));
    model.registerKey(QStringLiteral("piste:type"));
    model.registerKey(QStringLiteral("route"));
    model.registerKey(QStringLiteral("aerialway"));
    model.registerKey(QStringLiteral("layer"));

    const int BATCH_SIZE = 64;

    QThreadPool pool;
    int idealThreads = QThread::idealThreadCount();
    pool.setMaxThreadCount(qMax(1, idealThreads));

    std::atomic<bool> parseFailed(false);
    bool headerParsed = false;
    int totalBlocks = 0;
    int processedBlocks = 0;
    int lastReported = -1;

    QElapsedTimer timer;
    timer.start();

    // ==== Efficient batch parsing (simplified libosmium-style pipeline) ====
    // Double-buffered prefetch: while current batch is being decompressed in parallel,
    // I/O has already prefetched the next batch into memory
    // Avoids full 3-stage pipeline thread sync overhead while retaining I/O overlap benefits

    QByteArray blobBytesBatch[BATCH_SIZE];
    int batchCount = 0;

    // Prefetch buffer: prefetch next batch while current batch is being decompressed
    QVector<QByteArray> prefetchBuffer;

    auto readBatchBlobs = [&](QByteArray *batchOut, int &count) -> bool {
        count = 0;
        while (count < BATCH_SIZE && !file.atEnd()) {
            uint8_t headerLenBuf[4];
            if (file.read((char*)headerLenBuf, 4) != 4) {
                if (count == 0 && file.atEnd()) return true;
                qWarning() << "PBF truncated at header length";
                return false;
            }
            uint32_t headerLen = ((uint32_t)headerLenBuf[0] << 24) | ((uint32_t)headerLenBuf[1] << 16) |
                                 ((uint32_t)headerLenBuf[2] << 8)  | ((uint32_t)headerLenBuf[3]);
            if (headerLen > 64*1024) {
                qWarning() << "PBF blob header too large:" << headerLen;
                return false;
            }
            QByteArray headerBytes = file.read(headerLen);
            if (headerBytes.size() != (int)headerLen) {
                qWarning() << "PBF truncated at header";
                return false;
            }
            BlobHeader hdr;
            if (!parseBlobHeader(headerBytes, hdr)) {
                qWarning() << "PBF bad BlobHeader";
                return false;
            }
            QByteArray blobBytes = file.read(hdr.datasize);
            if (blobBytes.size() != hdr.datasize) {
                qWarning() << "PBF truncated at blob body";
                return false;
            }

            QString typeStr = QString::fromUtf8(hdr.typeBytes);
            if (typeStr == QLatin1String("OSMHeader")) {
                Blob blob;
                if (!parseBlob(blobBytes, blob)) return false;
                QByteArray uncompressed = decompressBlob(blob);
                if (uncompressed.isEmpty()) return false;
                headerParsed = true;
            } else if (typeStr == QLatin1String("OSMData")) {
                if (!headerParsed) {
                    qWarning() << "PBF data before header";
                    return false;
                }
                batchOut[count++] = blobBytes;
                totalBlocks++;
            }
        }
        return true;
    };

    // Prefetch first batch
    if (!readBatchBlobs(blobBytesBatch, batchCount)) {
        parseFailed.store(true);
    }

    // Future for async prefetch
    QFuture<QVector<QByteArray>> prefetchFuture;
    bool hasPrefetch = false;

    while (!parseFailed.load() && batchCount > 0) {
        int pct = 3 + (int)(85.0 * processedBlocks / qMax(1, totalBlocks + BATCH_SIZE));
        if (pct != lastReported) {
            lastReported = pct;
            emitProgress(pct, QStringLiteral("Parallel parsing in progress... %1 blocks processed").arg(processedBlocks));
        }

        // Submit current batch to thread pool for parallel decompression + parsing
        std::atomic<int> completedBlocks(0);
        QVector<ParsedBlock*> parsedResults(batchCount);
        for (int i = 0; i < batchCount; i++) {
            parsedResults[i] = new ParsedBlock();
            auto *task = new BlockParseTask(blobBytesBatch[i], parsedResults[i], &parseFailed, &completedBlocks);
            pool.start(task);
        }

        // Background async prefetch next batch of blobs (I/O parallel with CPU decompression)
        // Use QtConcurrent::run in a separate thread, does not block main thread
        if (!hasPrefetch && !file.atEnd() && !parseFailed.load()) {
            QFile *filePtr = &file;
            bool *headerParsedPtr = &headerParsed;
            int *totalBlocksPtr = &totalBlocks;
            std::atomic<bool> *failedPtr = &parseFailed;
            prefetchFuture = QtConcurrent::run([filePtr, headerParsedPtr, totalBlocksPtr, failedPtr]() -> QVector<QByteArray> {
                QVector<QByteArray> result;
                result.reserve(BATCH_SIZE);
                int count = 0;
                while (count < BATCH_SIZE && !filePtr->atEnd() && !failedPtr->load()) {
                    uint8_t headerLenBuf[4];
                    if (filePtr->read((char*)headerLenBuf, 4) != 4) {
                        if (filePtr->atEnd()) break;
                        failedPtr->store(true);
                        break;
                    }
                    uint32_t headerLen = ((uint32_t)headerLenBuf[0] << 24) | ((uint32_t)headerLenBuf[1] << 16) |
                                         ((uint32_t)headerLenBuf[2] << 8)  | ((uint32_t)headerLenBuf[3]);
                    if (headerLen > 64*1024) { failedPtr->store(true); break; }
                    QByteArray headerBytes = filePtr->read(headerLen);
                    if (headerBytes.size() != (int)headerLen) { failedPtr->store(true); break; }
                    BlobHeader hdr;
                    if (!parseBlobHeader(headerBytes, hdr)) { failedPtr->store(true); break; }
                    QByteArray blobBytes = filePtr->read(hdr.datasize);
                    if (blobBytes.size() != hdr.datasize) { failedPtr->store(true); break; }

                    QString typeStr = QString::fromUtf8(hdr.typeBytes);
                    if (typeStr == QLatin1String("OSMData") && *headerParsedPtr) {
                        result.append(blobBytes);
                        (*totalBlocksPtr)++;
                        count++;
                    } else if (typeStr == QLatin1String("OSMHeader")) {
                        Blob blob;
                        if (!parseBlob(blobBytes, blob)) { failedPtr->store(true); break; }
                        QByteArray uncompressed = decompressBlob(blob);
                        if (uncompressed.isEmpty()) { failedPtr->store(true); break; }
                        *headerParsedPtr = true;
                    }
                }
                return result;
            });
            hasPrefetch = true;
        }

        // Wait for current batch decompression to complete (non-blocking poll, allows UI response)
        while (!pool.waitForDone(100)) {
            if (parseFailed.load()) {
                pool.clear();
                pool.waitForDone();
                break;
            }
        }

        // Wait for async prefetch to complete (if any)
        if (hasPrefetch) {
            prefetchFuture.waitForFinished();
            prefetchBuffer = prefetchFuture.result();
            hasPrefetch = false;
        }

        if (parseFailed.load()) {
            qDeleteAll(parsedResults);
            emitProgress(0, QStringLiteral("Parsing failed"));
            emit parseFinished(false, tr("Parsing failed"));
            return false;
        }

        // Merge into model
        mergeAllBlocksFast(model, parsedResults);
        qDeleteAll(parsedResults);

        processedBlocks += batchCount;

        // Release current batch memory
        for (int i = 0; i < batchCount; i++) {
            blobBytesBatch[i] = QByteArray();
        }

        // Use prefetch buffer as next batch (avoid extra I/O)
        if (!prefetchBuffer.isEmpty()) {
            batchCount = qMin(prefetchBuffer.size(), BATCH_SIZE);
            for (int i = 0; i < batchCount; i++) {
                blobBytesBatch[i] = std::move(prefetchBuffer[i]);
            }
            prefetchBuffer.clear();
        } else if (file.atEnd()) {
            batchCount = 0;
        } else if (!parseFailed.load()) {
            // No prefetch result (file finished or prefetch failed), read next batch normally
            if (!readBatchBlobs(blobBytesBatch, batchCount)) {
                parseFailed.store(true);
            }
        }
    }

    file.close();

    if (parseFailed.load() || totalBlocks == 0) {
        emitProgress(0, QStringLiteral("Parsing failed or no data"));
        emit parseFinished(false, tr("Parsing failed"));
        return false;
    }

    qint64 parseTime = totalTimer.elapsed();
    qDebug() << "Parallel parsing done in" << parseTime << "ms, threads:" << pool.maxThreadCount()
             << "blocks:" << totalBlocks;
    emitProgress(90, QStringLiteral("Parsing complete (%1ms), building draw list...").arg(parseTime));

    QElapsedTimer buildTimer;
    buildTimer.start();
    model.finalizeAfterLoad();
    qDebug() << "Build draw list done in" << buildTimer.elapsed() << "ms";

    qint64 elapsed = totalTimer.elapsed();
    qDebug() << "=== PBF parsing total:" << elapsed << "ms, nodes:" << model.nodeCount()
             << "ways:" << model.wayCount() << "relations:" << model.relationCount()
             << "fill items:" << model.fillDrawList().size()
             << "line items:" << model.lineDrawList().size()
             << "threads:" << pool.maxThreadCount() << "blocks:" << totalBlocks << "===";
    emitProgress(100, QStringLiteral("Parsing complete (%1ms): nodes=%2 ways=%3 relations=%4")
        .arg(elapsed).arg(model.nodeCount()).arg(model.wayCount()).arg(model.relationCount()));
    emit parseFinished(true);
    return true;
}