MetadataPackScanStrategy.h

Go to the documentation of this file.

// SPDX-FileCopyrightText: 2026 fuddlesworth
// SPDX-License-Identifier: LGPL-2.1-or-later
 
#pragma once
 
#include <PhosphorFsLoader/DirectoryLoader.h>
#include <PhosphorFsLoader/IScanStrategy.h>
 
#include <QtCore/QByteArray>
#include <QtCore/QCryptographicHash>
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QHash>
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonObject>
#include <QtCore/QJsonParseError>
#include <QtCore/QList>
#include <QtCore/QLoggingCategory>
#include <QtCore/QString>
#include <QtCore/QStringList>
 
#include <algorithm>
#include <functional>
#include <optional>
#include <type_traits>
#include <utility>
#include <vector>
 
namespace PhosphorFsLoader {
 
template<typename Payload>
class MetadataPackScanStrategy : public IScanStrategy
{
    // The strategy hashes `id` into the per-rescan signature and uses it
    // as the QHash key for first-wins layering — neither works without
    // a public QString id member. ShaderInfo and AnimationShaderEffect
    // both satisfy this; bespoke payloads must too.
    //
    // `decltype(... .id)` on an lvalue Payload yields `QString&`; strip
    // the reference before comparing so the assertion fires only when
    // the field's type itself isn't `QString`.
    static_assert(std::is_same_v<std::remove_reference_t<decltype(std::declval<Payload&>().id)>, QString>,
                  "MetadataPackScanStrategy<Payload> requires Payload to expose a public 'QString id' member.");
 
public:
    static constexpr int kDefaultMaxEntries = 10'000;
 
    using Parser =
        std::function<std::optional<Payload>(const QString& subdirPath, const QJsonObject& root, bool isUser)>;
 
    using PerEntryWatchPaths = std::function<QStringList(const Payload&)>;
 
    using PerDirectoryWatchPaths = std::function<QStringList(const QString& searchPath)>;
 
    using PerSubdirSkip = std::function<bool(const QString& subdirName)>;
 
    using SignatureContrib = std::function<void(QCryptographicHash&, const Payload&)>;
 
    using OnCommit = std::function<void()>;
 
    MetadataPackScanStrategy(Parser parser, OnCommit onCommit)
        : m_parser(std::move(parser))
        , m_onCommit(std::move(onCommit))
    {
        // An empty `Parser` would silently skip every entry on every
        // rescan and look like a configuration bug from the outside ("my
        // packs all disappeared"). Both real consumers always pass a
        // real parser; assert in debug builds so a future caller doesn't
        // have to debug an empty registry from a default-constructed
        // `std::function`. `OnCommit` is allowed to be empty (a consumer
        // that doesn't care about content-changed signals).
        Q_ASSERT_X(static_cast<bool>(m_parser), "MetadataPackScanStrategy",
                   "Parser must not be empty — every rescan would silently skip every entry");
    }
 
    ~MetadataPackScanStrategy() override = default;
 
    MetadataPackScanStrategy(const MetadataPackScanStrategy&) = delete;
    MetadataPackScanStrategy& operator=(const MetadataPackScanStrategy&) = delete;
 
    void setPerEntryWatchPaths(PerEntryWatchPaths fn)
    {
        m_perEntryWatch = std::move(fn);
    }
 
    void setPerDirectoryWatchPaths(PerDirectoryWatchPaths fn)
    {
        m_perDirWatch = std::move(fn);
    }
 
    void setPerSubdirSkip(PerSubdirSkip fn)
    {
        m_subdirSkip = std::move(fn);
    }
 
    void setSignatureContrib(SignatureContrib fn)
    {
        m_sigContrib = std::move(fn);
    }
 
    void setUserPath(const QString& path)
    {
        m_userPath = path;
    }
 
    void setMaxEntries(int cap)
    {
        Q_ASSERT_X(cap >= 0, "MetadataPackScanStrategy::setMaxEntries", "cap must be non-negative");
        m_maxEntries = std::max(0, cap);
    }
 
    void setLoggingCategory(const QLoggingCategory& cat)
    {
        m_loggingCat = &cat;
    }
 
    QStringList performScan(const QStringList& directoriesInScanOrder) override;
 
    // ─── Accessors used by the consumer registry ────────────────────────────
 
    const QHash<QString, Payload>& packsById() const
    {
        return m_packs;
    }
 
    QList<Payload> packs() const
    {
        QList<Payload> sorted = m_packs.values();
        std::sort(sorted.begin(), sorted.end(), [](const Payload& a, const Payload& b) {
            return a.id < b.id;
        });
        return sorted;
    }
 
    QStringList packIds() const
    {
        QStringList ids = m_packs.keys();
        std::sort(ids.begin(), ids.end());
        return ids;
    }
 
    Payload pack(const QString& id) const
    {
        return m_packs.value(id);
    }
 
    bool contains(const QString& id) const
    {
        return m_packs.contains(id);
    }
 
    int size() const
    {
        return m_packs.size();
    }
 
private:
    Parser m_parser;
    OnCommit m_onCommit;
    PerEntryWatchPaths m_perEntryWatch;
    PerDirectoryWatchPaths m_perDirWatch;
    PerSubdirSkip m_subdirSkip;
    SignatureContrib m_sigContrib;
 
    QString m_userPath;
    int m_maxEntries = kDefaultMaxEntries;
    const QLoggingCategory* m_loggingCat = nullptr;
 
    QHash<QString, Payload> m_packs;
    QByteArray m_lastSignature;
    bool m_signatureSeeded = false;
};
 
// ─── Template implementation ─────────────────────────────────────────────────
 
namespace detail {
 
PHOSPHORFSLOADER_EXPORT Q_DECLARE_LOGGING_CATEGORY(lcMetadataPackScan)
 
} // namespace detail
 
template<typename Payload>
QStringList MetadataPackScanStrategy<Payload>::performScan(const QStringList& directoriesInScanOrder)
{
    // Per-entry filesystem fingerprint (`metadata.json` size+mtime +
    // `isUser`) captured during the parse loop and mixed into the SHA-1
    // signature below. Decoupled from `Payload` so the strategy can
    // fingerprint facts the parser doesn't necessarily store, without
    // requiring Payload to expose them.
    struct EntryFingerprint
    {
        qint64 metadataSize = 0;
        qint64 metadataMtimeMs = 0;
        bool isUser = false;
    };
    // Single accumulator holding everything we need about an entry: id
    // (for the signature key + final map key), fingerprint, payload.
    // Sorted-by-id at the end of the parse pass so signature mixing and
    // accessor population both consume a stable order without paying
    // the cost of a parallel `QHash<QString, EntryFingerprint>`.
    struct Entry
    {
        QString id;
        EntryFingerprint fp;
        Payload payload;
    };
    std::vector<Entry> entries;
    // Tracks claimed ids for first-wins collision detection during the
    // parse loop. Cheaper than a parallel `QHash<QString, Payload>` —
    // one O(1) key insert per entry, no payload duplication.
    QSet<QString> seenIds;
 
    QStringList desiredWatches;
 
    const QLoggingCategory& log = m_loggingCat ? *m_loggingCat : detail::lcMetadataPackScan();
 
    // Resolve the user path's canonical form once per rescan. Empty
    // (no user path configured, or the path doesn't exist yet) yields
    // `false` for every dir — the iterated-dir compare below short-
    // circuits when this is empty. Canonicalised once here, then
    // compared per-search-path inside the OUTER loop only — the inner
    // subdir loop never canonicalises, so cost stays O(searchPaths).
    const QString canonicalUserPath = m_userPath.isEmpty() ? QString() : QFileInfo(m_userPath).canonicalFilePath();
 
    bool capTripped = false;
 
    // Reverse-iterate: highest-priority dirs first, first-wins on id
    // collision. The base normalises caller input into the canonical
    // `[lowest, ..., highest]` shape at registration time, so this
    // reversal is the SSOT for the user-wins semantic the two
    // consumer registries promise.
    for (auto dirIt = directoriesInScanOrder.crbegin(); dirIt != directoriesInScanOrder.crend() && !capTripped;
         ++dirIt) {
        const QString& searchPath = *dirIt;
        QDir dirObj(searchPath);
        if (!dirObj.exists()) {
            qCDebug(log) << "MetadataPackScanStrategy: search path does not exist:" << searchPath;
            continue;
        }
 
        const bool isUserDir =
            !canonicalUserPath.isEmpty() && QFileInfo(searchPath).canonicalFilePath() == canonicalUserPath;
 
        // Per-search-path watch additions (top-level shared files —
        // shader-pack registry watches `*.glsl` includes here).
        if (m_perDirWatch) {
            desiredWatches.append(m_perDirWatch(searchPath));
        }
 
        const QStringList subdirs = dirObj.entryList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Name);
        for (const QString& subdir : subdirs) {
            if (m_subdirSkip && m_subdirSkip(subdir)) {
                continue;
            }
            // Per-rescan entry-count DoS guard. Reverse-iteration scans
            // user-first / system-last, so cap-trip drops *system*
            // overflow rather than user overrides.
            if (entries.size() >= static_cast<std::size_t>(m_maxEntries)) {
                capTripped = true;
                break;
            }
 
            const QString subdirPath = dirObj.filePath(subdir);
            const QString metadataPath = subdirPath + QStringLiteral("/metadata.json");
 
            // Always re-arm the metadata.json watch — even if parsing
            // fails or the id collides. An edit that fixes a broken
            // metadata.json is the most common way an entry transitions
            // from invisible to visible; we want to wake on it.
            desiredWatches.append(metadataPath);
 
            const QFileInfo metadataInfo(metadataPath);
            if (!metadataInfo.exists()) {
                qCDebug(log) << "MetadataPackScanStrategy: skipping subdir, no metadata.json:" << subdirPath;
                continue;
            }
 
            // DoS guard: untrusted same-user metadata.json must not
            // stall the GUI thread with a 2 GB blob. Reuse
            // `DirectoryLoader::kMaxFileBytes` as the SSOT — same cap
            // the sister `JsonScanStrategy` enforces on every JSON
            // file it loads.
            if (metadataInfo.size() > DirectoryLoader::kMaxFileBytes) {
                qCWarning(log) << "MetadataPackScanStrategy: skipping oversized metadata.json:" << metadataPath << "("
                               << metadataInfo.size() << "bytes, cap" << DirectoryLoader::kMaxFileBytes << ")";
                continue;
            }
 
            QFile file(metadataPath);
            if (!file.open(QIODevice::ReadOnly)) {
                qCWarning(log) << "MetadataPackScanStrategy: failed to open metadata.json:" << metadataPath;
                continue;
            }
 
            QJsonParseError parseError{};
            const QJsonDocument doc = QJsonDocument::fromJson(file.readAll(), &parseError);
            if (parseError.error != QJsonParseError::NoError) {
                qCWarning(log) << "MetadataPackScanStrategy: parse error in" << metadataPath << ":"
                               << parseError.errorString();
                continue;
            }
            if (!doc.isObject()) {
                qCWarning(log) << "MetadataPackScanStrategy: non-object root in" << metadataPath;
                continue;
            }
 
            // Schema-specific parse. The ctor asserts `m_parser` is
            // non-null and there is no setter to clear it, so it is
            // safe to invoke unconditionally.
            std::optional<Payload> parsed = m_parser(subdirPath, doc.object(), isUserDir);
            if (!parsed.has_value()) {
                qCDebug(log) << "MetadataPackScanStrategy: parser declined" << metadataPath;
                continue;
            }
            if (parsed->id.isEmpty()) {
                qCWarning(log) << "MetadataPackScanStrategy: skipping" << metadataPath << ": empty 'id' field";
                continue;
            }
 
            // First-wins on id collision. Reverse-iteration means a
            // user-dir entry claims its id before any system-dir entry
            // can; a colliding system entry is silently shadowed.
            if (seenIds.contains(parsed->id)) {
                qCDebug(log) << "MetadataPackScanStrategy: id" << parsed->id
                             << "already registered from a higher-priority dir; shadowed at:" << subdirPath;
                continue;
            }
 
            // Per-payload watches — frag/vert/kwin shaders, etc.
            if (m_perEntryWatch) {
                desiredWatches.append(m_perEntryWatch(*parsed));
            }
 
            // Capture the metadata.json fingerprint BEFORE the move so
            // we can mix it into the per-rescan signature below. Any
            // parser-consumed field's edit shifts the file's mtime
            // (POSIX guarantees this on a content-change write — the
            // editors used in our tests and the production save paths
            // rely on it), so a single mtime+size mix-in covers every
            // schema field without forcing per-field enumeration in
            // SignatureContrib.
            QString id = parsed->id;
            seenIds.insert(id);
            entries.push_back(
                Entry{std::move(id),
                      EntryFingerprint{metadataInfo.size(), metadataInfo.lastModified().toMSecsSinceEpoch(), isUserDir},
                      std::move(*parsed)});
        }
    }
 
    if (capTripped) {
        qCWarning(log).nospace() << "MetadataPackScanStrategy: reached entry cap (" << m_maxEntries
                                 << ") — later entries skipped to protect the GUI thread. Prune the watched search "
                                    "paths or raise the cap.";
    }
 
    // Sort by id once. Stable hash + stable accessor ordering both fall
    // out of this single pass — no parallel structures, no second
    // QHash::keys() + sort.
    std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) {
        return a.id < b.id;
    });
 
    // SHA-1 signature in two passes.
    //
    // Pass 1 — per-entry attribution. (id, isUser, metadata.json size+mtime,
    // payload-specific bytes via SignatureContrib). Stable iteration order
    // is the sorted entries vec; QHash randomisation never leaks into the
    // signature.
    //
    // Pass 2 — watch-set auto-fingerprint. Every distinct file the
    // strategy is watching (the per-entry metadata.json plus everything
    // returned by `PerEntryWatchPaths` and `PerDirectoryWatchPaths`)
    // contributes path|size|mtime|. This is load-bearing for change-only
    // emit completeness: any file the watcher fires on must shift the
    // signature, otherwise the rescan runs but `OnCommit` stays silent
    // and consumers hold stale state. Without this pass, top-level
    // shared `*.glsl` includes (`common.glsl`, `audio.glsl`, …) and
    // per-pack auxiliary files (`helpers.glsl`, etc.) would fire the
    // watcher but not the consumer's content-changed signal.
    //
    // The metadata.json mtime+size shows up in BOTH passes (per-entry +
    // watch-set). Deterministic redundancy — costs nothing, keeps the
    // per-entry attribution clean.
    QCryptographicHash hasher(QCryptographicHash::Sha1);
    for (const Entry& e : entries) {
        hasher.addData(e.id.toUtf8());
        hasher.addData(QByteArrayView("|"));
        hasher.addData(e.fp.isUser ? QByteArrayView("u") : QByteArrayView("s"));
        hasher.addData(QByteArrayView("|"));
        hasher.addData(QByteArray::number(e.fp.metadataSize));
        hasher.addData(QByteArrayView("|"));
        hasher.addData(QByteArray::number(e.fp.metadataMtimeMs));
        hasher.addData(QByteArrayView("|"));
        if (m_sigContrib) {
            m_sigContrib(hasher, e.payload);
        }
        hasher.addData(QByteArrayView("\n"));
    }
    // Watch-set pass. Sorted + deduped for deterministic ordering across
    // rescans regardless of the order paths were appended during the
    // outer / inner loops above. `removeDuplicates` is O(n log n) and
    // entry counts are bounded by `m_maxEntries`, so cost is negligible.
    QStringList sortedWatches = desiredWatches;
    sortedWatches.removeDuplicates();
    std::sort(sortedWatches.begin(), sortedWatches.end());
    for (const QString& path : sortedWatches) {
        hasher.addData(path.toUtf8());
        hasher.addData(QByteArrayView("|"));
        const QFileInfo fi(path);
        if (fi.exists()) {
            hasher.addData(QByteArray::number(fi.size()));
            hasher.addData(QByteArrayView("|"));
            hasher.addData(QByteArray::number(fi.lastModified().toMSecsSinceEpoch()));
        } else {
            // Stable sentinel for absent files. `lastModified()` on an
            // invalid datetime is implementation-defined — explicit
            // "missing" keeps the hash format portable across Qt
            // versions and filesystems.
            hasher.addData(QByteArrayView("missing"));
        }
        hasher.addData(QByteArrayView("\n"));
    }
    const QByteArray signature = hasher.result();
 
    QHash<QString, Payload> fresh;
    fresh.reserve(static_cast<int>(entries.size()));
    for (Entry& e : entries) {
        fresh.insert(e.id, std::move(e.payload));
    }
 
    const bool isFirstScan = !m_signatureSeeded;
    const bool changed = isFirstScan ? !fresh.isEmpty() : signature != m_lastSignature;
 
    m_packs = std::move(fresh);
    m_lastSignature = signature;
    m_signatureSeeded = true;
 
    if (changed && m_onCommit) {
        m_onCommit();
    }
 
    // Return the deduped + lex-sorted watch set rather than the raw
    // append-order `desiredWatches`. The caller (`WatchedDirectorySet`)
    // dedupes again internally via `QSet<QString> m_watchedFiles`, so
    // this isn't a correctness fix — but the strategy already paid for
    // dedup + sort during the signature pass, so handing the cleaned
    // list back saves the watcher its own dedup pass and removes a
    // class of "watcher diagnostics in nondeterministic order" from
    // the system. Costs nothing.
    return sortedWatches;
}
 
} // namespace PhosphorFsLoader