MetadataPackScanStrategy.h

Go to the documentation of this file.

// SPDX-FileCopyrightText: 2026 fuddlesworth
// SPDX-License-Identifier: LGPL-2.1-or-later
 
#pragma once
 
#include <PhosphorFsLoader/DirectoryLoader.h>
#include <PhosphorFsLoader/IScanStrategy.h>
#include <PhosphorFsLoader/phosphorfsloader_export.h>
 
#include <QtCore/QByteArray>
#include <QtCore/QCryptographicHash>
#include <QtCore/QDateTime>
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QHash>
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonObject>
#include <QtCore/QJsonParseError>
#include <QtCore/QList>
#include <QtCore/QLoggingCategory>
#include <QtCore/QSet>
#include <QtCore/QString>
#include <QtCore/QStringList>
 
#include <algorithm>
#include <functional>
#include <optional>
#include <type_traits>
#include <utility>
#include <vector>
 
namespace PhosphorFsLoader {
 
template<typename Payload>
class MetadataPackScanStrategy : public IScanStrategy
{
    // The strategy hashes `id` into the per-rescan signature and uses it
    // as the QHash key for first-wins layering — neither works without
    // a public QString id member. Every production Entry type satisfies
    // this; bespoke payloads must too.
    //
    // `decltype(... .id)` on an lvalue Payload yields `QString&`; strip
    // the reference before comparing so the assertion fires only when
    // the field's type itself isn't `QString`.
    static_assert(std::is_same_v<std::remove_reference_t<decltype(std::declval<Payload&>().id)>, QString>,
                  "MetadataPackScanStrategy<Payload> requires Payload to expose a public 'QString id' member.");
 
public:
    static constexpr int kDefaultMaxEntries = 10'000;
 
    using Parser =
        std::function<std::optional<Payload>(const QString& subdirPath, const QJsonObject& root, bool isUser)>;
 
    using PerEntryWatchPaths = std::function<QStringList(const Payload&)>;
 
    using PerDirectoryWatchPaths = std::function<QStringList(const QString& searchPath)>;
 
    using PerSubdirSkip = std::function<bool(const QString& subdirName)>;
 
    using SignatureContrib = std::function<void(QCryptographicHash&, const Payload&)>;
 
    using OnCommit = std::function<void()>;
 
    MetadataPackScanStrategy(Parser parser, OnCommit onCommit)
        : m_parser(std::move(parser))
        , m_onCommit(std::move(onCommit))
    {
        // An empty `Parser` would silently skip every entry on every
        // rescan and look like a configuration bug from the outside ("my
        // packs all disappeared"). Every real consumer always passes a
        // real parser; assert in debug builds so a future caller doesn't
        // have to debug an empty registry from a default-constructed
        // `std::function`. `OnCommit` is allowed to be empty (a consumer
        // that doesn't care about content-changed signals).
        Q_ASSERT_X(static_cast<bool>(m_parser), "MetadataPackScanStrategy",
                   "Parser must not be empty — every rescan would silently skip every entry");
    }
 
    ~MetadataPackScanStrategy() override = default;
 
    MetadataPackScanStrategy(const MetadataPackScanStrategy&) = delete;
    MetadataPackScanStrategy& operator=(const MetadataPackScanStrategy&) = delete;
 
    void setPerEntryWatchPaths(PerEntryWatchPaths fn)
    {
        m_perEntryWatch = std::move(fn);
    }
 
    void setPerDirectoryWatchPaths(PerDirectoryWatchPaths fn)
    {
        m_perDirWatch = std::move(fn);
    }
 
    void setPerSubdirSkip(PerSubdirSkip fn)
    {
        m_subdirSkip = std::move(fn);
    }
 
    void setSignatureContrib(SignatureContrib fn)
    {
        m_sigContrib = std::move(fn);
    }
 
    void setUserPath(const QString& path)
    {
        m_userPath = path;
    }
 
    void setMaxEntries(int cap)
    {
        Q_ASSERT_X(cap >= 0, "MetadataPackScanStrategy::setMaxEntries", "cap must be non-negative");
        // Release fallback for a NEGATIVE cap is the DEFAULT, not zero.
        // Clamping to zero would make the assert's release counterpart
        // destructive in exactly the way the assert exists to prevent: the cap
        // trips on the first subdir, `m_packs` rebuilds empty, and `OnCommit`
        // fires as though every pack had been uninstalled. Falling back to the
        // default degrades to "the caller's cap was ignored", which is
        // recoverable and obvious. An explicit zero is honoured — see above.
        m_maxEntries = cap >= 0 ? cap : kDefaultMaxEntries;
    }
 
    void setLoggingCategory(const QLoggingCategory& cat)
    {
        m_loggingCat = &cat;
    }
 
    QStringList performScan(const QStringList& directoriesInScanOrder) override;
 
    // ─── Accessors used by the consumer registry ────────────────────────────
 
    const QHash<QString, Payload>& packsById() const
    {
        return m_packs;
    }
 
    QList<Payload> packs() const
    {
        QList<Payload> sorted = m_packs.values();
        std::sort(sorted.begin(), sorted.end(), [](const Payload& a, const Payload& b) {
            return a.id < b.id;
        });
        return sorted;
    }
 
    QStringList packIds() const
    {
        QStringList ids = m_packs.keys();
        std::sort(ids.begin(), ids.end());
        return ids;
    }
 
    Payload pack(const QString& id) const
    {
        return m_packs.value(id);
    }
 
    bool contains(const QString& id) const
    {
        return m_packs.contains(id);
    }
 
    int size() const
    {
        return m_packs.size();
    }
 
private:
    Parser m_parser;
    OnCommit m_onCommit;
    PerEntryWatchPaths m_perEntryWatch;
    PerDirectoryWatchPaths m_perDirWatch;
    PerSubdirSkip m_subdirSkip;
    SignatureContrib m_sigContrib;
 
    QString m_userPath;
    int m_maxEntries = kDefaultMaxEntries;
    const QLoggingCategory* m_loggingCat = nullptr;
 
    QHash<QString, Payload> m_packs;
    QByteArray m_lastSignature;
    bool m_signatureSeeded = false;
};
 
// ─── Template implementation ─────────────────────────────────────────────────
 
namespace detail {
 
PHOSPHORFSLOADER_EXPORT Q_DECLARE_LOGGING_CATEGORY(lcMetadataPackScan)
 
} // namespace detail
 
template<typename Payload>
QStringList MetadataPackScanStrategy<Payload>::performScan(const QStringList& directoriesInScanOrder)
{
    // Release-build counterpart to the ctor's `Q_ASSERT_X(m_parser)`. An empty
    // `std::function` would throw `std::bad_function_call` out of the parse
    // loop below, on the GUI thread, from inside a filesystem-watch callback.
    // Refusing the whole scan instead leaves the previously registered packs
    // in place and logs something an operator can grep for.
    if (!m_parser) {
        // NOTE the second-order consequence of the empty return: the caller
        // (`WatchedDirectorySet::rescanAll`) feeds it straight into
        // `syncFileWatches`, which removes every per-file watch not in the
        // returned list — so a parserless strategy not only keeps the
        // previously registered packs frozen, it silently disarms live
        // reload for them, permanently (there is no parser setter, so the
        // condition never clears for the strategy's lifetime).
        qCWarning(m_loggingCat ? *m_loggingCat : detail::lcMetadataPackScan())
            << "MetadataPackScanStrategy: no parser configured, skipping scan";
        return {};
    }
 
    // Per-entry filesystem fingerprint (`metadata.json` size+mtime +
    // `isUser`) captured during the parse loop and mixed into the SHA-1
    // signature below. Decoupled from `Payload` so the strategy can
    // fingerprint facts the parser doesn't necessarily store, without
    // requiring Payload to expose them.
    struct EntryFingerprint
    {
        qint64 metadataSize = 0;
        qint64 metadataMtimeMs = 0;
        bool isUser = false;
    };
    // Single accumulator holding everything we need about an entry: id
    // (for the signature key + final map key), fingerprint, payload.
    // Sorted-by-id at the end of the parse pass so signature mixing and
    // accessor population both consume a stable order without paying
    // the cost of a parallel `QHash<QString, EntryFingerprint>`.
    struct Entry
    {
        QString id;
        EntryFingerprint fp;
        Payload payload;
    };
    std::vector<Entry> entries;
    // Tracks claimed ids for first-wins collision detection during the
    // parse loop. Cheaper than a parallel `QHash<QString, Payload>` —
    // one O(1) key insert per entry, no payload duplication.
    QSet<QString> seenIds;
 
    QStringList desiredWatches;
 
    const QLoggingCategory& log = m_loggingCat ? *m_loggingCat : detail::lcMetadataPackScan();
 
    // Resolve the user path's canonical form once per rescan. Empty
    // (no user path configured, or the path doesn't exist yet) yields
    // `false` for every dir — the iterated-dir compare below short-
    // circuits when this is empty. Canonicalised once here, then
    // compared per-search-path inside the OUTER loop only — the inner
    // subdir loop never canonicalises, so cost stays O(searchPaths).
    const QString canonicalUserPath = m_userPath.isEmpty() ? QString() : QFileInfo(m_userPath).canonicalFilePath();
 
    bool capTripped = false;
    int subdirsConsidered = 0;
 
    // Reverse-iterate: highest-priority dirs first, first-wins on id
    // collision. The base normalises caller input into the canonical
    // `[lowest, ..., highest]` shape at registration time, so this
    // reversal is the SSOT for the user-wins semantic the two
    // consumer registries promise.
    for (auto dirIt = directoriesInScanOrder.crbegin(); dirIt != directoriesInScanOrder.crend() && !capTripped;
         ++dirIt) {
        const QString& searchPath = *dirIt;
        QDir dirObj(searchPath);
        if (!dirObj.exists()) {
            qCDebug(log) << "MetadataPackScanStrategy: search path does not exist:" << searchPath;
            continue;
        }
 
        const bool isUserDir =
            !canonicalUserPath.isEmpty() && QFileInfo(searchPath).canonicalFilePath() == canonicalUserPath;
 
        // Per-search-path watch additions (top-level shared files —
        // shader-pack registry watches `*.glsl` includes here). These sit
        // OUTSIDE the entry cap: the cap charges per-subdir work, and a
        // per-directory extractor returning an unbounded glob grows the
        // watch list with nothing bounding it. Acceptable because the
        // extractor is trusted first-party code (unlike the pack subdirs,
        // which are user-writable data), but an extractor author adding a
        // recursive glob here should know the cap does not cover it.
        if (m_perDirWatch) {
            desiredWatches.append(m_perDirWatch(searchPath));
        }
 
        // Symlinked pack directories are FOLLOWED, with no canonical-
        // containment check. Symlinking a pack directory out of a dotfiles repo
        // or a shared drive is the normal way people manage them, and the
        // threat model here is same-user: anything a symlink reaches, the user
        // could have copied in.
        //
        // This is NOT the claim that a pack is inert. Every production consumer
        // is a shader registry, and a pack's metadata NAMES source files that
        // are compiled and run on the GPU — so the containment that matters is
        // on the paths a pack DECLARES, not on the directory it was found
        // through, and it belongs in the parser that resolves them.
        // PluginLoader is the counter-example in this library and uses
        // QDir::NoSymLinks, because a plugin subdir leads to a dlopen.
        const QStringList subdirs = dirObj.entryList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Name);
        for (const QString& subdir : subdirs) {
            // Per-rescan DoS guard, counting subdirs considered rather than
            // entries registered. Reverse-iteration scans user-first /
            // system-last, so cap-trip drops *system* overflow rather than
            // user overrides.
            //
            // Checked BEFORE the watch is armed below, not after: the watch
            // list is the other unbounded quantity here, and a spray of
            // broken packs that never register would otherwise grow it until
            // the inotify per-user watch limit is exhausted.
            //
            // Also before the skip predicate, so a skipped subdir is still
            // CHARGED. Every sibling scanner charges before its own first
            // filter, for the same reason: a caller-supplied predicate that
            // inspects the name is work a spray can buy, and "considered" has
            // to mean every entry the loop reaches or the cap bounds something
            // other than what its docs claim.
            if (subdirsConsidered >= m_maxEntries) {
                capTripped = true;
                break;
            }
            ++subdirsConsidered;
            if (m_subdirSkip && m_subdirSkip(subdir)) {
                continue;
            }
 
            const QString subdirPath = dirObj.filePath(subdir);
            const QString metadataPath = subdirPath + QStringLiteral("/metadata.json");
 
            // Always re-arm the metadata.json watch — even if parsing
            // fails or the id collides. An edit that fixes a broken
            // metadata.json is the most common way an entry transitions
            // from invisible to visible; we want to wake on it.
            desiredWatches.append(metadataPath);
 
            const QFileInfo metadataInfo(metadataPath);
            // `isFile()`, not `exists()`. A FIFO in a user-writable pack dir
            // satisfies exists(), and `QFile::open(ReadOnly)` on a FIFO BLOCKS
            // until a writer appears — indefinitely, on the GUI thread, and the
            // size cap cannot help because a FIFO reports size 0. The two sibling
            // scanners are immune only incidentally: they enumerate with
            // QDir::Files, which excludes a FIFO, whereas this one constructs the
            // path. A directory named metadata.json is refused by the same check.
            if (!metadataInfo.isFile()) {
                // Watch the SUBDIRECTORY instead. `QFileSystemWatcher` cannot
                // watch a path that does not exist, so the metadata.json entry
                // appended above arms nothing here — and only the registered
                // SEARCH paths are directory-watched, not the pack dirs under
                // them. Without this, `cp -r mypack <packs>/` races: the mkdir
                // wakes the debounced rescan, the rescan finds an empty (or
                // half-copied) subdir, and the metadata.json landing afterwards
                // fires no event at all, so the pack stays invisible until
                // something unrelated triggers a rescan. Watching the subdir
                // makes the file's creation the wake-up.
                desiredWatches.append(subdirPath);
                qCDebug(log) << "MetadataPackScanStrategy: skipping subdir, no metadata.json:" << subdirPath;
                continue;
            }
 
            // DoS guard: untrusted same-user metadata.json must not
            // stall the GUI thread with a 2 GB blob. Reuse
            // `DirectoryLoader::kMaxFileBytes` as the SSOT — same cap
            // the sister `JsonScanStrategy` enforces on every JSON
            // file it loads.
            if (metadataInfo.size() > DirectoryLoader::kMaxFileBytes) {
                qCWarning(log) << "MetadataPackScanStrategy: skipping oversized metadata.json:" << metadataPath << "("
                               << metadataInfo.size() << "bytes, cap" << DirectoryLoader::kMaxFileBytes << ")";
                continue;
            }
 
            QFile file(metadataPath);
            if (!file.open(QIODevice::ReadOnly)) {
                qCWarning(log) << "MetadataPackScanStrategy: failed to open metadata.json:" << metadataPath;
                continue;
            }
            // Re-checked on the OPEN descriptor. The pre-open stat above can be
            // beaten by a rewrite between the stat and the open, which leaves
            // `readAll()` below unbounded — the same TOCTOU that was closed in
            // `validateJsonEnvelope`, and this was the one remaining site that
            // still enforced the cap on the path rather than the descriptor.
            //
            // Deliberately untested: hitting this branch requires losing the
            // stat/open race on purpose, and a test cannot schedule a
            // same-process rewrite into that window deterministically. The
            // pre-open cap two blocks up carries the observable coverage; this
            // recheck is the race-closing twin and is kept correct by review.
            if (file.size() > DirectoryLoader::kMaxFileBytes) {
                qCWarning(log) << "MetadataPackScanStrategy: skipping oversized metadata.json:" << metadataPath << "("
                               << file.size() << "bytes, cap" << DirectoryLoader::kMaxFileBytes << ")";
                continue;
            }
 
            QJsonParseError parseError{};
            const QJsonDocument doc = QJsonDocument::fromJson(file.readAll(), &parseError);
            if (parseError.error != QJsonParseError::NoError) {
                qCWarning(log) << "MetadataPackScanStrategy: parse error in" << metadataPath << ":"
                               << parseError.errorString();
                continue;
            }
            if (!doc.isObject()) {
                qCWarning(log) << "MetadataPackScanStrategy: non-object root in" << metadataPath;
                continue;
            }
 
            // Schema-specific parse. `m_parser` is guarded once at the top of
            // this function rather than here, so the ctor's debug assert has a
            // release-build counterpart and this call cannot throw
            // `std::bad_function_call` out of a GUI-thread rescan.
            std::optional<Payload> parsed = m_parser(subdirPath, doc.object(), isUserDir);
            if (!parsed.has_value()) {
                qCDebug(log) << "MetadataPackScanStrategy: parser declined" << metadataPath;
                continue;
            }
            if (parsed->id.isEmpty()) {
                qCWarning(log) << "MetadataPackScanStrategy: skipping" << metadataPath << ": empty 'id' field";
                continue;
            }
 
            // First-wins on id collision. Reverse-iteration means a
            // user-dir entry claims its id before any system-dir entry
            // can; a colliding system entry is silently shadowed.
            if (seenIds.contains(parsed->id)) {
                qCDebug(log) << "MetadataPackScanStrategy: id" << parsed->id
                             << "already registered from a higher-priority dir; shadowed at:" << subdirPath;
                continue;
            }
 
            // Per-payload watches — frag/vert/kwin shaders, etc.
            if (m_perEntryWatch) {
                desiredWatches.append(m_perEntryWatch(*parsed));
            }
 
            // Capture the metadata.json fingerprint BEFORE the move so
            // we can mix it into the per-rescan signature below. Any
            // parser-consumed field's edit shifts the file's mtime (POSIX
            // guarantees the mtime is UPDATED on a content-change write;
            // the value mixed here is truncated to milliseconds, so a
            // same-size rewrite landing in the same millisecond as the
            // prior stat is theoretically invisible — accepted, because
            // the alternative is content-hashing every file on every
            // rescan, and every production save path here goes through an
            // atomic rename which also changes the inode), so a single
            // mtime+size mix-in covers every schema field without forcing
            // per-field enumeration in SignatureContrib.
            QString id = parsed->id;
            seenIds.insert(id);
            entries.push_back(
                Entry{std::move(id),
                      EntryFingerprint{metadataInfo.size(), metadataInfo.lastModified().toMSecsSinceEpoch(), isUserDir},
                      std::move(*parsed)});
        }
    }
 
    if (capTripped) {
        qCWarning(log).nospace() << "MetadataPackScanStrategy: reached entry cap (" << m_maxEntries
                                 << ") — later entries skipped to protect the GUI thread. Prune the watched search "
                                    "paths or raise the cap.";
        // The truncated set IS still committed, so `MetadataPackLoader::
        // reconcile` unregisters every pack that sorted past the cap even
        // though it is still on disk. That is deliberate here and differs from
        // PluginLoader, which skips its removal sweep on a cap trip: a pack
        // registry's accessors are queried by id from QML on every frame, and
        // leaving entries registered that this scan never validated would serve
        // stale payloads indefinitely. Dropping them is visible and recoverable
        // (prune the search path, or raise the cap, and the next scan restores
        // them); serving a stale pack is neither.
    }
 
    // Sort by id once. Stable hash + stable accessor ordering both fall
    // out of this single pass — no parallel structures, no second
    // QHash::keys() + sort.
    std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) {
        return a.id < b.id;
    });
 
    // SHA-1 signature in two passes.
    //
    // Pass 1 — per-entry attribution. (id, isUser, metadata.json size+mtime,
    // payload-specific bytes via SignatureContrib). Stable iteration order
    // is the sorted entries vec; QHash randomisation never leaks into the
    // signature.
    //
    // Pass 2 — watch-set auto-fingerprint. Every distinct file the
    // strategy is watching (the per-entry metadata.json plus everything
    // returned by `PerEntryWatchPaths` and `PerDirectoryWatchPaths`)
    // contributes path|size|mtime|. This is load-bearing for change-only
    // emit completeness: any file the watcher fires on must shift the
    // signature, otherwise the rescan runs but `OnCommit` stays silent
    // and consumers hold stale state. Without this pass, top-level
    // shared `*.glsl` includes (`common.glsl`, `audio.glsl`, …) and
    // per-pack auxiliary files (`helpers.glsl`, etc.) would fire the
    // watcher but not the consumer's content-changed signal.
    //
    // The metadata.json mtime+size shows up in BOTH passes (per-entry +
    // watch-set). Deterministic redundancy — costs nothing, keeps the
    // per-entry attribution clean.
    QCryptographicHash hasher(QCryptographicHash::Sha1);
    for (const Entry& e : entries) {
        hasher.addData(e.id.toUtf8());
        hasher.addData(QByteArrayView("|"));
        hasher.addData(e.fp.isUser ? QByteArrayView("u") : QByteArrayView("s"));
        hasher.addData(QByteArrayView("|"));
        hasher.addData(QByteArray::number(e.fp.metadataSize));
        hasher.addData(QByteArrayView("|"));
        hasher.addData(QByteArray::number(e.fp.metadataMtimeMs));
        hasher.addData(QByteArrayView("|"));
        if (m_sigContrib) {
            m_sigContrib(hasher, e.payload);
        }
        hasher.addData(QByteArrayView("\n"));
    }
    // Watch-set pass. Sorted + deduped for deterministic ordering across
    // rescans regardless of the order paths were appended during the
    // outer / inner loops above. The sort/dedup itself is cheap, but note the
    // list length is NOT bounded by m_maxEntries: the per-directory and
    // per-entry watch extractors return arbitrary-length globs (see the
    // "sit OUTSIDE the entry cap" note earlier in this file). The real
    // per-rescan cost is the QFileInfo stat (exists/size/lastModified) in the
    // loop below, one per watch path, on the GUI thread on every debounced
    // fire — accepted as the dominant syscall cost of a rescan.
    QStringList sortedWatches = desiredWatches;
    sortedWatches.removeDuplicates();
    std::sort(sortedWatches.begin(), sortedWatches.end());
    for (const QString& path : sortedWatches) {
        hasher.addData(path.toUtf8());
        hasher.addData(QByteArrayView("|"));
        const QFileInfo fi(path);
        if (fi.exists()) {
            hasher.addData(QByteArray::number(fi.size()));
            hasher.addData(QByteArrayView("|"));
            hasher.addData(QByteArray::number(fi.lastModified().toMSecsSinceEpoch()));
        } else {
            // Stable sentinel for absent files. `lastModified()` on an
            // invalid datetime is implementation-defined — explicit
            // "missing" keeps the hash format portable across Qt
            // versions and filesystems.
            hasher.addData(QByteArrayView("missing"));
        }
        hasher.addData(QByteArrayView("\n"));
    }
    const QByteArray signature = hasher.result();
 
    QHash<QString, Payload> fresh;
    fresh.reserve(static_cast<int>(entries.size()));
    for (Entry& e : entries) {
        fresh.insert(e.id, std::move(e.payload));
    }
 
    const bool isFirstScan = !m_signatureSeeded;
    const bool changed = isFirstScan ? !fresh.isEmpty() : signature != m_lastSignature;
 
    m_packs = std::move(fresh);
    m_lastSignature = signature;
    m_signatureSeeded = true;
 
    if (changed && m_onCommit) {
        m_onCommit();
    }
 
    // Return the deduped + lex-sorted watch set rather than the raw
    // append-order `desiredWatches`. The caller (`WatchedDirectorySet`)
    // dedupes again internally via `QSet<QString> m_watchedFiles`, so
    // this isn't a correctness fix — but the strategy already paid for
    // dedup + sort during the signature pass, so handing the cleaned
    // list back saves the watcher its own dedup pass, gives the signature
    // pass a deterministic input, and lets a test assert on the returned
    // list directly. (It does NOT make the watcher's own diagnostics
    // deterministic — syncFileWatches copies into a QSet and iterates that.)
    // Costs nothing.
    return sortedWatches;
}
 
} // namespace PhosphorFsLoader