render: add background shader compile skeleton and persistent shader cache

src/render/shadercache/README.md

@@ -0,0 +1,4 @@
+# Shader cache
+
+This folder contains a small background shader compile + cache skeleton.
+Cache files are written to the configured cache directory as `<key>.bin` and `<key>.meta`.

src/render/shadercache/ShaderCache.cpp

@@ -0,0 +1,107 @@
+#include "ShaderCache.h"
+#include <fstream>
+#include <chrono>
+#include <iostream>
+
+using namespace std::chrono;
+namespace fs = std::filesystem;
+
+ShaderCache::ShaderCache(const std::string& cacheDir) : cacheDir_(cacheDir) {
+  // ensure dir exists
+  try {
+    fs::create_directories(cacheDir_);
+  } catch (...) {}
+}
+
+ShaderCache::~ShaderCache() = default;
+
+std::string ShaderCache::cacheDir() const { return cacheDir_; }
+
+void ShaderCache::loadFromDisk() {
+  std::lock_guard<std::mutex> lk(m_);
+  try {
+    for (auto &p : fs::directory_iterator(cacheDir_)) {
+      if (!p.is_regular_file()) continue;
+      auto path = p.path();
+      if (path.extension() == ".bin") {
+        try {
+          auto entry = readEntryFromFile(path);
+          map_.emplace(entry.key, std::move(entry));
+        } catch (const std::exception& e) {
+          std::cerr << "[ShaderCache] error reading " << path << ": " << e.what() << std::endl;
+        }
+      }
+    }
+    std::cerr << "[ShaderCache] loaded " << map_.size() << " entries from disk\n";
+  } catch (const std::exception& e) {
+    std::cerr << "[ShaderCache] loadFromDisk error: " << e.what() << std::endl;
+  }
+}
+
+bool ShaderCache::tryGet(const std::string& key, std::vector<uint8_t>& outBlob) {
+  std::lock_guard<std::mutex> lk(m_);
+  auto it = map_.find(key);
+  if (it == map_.end()) return false;
+  outBlob = it->second.blob;
+  std::cerr << "[ShaderCache] cache hit: " << key << " (size=" << outBlob.size() << ")\n";
+  return true;
+}
+
+void ShaderCache::put(const std::string& key, const std::vector<uint8_t>& blob, const std::string& gpuId) {
+  ShaderCacheEntry e;
+  e.key = key;
+  e.blob = blob;
+  e.gpu_identifier = gpuId;
+  e.timestamp = static_cast<uint64_t>(duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count());
+
+  {
+    std::lock_guard<std::mutex> lk(m_);
+    map_[key] = e;
+  }
+  persistEntryToDisk(e);
+  std::cerr << "[ShaderCache] put: " << key << " (size=" << blob.size() << ")\n";
+}
+
+void ShaderCache::persistEntryToDisk(const ShaderCacheEntry& e) {
+  try {
+    fs::create_directories(cacheDir_);
+    fs::path binPath = fs::path(cacheDir_) / (e.key + ".bin");
+    fs::path metaPath = fs::path(cacheDir_) / (e.key + ".meta");
+
+    {
+      std::ofstream out(binPath, std::ios::binary);
+      out.write(reinterpret_cast<const char*>(e.blob.data()), static_cast<std::streamsize>(e.blob.size()));
+    }
+    {
+      std::ofstream meta(metaPath);
+      meta << e.gpu_identifier << "\n" << e.timestamp << "\n";
+    }
+  } catch (const std::exception& ex) {
+    std::cerr << "[ShaderCache] persist error: " << ex.what() << std::endl;
+  }
+}
+
+ShaderCacheEntry ShaderCache::readEntryFromFile(const std::filesystem::path& p) {
+  ShaderCacheEntry e;
+  e.key = p.stem().string(); // filename without extension
+  // read blob
+  std::ifstream in(p, std::ios::binary | std::ios::ate);
+  if (!in) throw std::runtime_error("cannot open bin file");
+  auto size = in.tellg();
+  in.seekg(0);
+  e.blob.resize(static_cast<size_t>(size));
+  in.read(reinterpret_cast<char*>(e.blob.data()), size);
+
+  // read metadata file
+  fs::path meta = p.parent_path() / (e.key + ".meta");
+  if (fs::exists(meta)) {
+    std::ifstream m(meta);
+    if (m) {
+      std::getline(m, e.gpu_identifier);
+      std::string ts;
+      std::getline(m, ts);
+      if (!ts.empty()) e.timestamp = std::stoull(ts);
+    }
+  }
+  return e;
+}

src/render/shadercache/ShaderCache.h

@@ -0,0 +1,39 @@
+#pragma once
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <mutex>
+#include <filesystem>
+
+struct ShaderCacheEntry {
+  std::string key;
+  std::vector<uint8_t> blob;
+  std::string gpu_identifier;
+  uint64_t timestamp;
+};
+
+class ShaderCache {
+public:
+  explicit ShaderCache(const std::string& cacheDir);
+  ~ShaderCache();
+
+  // load persisted entries metadata (called at startup)
+  void loadFromDisk();
+
+  // try to fetch compiled blob (returns true if found and set outBlob)
+  bool tryGet(const std::string& key, std::vector<uint8_t>& outBlob);
+
+  // put compiled blob into cache and persist to disk
+  void put(const std::string& key, const std::vector<uint8_t>& blob, const std::string& gpuId);
+
+  // simple helper: returns cache dir
+  std::string cacheDir() const;
+
+private:
+  void persistEntryToDisk(const ShaderCacheEntry& e);
+  ShaderCacheEntry readEntryFromFile(const std::filesystem::path& p);
+
+  std::string cacheDir_;
+  std::unordered_map<std::string, ShaderCacheEntry> map_;
+  std::mutex m_;
+};

src/render/shadercache/ShaderCompileWorker.cpp

@@ -0,0 +1,57 @@
+#include "ShaderCompileWorker.h"
+#include <iostream>
+#include <chrono>
+#include <thread>
+
+// A tiny "fake" compile function to simulate compilation work.
+// Replace this with actual renderer compile in the integration step.
+static bool FakeCompileToBlob(const std::string& src, std::vector<uint8_t>& out) {
+  // Simulate some work and produce a small blob
+  std::this_thread::sleep_for(std::chrono::milliseconds(150)); // simulate compile latency
+  out.assign(src.begin(), src.end()); // just copy source bytes as a stand-in
+  return true;
+}
+
+ShaderCompileWorker::ShaderCompileWorker(ShaderCache& cache) : cache_(cache) {}
+ShaderCompileWorker::~ShaderCompileWorker() { stop(); }
+
+void ShaderCompileWorker::start() {
+  stop_.store(false);
+  worker_ = std::thread([this]{ threadMain(); });
+}
+
+void ShaderCompileWorker::stop() {
+  stop_.store(true);
+  // push a dummy to wake up
+  queue_.push(CompileTask{});
+  if (worker_.joinable()) worker_.join();
+}
+
+void ShaderCompileWorker::enqueue(CompileTask task) {
+  queue_.push(std::move(task));
+}
+
+void ShaderCompileWorker::threadMain() {
+  while (true) {
+    auto opt = queue_.pop_wait(stop_);
+    if (!opt.has_value()) break;
+    CompileTask task = std::move(*opt);
+    if (task.key.empty()) continue; // ignore wake-up dummy
+
+    // Double-check cache to avoid duplicate work
+    std::vector<uint8_t> existing;
+    if (cache_.tryGet(task.key, existing)) continue;
+
+    std::vector<uint8_t> compiled;
+    bool ok = FakeCompileToBlob(task.source, compiled); // TODO replace with real compile
+
+    if (ok) {
+      cache_.put(task.key, compiled, task.gpuId);
+      std::cerr << "[ShaderWorker] compiled key=" << task.key << " size=" << compiled.size() << "\n";
+      // Optionally: enqueue a main-thread action to create the GPU shader object from compiled blob.
+      // This must be executed on the render thread because many APIs require a valid context.
+    } else {
+      std::cerr << "[ShaderWorker] compile failed for key=" << task.key << "\n";
+    }
+  }
+}

src/render/shadercache/ShaderCompileWorker.h

@@ -0,0 +1,31 @@
+#pragma once
+#include "ThreadSafeQueue.h"
+#include "ShaderCache.h"
+#include <thread>
+#include <atomic>
+#include <string>
+#include <vector>
+
+struct CompileTask {
+  std::string key;        // hash of source + options
+  std::string source;     // shader source (or path to IR)
+  std::string gpuId;      // GPU id used for this compile
+};
+
+class ShaderCompileWorker {
+public:
+  explicit ShaderCompileWorker(ShaderCache& cache);
+  ~ShaderCompileWorker();
+
+  void start();
+  void stop();
+
+  void enqueue(CompileTask task);
+
+private:
+  void threadMain();
+  ThreadSafeQueue<CompileTask> queue_;
+  std::thread worker_;
+  std::atomic<bool> stop_{false};
+  ShaderCache& cache_;
+};

src/render/shadercache/ThreadSafeQueue.h

@@ -0,0 +1,37 @@
+#pragma once
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <optional>
+
+template<typename T>
+class ThreadSafeQueue {
+public:
+  ThreadSafeQueue() = default;
+  ~ThreadSafeQueue() = default;
+
+  void push(T item) {
+    {
+      std::lock_guard<std::mutex> lk(m_);
+      q_.push(std::move(item));
+    }
+    cv_.notify_one();
+  }
+
+  std::optional<T> pop_wait(std::atomic<bool>& stopFlag) {
+    std::unique_lock<std::mutex> lk(m_);
+    cv_.wait(lk, [&]{ return stopFlag.load() || !q_.empty(); });
+    if (stopFlag.load() && q_.empty()) return std::nullopt;
+    T it = std::move(q_.front());
+    q_.pop();
+    return it;
+  }size_t size() const {
+    std::lock_guard<std::mutex> lk(m_);
+    return q_.size();
+  }
+
+private:
+  mutable std::mutex m_;
+  std::condition_variable cv_;
+  std::queue<T> q_;
+};EOF