summaryrefslogtreecommitdiff
path: root/src/util/asset_manager.cc
diff options
context:
space:
mode:
authorskal <pascal.massimino@gmail.com>2026-05-14 19:09:39 +0200
committerskal <pascal.massimino@gmail.com>2026-05-14 19:11:28 +0200
commit6ef8f578817ee0134fd5867ca3b80590e3eb2368 (patch)
tree5550607e5c4a16ca237bfa4430ac1ef1f5d80c5d /src/util/asset_manager.cc
parent4bcbe13dab5ffb64d93cc61956f07ee5168a84c9 (diff)
ans: order-0 rANS coder + WGSL asset compression
Adds src/util/ans.{h,cc}, a per-chunk-adaptive order-0 rANS entropy coder. Decoder is always built; encoder is gated on ANS_ENABLE_ENCODER (tools only). Both sides take an optional 256-entry initial_counts table to seed the adaptive model. The per-chunk initial state is (1 << kBits). Higher initial states (e.g. with a signature packed into the upper bits) force a renorm-emit at iter 0 that the decoder never consumes, corrupting multi-chunk streams once stats become skewed. Asset pipeline: - AssetRecord gains 'compression' and 'uncompressed_size' fields. - asset_packer scans every WGSL file to build a corpus-wide byte histogram, then ANS-encodes each shader using that histogram as the seed. Histogram and accessor are emitted alongside the asset table. Round-trip verification runs at pack time for every compressed asset; failures fall back to uncompressed storage. - asset_manager decompresses on first GetAsset(), caches the heap-allocated buffer, and DropAsset / ReloadAssetsFromFile free it along with the procedural cache. - Disk-load (dev) builds are unchanged: WGSL paths stay as filenames. Tests: - src/tests/util/test_ans.cc: roundtrip variants (empty, single byte, single-symbol run, all-zeros, random uniform/skewed, repeated ASCII), seeded-vs-uniform compression, rejection of mismatched counts / corruption / truncation, PeekUncompressedSize. - 37/37 dev, 36/36 STRIP_ALL. Compression observed: WGSL shaders shrink to ~0.62-0.71x in the main workspace (81 of 105 assets qualify). Docs: - doc/ANS.md (new): algorithm, bitstream, API, asset pipeline integration, compression numbers, limitations, tests. - doc/ASSET_SYSTEM.md: new Compression section + updated technical guarantees for compressed assets. - doc/COMPLETED.md: May 2026 entry. - PROJECT_CONTEXT.md: Build status line mentions WGSL ANS compression. - CLAUDE.md, GEMINI.md: tier-3 build doc list includes ANS.md.
Diffstat (limited to 'src/util/asset_manager.cc')
-rw-r--r--src/util/asset_manager.cc44
1 files changed, 40 insertions, 4 deletions
diff --git a/src/util/asset_manager.cc b/src/util/asset_manager.cc
index 26a82bf..264ddda 100644
--- a/src/util/asset_manager.cc
+++ b/src/util/asset_manager.cc
@@ -2,6 +2,7 @@
// It implements the generic asset retrieval logic with runtime caching.
#include "util/asset_manager.h"
+#include "util/ans.h"
#include "util/asset_manager_utils.h"
#include "util/check_return.h"
#if defined(USE_TEST_ASSETS)
@@ -114,6 +115,34 @@ const uint8_t* GetAsset(AssetId asset_id, size_t* out_size) {
AssetRecord cached_record = source_record;
+ if (source_record.compression == AssetCompression::ANS_ASCII) {
+ // Decompress on first access. Buffer is null-terminated for C-string use.
+ const size_t uncomp = source_record.uncompressed_size;
+ uint8_t* buffer = new (std::nothrow) uint8_t[uncomp + 1];
+ CHECK_RETURN_BEGIN(!buffer, nullptr)
+ if (out_size) *out_size = 0;
+ ERROR_MSG("Failed to allocate buffer for ANS-compressed asset id=%u",
+ index);
+ return nullptr;
+ CHECK_RETURN_END
+ size_t got = 0;
+ if (!ans::Decode(source_record.data, source_record.size, buffer, uncomp,
+ &got, GetAnsAsciiHistogram()) ||
+ got != uncomp) {
+ delete[] buffer;
+ if (out_size) *out_size = 0;
+ ERROR_MSG("ANS decode failed for asset id=%u", index);
+ return nullptr;
+ }
+ buffer[uncomp] = 0;
+ cached_record.data = buffer;
+ cached_record.size = uncomp;
+ cached_record.uncompressed_size = uncomp;
+ g_asset_cache[index] = cached_record;
+ if (out_size) *out_size = uncomp;
+ return buffer;
+ }
+
if (source_record.type == AssetType::PROC ||
source_record.type == AssetType::PROC_GPU) {
// Dynamically generate the asset
@@ -224,6 +253,12 @@ void DropAsset(AssetId asset_id, const uint8_t* asset) {
delete[] g_asset_cache[index].data;
g_asset_cache[index] = {}; // Zero out the struct to force re-generation
}
+ // Heap-allocated decompressed buffer (compression != NONE): cache owns it.
+ if (g_asset_cache[index].data == asset &&
+ g_asset_cache[index].compression != AssetCompression::NONE) {
+ delete[] g_asset_cache[index].data;
+ g_asset_cache[index] = {};
+ }
#if !defined(STRIP_ALL)
if (g_asset_cache[index].data == asset &&
(g_asset_cache[index].type == AssetType::SPEC ||
@@ -245,10 +280,11 @@ bool ReloadAssetsFromFile(const char* config_path) {
// Clear cache to force reload
for (size_t i = 0; i < (size_t)AssetId::ASSET_LAST_ID; ++i) {
- if ((g_asset_cache[i].type == AssetType::PROC ||
- g_asset_cache[i].type == AssetType::PROC_GPU) &&
- g_asset_cache[i].data) {
- delete[] g_asset_cache[i].data;
+ const AssetRecord& e = g_asset_cache[i];
+ if (e.data &&
+ (e.type == AssetType::PROC || e.type == AssetType::PROC_GPU ||
+ e.compression != AssetCompression::NONE)) {
+ delete[] e.data;
}
g_asset_cache[i] = {};
}