2021-08-25 14:24:54 (UTC-03:00)

Marcel Rodrigues <marcelgmr@gmail.com>

map: use rice coding for deltas in cache

This cuts ~1/3 of the cache size (in relation to VLPs).

diff --git a/bio.lua b/bio.lua
index 57a06c1..43dba6a 100644
--- a/bio.lua
+++ b/bio.lua
@@ -59,26 +59,13 @@ local function read_led64(fp)
     return ffi.cast("double *", ffi.new("char[8]", fp:read(8)))[0]
 end
 
-local function read_leuvlp(fp)
-    local x, y = 0, 0
-    local s = 0
-    repeat
-        local byte = fp:read(1):byte(1)
-        l, r = bit.band(bit.rshift(byte, 4), 0x07), bit.band(byte, 0x07)
-        x = bit.bor(bit.lshift(l, s), x)
-        y = bit.bor(bit.lshift(r, s), y)
-        s = s + 3
-    until bit.band(byte, 0x88) == 0
-    return x, y
-end
-
-local function read_leivlp(fp)
-    local x, y = read_leuvlp(fp)
-    return u2s(x), u2s(y)
+local function write_byte(fp, n)
+    fp:write(string.char(n))
 end
 
 local function write_beu16(fp, n)
-    fp:write(string.char(bit.rshift(n, 8), bit.band(n, 0xFF)))
+    write_byte(fp, bit.rshift(n, 8))
+    write_byte(fp, bit.band(n, 0xFF))
 end
 
 local function write_beu32(fp, n)
@@ -93,26 +80,89 @@ local function write_bei16(fp, n)
     write_beu16(fp, n)
 end
 
-local function write_leuvlp(fp, x, y)
-    repeat
-        local byte = bit.bor(bit.lshift(bit.band(x, 0x07), 4), bit.band(y, 0x07))
-        x, y = bit.rshift(x, 3), bit.rshift(y, 3)
-        if x ~= 0 or y ~= 0 then
-            byte = bit.bor(byte, 0x88)
-        end
-        fp:write(string.char(byte))
-    until x == 0 and y == 0
+local RiceR = {}
+RiceR.__index = RiceR
+
+function RiceR:get_bit()
+    if self.n == 0 then
+        self.b = read_byte(self.fp)
+        self.n = 8
+    end
+    self.n = self.n - 1
+    return bit.band(bit.rshift(self.b, self.n), 1)
+end
+
+function RiceR:get_unsigned()
+    local q = 0
+    while self:get_bit() == 1 do
+        q = q + 1
+    end
+    local r = 0
+    for i = 1, self.k do
+        r = bit.bor(bit.lshift(r, 1), self:get_bit())
+    end
+    return bit.bor(bit.lshift(q, self.k), r)
+end
+
+function RiceR:get_signed()
+    return u2s(self:get_unsigned())
+end
+
+local function rice_r(fp, k)
+    local self = setmetatable({}, RiceR)
+    self.fp = fp    -- already opened file, read mode
+    self.k = k or 1 -- rice parameter
+    self.b = 0      -- value of last byte read
+    self.n = 0      -- number of bits available in self.b
+    return self
+end
+
+local RiceW = {}
+RiceW.__index = RiceW
+
+function RiceW:put_bit(b)
+    self.n = self.n - 1
+    self.b = bit.bor(self.b, bit.lshift(b, self.n))
+    if self.n == 0 then
+        self:flush()
+    end
+end
+
+function RiceW:flush()
+    if self.n ~= 8 then
+        write_byte(self.fp, self.b)
+        self.b = 0
+        self.n = 8
+    end
+end
+
+function RiceW:put_unsigned(n)
+    for i = 1, bit.rshift(n, self.k) do
+        self:put_bit(1)
+    end
+    self:put_bit(0)
+    for i = self.k-1, 0, -1 do
+        self:put_bit(bit.band(bit.rshift(n, i), 1))
+    end
+end
+
+function RiceW:put_signed(n)
+    self:put_unsigned(s2u(n))
 end
 
-local function write_leivlp(fp, x, y)
-    write_leuvlp(fp, s2u(x), s2u(y))
+local function rice_w(fp, k)
+    local self = setmetatable({}, RiceW)
+    self.fp = fp    -- already opened file, write mode
+    self.k = k or 1 -- rice parameter
+    self.b = 0      -- value of next byte to write
+    self.n = 8      -- number of bits available in self.b
+    return self
 end
 
 return {
     read_byte=read_byte, read_leu16=read_leu16, read_leu32=read_leu32,
     read_beu16=read_beu16, read_beu32=read_beu32, read_lei16=read_lei16,
     read_lei32=read_lei32, read_bei16=read_bei16, read_bei32=read_bei32,
-    read_led64=read_led64, read_leuvlp=read_leuvlp, read_leivlp=read_leivlp,
-    write_beu16=write_beu16, write_beu32=write_beu32, write_bei16=write_bei16,
-    write_leuvlp=write_leuvlp, write_leivlp=write_leivlp
+    read_led64=read_led64, write_byte=write_byte, write_beu16=write_beu16,
+    write_beu32=write_beu32, write_bei16=write_bei16, rice_r=rice_r, rice_w=rice_w
 }

diff --git a/map.lua b/map.lua
index f04fd09..9d1a189 100644
--- a/map.lua
+++ b/map.lua
@@ -249,17 +249,22 @@ function Frame:save_cache(fname, sf, filter)
         for poly in self:mapped(polys) do
             local ox, oy = unpack(poly())
             ox, oy = math.floor(ox + 0.5), math.floor(oy + 0.5)
-            bio.write_leivlp(cache, ox, oy)
+            bio.write_bei16(cache, ox)
+            bio.write_bei16(cache, oy)
+            local rice = bio.rice_w(cache)
             for point in poly do
                 local x, y = unpack(point)
                 x, y = math.floor(x + 0.5), math.floor(y + 0.5)
                 local dx, dy = x-ox, y-oy
                 if dx ~= 0 or dy ~= 0 then
-                    bio.write_leivlp(cache, dx, dy)
+                    rice:put_signed(dx)
+                    rice:put_signed(dy)
                     ox, oy = x, y
                 end
             end
-            bio.write_leivlp(cache, 0, 0)
+            rice:put_signed(0)
+            rice:put_signed(0)
+            rice:flush()
         end
     end
     cache:close()
@@ -362,12 +367,13 @@ function Cache:get_polys(key)
     return function()
         if npolys > 0 then
             npolys = npolys - 1
-            local ox, oy = bio.read_leivlp(cache)
+            local ox, oy = bio.read_bei16(cache), bio.read_bei16(cache)
+            local rice = bio.rice_r(cache)
             local x, y = 0, 0
             return function()
                 if x ~= ox or y ~= oy then
                     x, y = ox, oy
-                    local dx, dy = bio.read_leivlp(cache)
+                    local dx, dy = rice:get_signed(), rice:get_signed()
                     ox, oy = ox+dx, oy+dy
                     return {x, y}
                 end