Update with palettes and dual neural network

.gitignore

@@ -2,4 +2,5 @@
 __pycache__
 blender-3.3.1-linux-x64
 data
-voxelsight.ckpt
+*.ckpt
+wandb

config.py

@@ -1,7 +1,10 @@
 downsample = 4
+native_screen_res = (640, 360)
 # C x H x W
-screen_size = (4, int(360/downsample), int(640/downsample))
+screen_size = (4, int(native_screen_res[1]/downsample), int(native_screen_res[0]/downsample))
 quantized_screen_size = (screen_size[1] * screen_size[2], 2)
 
 voxel_grid_size = (16*16,)
-dataset_size = 1000
+
+datasets = {'train':100}
+# datasets = {'train':5000, 'test':1000, 'validate':500}

find_visible_voxels.py

@@ -1,25 +1,26 @@
 from PIL import Image
 from config import *
+import os
 
-dir = "data/"
-
-for i in range(dataset_size):
-    with Image.open(dir + "occlusion_elements_{}.png".format(i)) as img:
-        d = img.getdata()
-        pixel_counts = {}
-        for pixel in d:
-            if pixel[3] > 0.0:
-                pixel = pixel[:3] # remove alpha channel
-                if pixel in pixel_counts:
-                    pixel_counts[pixel] = pixel_counts[pixel] + 1
-                else:
-                    pixel_counts[pixel] = 1
-        with open(dir + "voxel_scene_{}".format(i)) as scene:
-            with open(dir + "visible_elements_{}".format(i), "w+") as visible:
-                lines = scene.readlines()
-                for line in lines:
-                    data = line.split("/")
-                    color = tuple([int(v) for v in data[1][1:-1].split(",")])
-                    height = int(data[2]) 
-                    if color in pixel_counts:
-                        visible.write(str(data[0]) + "," + str(height) + "," + str(pixel_counts[color]) + "\n")
+for dataset in datasets.items():
+    dir = os.path.join("data", dataset[0])
+    for i in range(dataset[1]):
+        with Image.open(os.path.join(dir, "occlusion_elements_{}.png".format(i))) as img:
+            d = img.getdata()
+            pixel_counts = {}
+            for pixel in d:
+                if pixel[3] > 0.0:
+                    pixel = pixel[:3] # remove alpha channel
+                    if pixel in pixel_counts:
+                        pixel_counts[pixel] = pixel_counts[pixel] + 1
+                    else:
+                        pixel_counts[pixel] = 1
+            with open(os.path.join(dir, "voxel_scene_{}".format(i))) as scene:
+                with open(os.path.join(dir, "visible_elements_{}".format(i)), "w+") as visible:
+                    lines = scene.readlines()
+                    for line in lines:
+                        data = line.split("/")
+                        color = tuple([int(v) for v in data[1][1:-1].split(",")])
+                        height = int(data[2]) 
+                        if color in pixel_counts:
+                            visible.write(str(data[0]) + "," + str(height) + "," + str(pixel_counts[color]) + "\n")

gendata.sh

@@ -1,2 +1 @@
-./blender-3.3.1-linux-x64/blender -b voxel.blend --python generate_dataset.py &&
-python find_visible_voxels.py
+./blender-3.3.1-linux-x64/blender -b voxel.blend --python generate_dataset.py

generate_dataset.py

@@ -8,6 +8,13 @@
 import numpy as np
 import math
 import bmesh
+import os
+import sys
+
+sys.path.append(".") # <--- voxel_sight directory packages
+from config import *
+
+# TODO: Should be able to import config package by manually specifying search directory
 
 class TerrainNoiseLayer():
     scale = 1.0
@@ -39,7 +46,7 @@ def spawn_voxel_occlusion_heightmap(noise, verts, colors, rgb, start_x, start_y,
         for x in range(width):
             n = noise[int(start_y) + y][int(start_x) + x]
             location = Vector(((start_x + x), (start_y + y), round(n)))
-            byte_color = [int(start_x+x), int(start_y+y), int(random.randint(0, 255))]
+            byte_color = [64+int(start_x*2+x), 64+int(start_y*2+y), int(random.randint(0, 255))]
             color = [byte_color[0]/255.0, byte_color[1]/255.0, byte_color[2]/255.0, 1.0]
             verts.append(location)
             colors.append(color)
@@ -55,18 +62,20 @@ def spawn_voxel_heightmap(noise, verts, start_x, start_y, width, height, scale):
 def get_heightmap_baseline():
     pass
 
-def spawn_occlusion_chunks(radius, noise, d):
+def spawn_occlusion_chunks(radius, noise, d, dir):
     mesh = bpy.data.meshes.new("example")
     ob = bpy.data.objects.new("placer", mesh)
     bpy.context.collection.objects.link(ob)
     ob.location = Vector((0,0,0))
     verts = []
     rgb = []
     colors = []
-    for y in range(0, radius):
-        for x in range(0, radius*2):
-            chunk_origin = Vector((x*chunk_size, y*chunk_size, 0.0))
-            spawn_voxel_occlusion_heightmap(noise, verts, colors, rgb, chunk_origin.x, chunk_origin.y, chunk_size, chunk_size, 1.0)
+    chunk_origin = Vector((0.0, 0.0, 0.0))
+    spawn_voxel_occlusion_heightmap(noise, verts, colors, rgb, chunk_origin.x, chunk_origin.y, chunk_size * 2, chunk_size * 2, 1.0)
+    # for y in range(0, radius):
+    #     for x in range(0, radius*2):
+    #         chunk_origin = Vector((x*chunk_size, y*chunk_size, 0.0))
+    #         spawn_voxel_occlusion_heightmap(noise, verts, colors, rgb, chunk_origin.x, chunk_origin.y, chunk_size, chunk_size, 1.0)
     mesh.from_pydata(verts, [], [])
     bpy.context.view_layer.objects.active = ob
     bpy.ops.object.modifier_add(type='NODES')
@@ -78,18 +87,19 @@ def spawn_occlusion_chunks(radius, noise, d):
     ob.scale = Vector((block_size, block_size, block_size))
     for i, cd in enumerate(bpy.context.active_object.data.attributes['color'].data):
         cd.color = colors[i]
-    with open('data/voxel_scene_{}'.format(d), 'w+', encoding='utf-8') as vs:
+    with open(os.path.join(dir, 'voxel_scene_{}'.format(d)), 'w+', encoding='utf-8') as vs:
         for i in range(len(verts)):
             vs.write("{}/{}/{}\n".format(str(i), str(rgb[i]), str(int((verts[i].z-player_floor)))))
     return ob
 
 def spawn_chunks(radius, noise):
     terrain = []
     verts = []
-    for y in range(0, radius):
-        for x in range(0, radius*2):
-            chunk_origin = Vector((x*chunk_size, y*chunk_size, 0.0))
-            spawn_voxel_heightmap(noise, verts, chunk_origin.x, chunk_origin.y, chunk_size, chunk_size, 1.0)
+    chunk_origin = Vector((0.0, 0.0, 0.0))
+    spawn_voxel_heightmap(noise, verts, chunk_origin.x, chunk_origin.y, chunk_size * 2, chunk_size * 2, 1.0)
+    # for y in range(0, radius):
+    #     for x in range(0, radius*2):
+            # chunk_origin = Vector((x*chunk_size, y*chunk_size, 0.0))
     voxel_types = ['dirt', 'grass', 'sand', 'stone']
     voxel_verts = {}
     for vt in voxel_types:
@@ -118,42 +128,48 @@ def spawn_chunks(radius, noise):
 bpy.context.scene.render.image_settings.color_depth = "16"
 bpy.context.scene.render.image_settings.compression = 0
 
-downsample = 4
-bpy.context.scene.render.resolution_x = int(640/downsample)
-bpy.context.scene.render.resolution_y = int(360/downsample)
+bpy.context.scene.render.resolution_x = screen_size[2]
+bpy.context.scene.render.resolution_y = screen_size[1]
+
+dir = "train"
+data_count = 10
 
-dataset_size = 1000
-for d in range(dataset_size):
-    # Recreate noise layers for random seeds!
-    for layer in noise_layers:
-        layer.set_random_seed() # TODO: Use a deterministic seed chain for replicability 
-    # Make voxels random sizes to make the prediction robust to distinct voxel grid sizes
-    # 0.1 to 0.5
-    block_size = max(0.1, random.random()/2.0)
-#    block_size = 0.2
-    bpy.data.objects["Camera"].location.z = block_size * 1.8
+def generate_scene_data(dir, data_count):
+    global block_size
+    global noise_layers
+    for d in range(data_count):
+        # Recreate noise layers for random seeds!
+        for layer in noise_layers:
+            layer.set_random_seed() # TODO: Use a deterministic seed chain for replicability 
+        # Make voxels random sizes to make the prediction robust to distinct voxel grid sizes
+        # 0.1 to 0.5
+        block_size = max(0.1, random.random()/2.0)
+    #    block_size = 0.2
+        bpy.data.objects["Camera"].location.z = block_size * 1.8
+        bpy.context.scene.view_settings.view_transform = 'Filmic'
+        # Rotate camera randomly (always pointing forward to ensure reward is met)
+        bpy.data.objects["Camera"].rotation_euler.x = math.radians(random.randint(45, 115))
+        bpy.data.objects["Camera"].rotation_euler.z = math.radians(random.randint(-44, 44))
+        # Spawn random terrain
+        radius = 1
+        noise = np.zeros(shape=[chunk_size*radius*2, chunk_size*radius*2])
+        for layer in noise_layers:
+            opensimplex.seed(layer.seed)
+            ix, iy = np.array([x * layer.scale for x in range(chunk_size*radius*2)]), np.array([y * layer.scale for y in range(chunk_size*radius*2)])
+            noise = np.add(noise, opensimplex.noise2array(ix, iy)*layer.multiplier)
+        # Render voxel terrain
+        terrain = spawn_chunks(radius, noise)
+        bpy.context.scene.render.filepath = dir + "/" + "voxels_%d.png" % d
+        bpy.ops.render.render(write_still = True)
+        for ob in terrain:
+            bpy.data.objects.remove(ob, do_unlink=True)
+        # Destroy textured voxels
+        # Render occlusion terrain
+        occlusion = spawn_occlusion_chunks(radius, noise, d, dir)
+        bpy.context.scene.view_settings.view_transform = 'Raw'
+        bpy.context.scene.render.filepath = os.path.join(dir, "occlusion_elements_%d.png" % d)
+        bpy.ops.render.render(write_still = True)
+        bpy.data.objects.remove(occlusion, do_unlink=True)
 
-    bpy.context.scene.view_settings.view_transform = 'Filmic'
-    # Rotate camera randomly (always pointing forward to ensure reward is met)
-    bpy.data.objects["Camera"].rotation_euler.x = math.radians(random.randint(45, 115))
-    bpy.data.objects["Camera"].rotation_euler.z = math.radians(random.randint(-44, 44))
-    # Spawn random terrain
-    radius = 1
-    noise = np.zeros(shape=[chunk_size*radius*2, chunk_size*radius*2])
-    for layer in noise_layers:
-        opensimplex.seed(layer.seed)
-        ix, iy = np.array([x * layer.scale for x in range(chunk_size*radius*2)]), np.array([y * layer.scale for y in range(chunk_size*radius*2)])
-        noise = np.add(noise, opensimplex.noise2array(ix, iy)*layer.multiplier)
-    # Render voxel terrain
-    terrain = spawn_chunks(radius, noise)
-    bpy.context.scene.render.filepath = "data/voxels_%d.png" % d
-    bpy.ops.render.render(write_still = True)
-    for ob in terrain:
-        bpy.data.objects.remove(ob, do_unlink=True)
-    # Destroy textured voxels
-    # Render occlusion terrain
-    occlusion = spawn_occlusion_chunks(radius, noise, d)
-    bpy.context.scene.view_settings.view_transform = 'Raw'
-    bpy.context.scene.render.filepath = "data/occlusion_elements_%d.png" % d
-    bpy.ops.render.render(write_still = True)
-    bpy.data.objects.remove(occlusion, do_unlink=True)
+for dataset in datasets.items():
+    generate_scene_data(os.path.join("data", dataset[0]), dataset[1])

load_scene.py

@@ -0,0 +1,4 @@
+load_scene = "/home/aeri/il/minerl/voxel_sight/data/train/voxel_scene_0"
+with open(load_scene) as scene:
+    voxels = [line.strip() for line in scene.readlines()]
+    print(voxels)

p32/aerugo.hex

@@ -0,0 +1,32 @@
+2f1e1a
+4f3322
+723627
+95392c
+c75533
+e76d46
+934e28
+a2663c
+c87d40
+f5a95b
+6b8b8c
+81a38e
+aac39e
+ffffff
+d1d0ce
+bab7b2
+898a8a
+686461
+554d4b
+3c3d3b
+343230
+87d1ef
+64a1c2
+466480
+2f485c
+242e35
+1b2026
+aa9c8a
+917f6d
+86624a
+715b48
+5e4835

p32/dawnbringer-32.hex

@@ -0,0 +1,32 @@
+000000
+222034
+45283c
+663931
+8f563b
+df7126
+d9a066
+eec39a
+fbf236
+99e550
+6abe30
+37946e
+4b692f
+524b24
+323c39
+3f3f74
+306082
+5b6ee1
+639bff
+5fcde4
+cbdbfc
+ffffff
+9badb7
+847e87
+696a6a
+595652
+76428a
+ac3232
+d95763
+d77bba
+8f974a
+8a6f30

p32/fantasy.hex

@@ -0,0 +1,32 @@
+ede4da
+bfb8b4
+918d8d
+636167
+353540
+a94949
+ca5954
+e56f4b
+e39347
+eeb551
+e8c65b
+bda351
+8b9150
+557d55
+446350
+3e554c
+8bb0ad
+769fa6
+668da9
+5c699f
+5a5888
+7c6da2
+947a9d
+bc87a5
+d9a6a6
+d4c2b6
+bdaa97
+86735b
+7e674c
+735b42
+604b3d
+4d3f38

p32/hept32.hex

@@ -0,0 +1,32 @@
+000000
+180d2f
+353658
+686b72
+8b97b6
+c5cddb
+ffffff
+5ee9e9
+2890dc
+1831a7
+053239
+005f41
+08b23b
+47f641
+e8ff75
+fbbe82
+de9751
+b66831
+8a4926
+461c14
+1e090d
+720d0d
+813704
+da2424
+ef6e10
+ecab11
+ece910
+f78d8d
+f94e6d
+c12458
+841252
+3d083b