Distributed rendering with extension

This test demonstrates distributed rendering with user-defined component.

%load_ext autoreload
%autoreload 2

import os
import imageio
import pandas as pd
import numpy as np
import multiprocessing as mp
%matplotlib inline
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import lmfunctest as ft
import lmscene
import lightmetrica as lm

os.getpid()

794

%load_ext lightmetrica_jupyter

Worker process

To create an process on Jupyter notebook in Windows, we need to separate the function to be processed in a different file and add the invocation of the process must be enclosed by if __name__ == '__main__' clause.

%%writefile _lm_renderer_ao.py
import lightmetrica as lm
import pickle
import numpy as np

@lm.pylm_component('renderer::ao')
class Renderer_AO(lm.Renderer):
    """Simple ambient occlusion renderer"""

    def construct(self, prop):
        self.film = lm.Film.castFrom(lm.comp.get(prop['output']))
        if self.film is None:
            return False
        self.spp = prop['spp']
        return True

    def save(self):
        return pickle.dumps((self.film.loc(), self.spp))

    def load(self, s):
        loc, self.spp = pickle.loads(s)
        self.film = lm.Film.castFrom(lm.comp.get(loc))

    def render(self, scene):
        w = self.film.size().w
        h = self.film.size().h
        rng = lm.Rng(42)
        def process(index, threadid):
            x = index % w
            y = int(index / w)
            rp = np.array([(x+.5)/w, (y+.5)/h])
            ray = scene.primaryRay(rp, self.film.aspectRatio())
            hit = scene.intersect(ray)
            if hit is None:
                return
            V = 0
            for i in range(self.spp):
                n, u, v = hit.geom.orthonormalBasis(-ray.d)
                d = lm.math.sampleCosineWeighted(rng)
                r = lm.Ray(hit.geom.p, np.dot(d, [u,v,n]))
                if scene.intersect(r, lm.Eps, .2) is None:
                    V += 1
            V /= self.spp
            self.film.setPixel(x, y, np.full(3, V))
        lm.parallel.foreach(w*h, process)

Writing _lm_renderer_ao.py

%%writefile _run_worker_process.py
import os
import uuid
import traceback
import lightmetrica as lm
import _lm_renderer_ao
def run_worker_process():
    try:
        lm.init('user::default', {})
        lm.info()
        lm.log.setSeverity(1000)
        lm.log.log(lm.log.LogLevel.Err, lm.log.LogLevel.Info, '', 0, 'pid={}'.format(os.getpid()))
        lm.dist.worker.init('dist::worker::default', {
            'name': uuid.uuid4().hex,
            'address': 'localhost',
            'port': 5000,
            'numThreads': 1
        })
        lm.dist.worker.run()
        lm.dist.shutdown()
        lm.shutdown()
    except Exception:
        tr = traceback.print_exc()
        lm.log.log(lm.log.LogLevel.Err, lm.log.LogLevel.Info, '', 0, str(tr))

Overwriting _run_worker_process.py

from _run_worker_process import *
if __name__ == '__main__':
    pool = mp.Pool(4, run_worker_process)

Master process

import _lm_renderer_ao

lm.init()
lm.log.init('logger::jupyter', {})
lm.progress.init('progress::jupyter', {})
lm.dist.init('dist::master::default', {
    'port': 5000
})
lm.dist.printWorkerInfo()

[I|0.000|173@dist  ] Listening [port='5000']

lmscene.load(ft.env.scene_path, 'fireplace_room')
lm.build('accel::sahbvh', {})
lm.asset('film_output', 'film::bitmap', {'w': 1920, 'h': 1080})
lm.renderer('renderer::ao', {
    'output': lm.asset('film_output'),
    'spp': 3
})

[I|0.021|48@assets ] Loading asset [name='camera_main']
[I|0.022|48@assets ] Loading asset [name='model_obj']
[I|0.022|29@objload]   Loading OBJ file [path='fireplace_room.obj']
[I|0.022|169@objloa]   Loading MTL file [path='fireplace_room.mtl']
[I|0.023|44@texture]   Loading texture [path='wood.ppm']
[I|0.134|91@dist   ]   Accepted [name='rep', addr='tcp://0.0.0.0:5003']
[I|0.135|44@texture]   Loading texture [path='leaf.ppm']
[I|0.138|44@texture]   Loading texture [path='picture8.ppm']
[I|0.178|263@dist  ]   Connected worker [name='07d2557f31114603b4e4b46bb6cfd839']
[I|0.179|44@texture]   Loading texture [path='wood5.ppm']
[I|0.935|263@dist  ]   Connected worker [name='820b386cb899418b88b0d4a0f51ef05d']
[I|0.935|246@scene ] Building acceleration structure [name='accel::sahbvh'][I|0.935|91@dist   ] Accepted [name='rep', addr='tcp://0.0.0.0:5003']
[I|0.936|263@dist  ] Connected worker [name='c4427eda18ef472184e704bd7bc46a2e']
[I|0.936|263@dist  ] Connected worker [name='73f9da987bfc4c22a7cb3f56922edd29']
[I|0.936|91@dist   ] Accepted [name='rep', addr='tcp://0.0.0.0:5003']
[I|0.936|91@dist   ] Accepted [name='rep', addr='tcp://0.0.0.0:5003']

[I|0.963|131@accel_]   Flattening scene
[I|1.091|261@accel_]   Building
[I|4.322|48@assets ] Loading asset [name='film_output']

lm.dist.allowWorkerConnection(False)
lm.dist.sync()
lm.render()
lm.dist.gatherFilm(lm.asset('film_output'))
lm.dist.allowWorkerConnection(True)

[I|4.848|179@user  ] Saving state to stream
[I|7.560|151@user  ] Starting render [name='renderer::ao']

img = np.copy(lm.buffer(lm.asset('film_output')))
f = plt.figure(figsize=(15,15))
ax = f.add_subplot(111)
ax.imshow(np.clip(np.power(img,1/2.2),0,1), origin='lower')
plt.show()

# Termination of the worker process is necessary for Windows
# because fork() is not supported in Windows.
# cf. https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
pool.terminate()
pool.join()

[I|122.901|103@dist  ] Disconnected [name='rep', addr='tcp://0.0.0.0:5003']
[I|122.903|103@dist  ] Disconnected [name='rep', addr='tcp://0.0.0.0:5003']
[I|122.907|103@dist  ] Disconnected [name='rep', addr='tcp://0.0.0.0:5003']
[I|122.910|103@dist  ] Disconnected [name='rep', addr='tcp://0.0.0.0:5003']