Remote Execution Example¶
Example Code¶
The following example runs an app on another rig through its XML-RPC server, then transfers the data of both machines with the watchdog. See Running on other rigs for how each step works.
import logging
from pathlib import Path
from _mocks import (
LIB_CONFIG,
RIG,
SUGGESTION,
MockTask,
create_fake_rig,
create_fake_subjects,
)
from pydantic_settings import CliApp
from clabe import resource_monitor
from clabe.apps import AindBehaviorServicesBonsaiApp
from clabe.data_transfer.aind_watchdog import WatchdogDataTransferService, WatchdogSettings
from clabe.launcher import Launcher, LauncherCliArgs, experiment
from clabe.session import SessionBuilder
from clabe.stores import LocalFileStore
from clabe.xml_rpc import RemoteRepository, XmlRpcClient, XmlRpcClientSettings
logger = logging.getLogger(__name__)
# Data root on the remote machine, and the network share other machines reach it through
REMOTE_DATA_DIRECTORY = r"D:\data"
REMOTE_DATA_SHARE = r"\\rig-b\data"
@experiment()
async def client_experiment(launcher: Launcher) -> None:
"""Demo experiment showcasing CLABE functionality."""
session = SessionBuilder(launcher, experimenter_validator=lambda name: name).build()
store = LocalFileStore(LIB_CONFIG).scoped(subject=session.subject)
rig = store.resolve(RIG)
launcher.register_session(session, rig.data_directory)
trainer_state = store.resolve(SUGGESTION)
task = MockTask.model_validate_json(trainer_state.stage.task.model_dump_json())
resource_monitor.ResourceMonitor(
constrains=[
resource_monitor.available_storage_constraint_factory_from_rig(rig, 1e9),
]
).run()
remote = RemoteRepository(
XmlRpcClient(settings=XmlRpcClientSettings(server_url="http://localhost:8000", token="42")),
local_path=".",
data_roots={REMOTE_DATA_DIRECTORY: REMOTE_DATA_SHARE},
)
remote.ensure_running()
remote.verify(allow_dirty=True)
# The remote machine saves its data under its own session directory, named like the launcher's
remote.register_session(session, REMOTE_DATA_DIRECTORY)
# The app is built exactly as for a local run. Running it on the remote executor copies the
# rig, session and task to the remote machine and maps paths in this repository to the remote one.
bonsai_root = Path(r"C:\git\AllenNeuralDynamics\Aind.Behavior.VrForaging")
bonsai_app = AindBehaviorServicesBonsaiApp(
workflow=bonsai_root / "src/test_deserialization.bonsai",
executable=bonsai_root / "bonsai/bonsai.exe",
temp_directory=launcher.temp_dir,
rig=rig,
session=session,
task=task,
)
bonsai_app_result = await bonsai_app.run_async(executor=remote)
print(bonsai_app_result)
# One watchdog manifest covering both machines: the local session directory, and the remote one
# through its network share. Requires the WATCHDOG_EXE and WATCHDOG_CONFIG environment variables.
watchdog = WatchdogDataTransferService(
source=[launcher.session_directory, remote.session_directory_unc],
settings=WatchdogSettings(destination=r"\\allen\aind\scratch\data-uploads", project_name="clabe"),
session=session,
)
watchdog.transfer()
def main():
create_fake_subjects()
create_fake_rig()
behavior_cli_args = CliApp.run(
LauncherCliArgs,
cli_args=[
"--debug-mode",
"--allow-dirty",
"--skip-hardware-validation",
],
)
launcher = Launcher(settings=behavior_cli_args)
launcher.run_experiment(client_experiment)
if __name__ == "__main__":
main()