Source code for bluesky_adaptive.queueserver

"""
Queueserver integration for running optimization through a Bluesky queueserver.

.. warning::

    This module is **experimental**. The API is not yet stable and may change
    in future releases without a deprecation period. It is not recommended for
    production use.

This module provides components for running optimization loops remotely through
a queueserver, rather than directly through a RunEngine.
"""

import asyncio
import logging
import threading
from collections.abc import Awaitable, Callable, Mapping, Sequence
from concurrent.futures import FIRST_COMPLETED, Future, wait
from dataclasses import dataclass, field
from typing import Any

try:
    import bluesky_queueserver_api.http
    import bluesky_queueserver_api.zmq
    from bluesky_queueserver_api import BPlan
except ImportError as e:
    raise ImportError(
        "The queueserver integration requires additional dependencies. "
        "Install them with: pip install bluesky-adaptive[queueserver]"
    ) from e

from .plans import default_acquire
from .protocols import ID_KEY, CanRegisterSuggestions, QueueserverOptimizationProblem, TrialFaultAware
from .utils import _maybe_checkpoint, _validate_outcomes, _validate_suggestions

logger = logging.getLogger(__name__)


DEFAULT_ACQUIRE_PLAN_NAME: str = default_acquire.__name__

_background_loop_lock = threading.Lock()
_background_loop: asyncio.AbstractEventLoop | None = None


def _get_background_loop() -> asyncio.AbstractEventLoop:
    """Return the event loop that runs optimizations started from synchronous code, starting it on first use."""
    global _background_loop
    with _background_loop_lock:
        if _background_loop is None:
            loop = asyncio.new_event_loop()
            threading.Thread(target=loop.run_forever, name="qserver-optimization-runner", daemon=True).start()
            _background_loop = loop
        return _background_loop


[docs] @dataclass(frozen=True) class OptimizationResult: """ The result of a completed or stopped optimization run. .. warning:: This class is part of the **experimental** queueserver integration. The API may change in future releases without a deprecation period. Parameters ---------- iterations_completed : int The number of suggest -> acquire -> ingest cycles that finished successfully. For a run stopped early via :meth:`QueueserverOptimizationRunner.stop`, this reflects however many iterations completed before the stop. num_points : int The number of points suggested per iteration. uids : tuple[str, ...] The queue item UIDs of the completed acquisitions, in order. Each item's ``history_get()`` entry lists the UIDs of the runs it opened in ``result["run_uids"]``. """ iterations_completed: int num_points: int uids: tuple[str, ...]
[docs] class QueueserverClient: """ Handles communication with a Bluesky queueserver. This class encapsulates queueserver communication: plan submission and environment checks. The queue must be running or have autostart enabled. .. warning:: This class is part of the **experimental** queueserver integration. The API may change in future releases without a deprecation period. Parameters ---------- re_manager_api : bluesky_queueserver_api.zmq.REManagerAPI | bluesky_queueserver_api.http.REManagerAPI Manager instance for communication with Bluesky Queueserver """ def __init__( self, re_manager_api: bluesky_queueserver_api.zmq.REManagerAPI | bluesky_queueserver_api.http.REManagerAPI, ): self._rm = re_manager_api
[docs] def check_environment(self) -> None: """ Verify that the queueserver environment is ready. Raises ------ RuntimeError If the queueserver environment is not open. """ status = self._rm.status() if status is None or not status.get("worker_environment_exists", False): raise RuntimeError("The queueserver environment is not open")
[docs] def submit_plan(self, plan: BPlan) -> str: """ Submit a plan to the queueserver queue. Parameters ---------- plan : BPlan The plan to submit. Returns ------- str The queue item UID assigned by the RE Manager. """ response = self._rm.item_add(plan) logger.debug(f"Submitted plan to queue. Response: {response}") return response["item"]["item_uid"]
@dataclass class _OptimizationState: """Internal mutable state for an optimization run.""" max_iterations: int = 1 num_points: int = 1 checkpoint_interval: int | None = None current_iteration: int = 0 current_suggestions: Sequence[Mapping] = field(default_factory=list) uids: list[str] = field(default_factory=list) def build_result(self) -> OptimizationResult: """Build an :class:`OptimizationResult` from the current state.""" return OptimizationResult( iterations_completed=len(self.uids), num_points=self.num_points, uids=tuple(self.uids), )
[docs] class QueueserverOptimizationRunner: """ Runs optimization loops through a Bluesky queueserver. This class coordinates the optimization workflow by getting suggestions from the optimizer, submitting acquisition plans to the queueserver, and ingesting the outcomes the evaluation function returns for each submitted plan. The loop is an asyncio coroutine. :meth:`run_async` and :meth:`submit_suggestions_async` run it on the caller's event loop; :meth:`run` and :meth:`submit_suggestions` start it on a shared background event loop and return a :class:`concurrent.futures.Future`. Queueserver requests, optimizer calls, and the evaluation function are synchronous, so they run in worker threads via :func:`asyncio.to_thread` and never block the event loop. A runner executes one loop at a time. .. warning:: This class is part of the **experimental** queueserver integration. The API may change in future releases without a deprecation period. Parameters ---------- optimization_problem : QueueserverOptimizationProblem The optimization problem to solve, containing the optimizer, actuators, sensors, and evaluation function. queueserver_client : QueueserverClient Client for communicating with the queueserver. """ def __init__( self, optimization_problem: QueueserverOptimizationProblem, queueserver_client: QueueserverClient, ): self._problem = optimization_problem self._client = queueserver_client self._plan_name = optimization_problem.acquisition_plan or DEFAULT_ACQUIRE_PLAN_NAME self._state: _OptimizationState | None = None self._continuous = True self._running = False @property def optimization_problem(self) -> QueueserverOptimizationProblem: """The optimization problem being solved.""" return self._problem @property def current_iteration(self) -> int: """The current iteration number (0 if not running).""" return self._state.current_iteration if self._state else 0
[docs] def run( self, iterations: int = 1, num_points: int = 1, checkpoint_interval: int | None = None ) -> Future[OptimizationResult]: """ Run the optimization loop. Validates the queueserver state and submits the first plan, then returns while the suggest -> acquire -> ingest cycle continues on a background event loop that calls the evaluation function with each submitted plan's queue item UID. :meth:`run_async` is the coroutine version. Parameters ---------- iterations : int Number of optimization iterations to run. num_points : int Number of points to suggest per iteration. checkpoint_interval : int | None The number of iterations between optimizer checkpoints. If None, checkpoints will not be saved. Optimizer must implement the :class:`bluesky_adaptive.protocols.Checkpointable` protocol. Returns ------- concurrent.futures.Future[OptimizationResult] A future that resolves to an :class:`OptimizationResult` when all iterations complete, or after :meth:`stop` once the in-flight evaluation returns. If the optimization loop raises an unhandled exception the future will hold that exception and re-raise it on ``.result()``. Cancelling the future cancels the loop like cancelling :meth:`run_async`. Raises ------ RuntimeError If the queueserver environment is not ready. ValueError If required devices or plans are not available, or the suggestions are invalid. """ return self._run_in_background(lambda: self._start_run(iterations, num_points, checkpoint_interval))
[docs] async def run_async( self, iterations: int = 1, num_points: int = 1, checkpoint_interval: int | None = None ) -> OptimizationResult: """ Run the optimization loop on the running event loop and return its result. Coroutine version of :meth:`run` for callers with a running event loop, such as Jupyter or a service; parameters and errors are the same, and loop errors are raised instead of held in a future. Cancelling it abandons the in-flight acquisition: its plan stays queued, its evaluation runs to completion in its worker thread, and its outcomes are not ingested. """ return await self._run(lambda: self._start_run(iterations, num_points, checkpoint_interval))
[docs] def submit_suggestions(self, suggestions: Sequence[Mapping]) -> Future[OptimizationResult]: """ Manually submit suggestions to the queue. This method returns once the plan is submitted; a background event loop calls the evaluation function with the plan's queue item UID and ingests the outcomes. :meth:`submit_suggestions_async` is the coroutine version. Parameters ---------- suggestions : Sequence[Mapping] Parameter combinations to evaluate. Can be: - Optimizer suggestions (with "_id" keys from suggest()) - Manual points (without "_id", requires CanRegisterSuggestions protocol) Returns ------- concurrent.futures.Future[OptimizationResult] A future that resolves to an :class:`OptimizationResult` when the acquisition completes. If an unhandled exception occurs the future will hold it and re-raise on ``.result()``. """ return self._run_in_background(lambda: self._start_submit(suggestions))
[docs] async def submit_suggestions_async(self, suggestions: Sequence[Mapping]) -> OptimizationResult: """ Manually submit suggestions to the queue on the running event loop and return the result. Coroutine version of :meth:`submit_suggestions` for callers with a running event loop, such as Jupyter or a service; parameters and errors are the same. """ return await self._run(lambda: self._start_submit(suggestions))
[docs] def stop(self) -> None: """ Stop the optimization loop gracefully. No further plans are submitted, and the in-flight evaluation is not interrupted. After that evaluation returns and is ingested, the future returned by :meth:`run` or :meth:`submit_suggestions` resolves (and :meth:`run_async` or :meth:`submit_suggestions_async` returns) with a partial :class:`OptimizationResult`, or with the evaluation's exception. Plans already in the queue are not removed. Evaluation functions should bound their waits so that a stop takes effect. This method only sets a flag, so it can be called from any thread. """ self._continuous = False logger.info("Optimization will stop after the in-flight acquisition is evaluated")
def _claim(self) -> None: """Mark the runner as running, raising if a loop is already running.""" if self._running: raise RuntimeError("Optimization loop is already running.") self._running = True def _run_in_background(self, start: Callable[[], Awaitable[str]]) -> Future[OptimizationResult]: """Run a loop on the background event loop, raising here any error before its first plan is submitted.""" started: Future[None] = Future() future = asyncio.run_coroutine_threadsafe(self._run(start, started), _get_background_loop()) pending: list[Future[Any]] = [started, future] try: wait(pending, return_when=FIRST_COMPLETED) except KeyboardInterrupt: future.cancel() # Interrupted before the first plan was submitted, so abandon the run. raise if not started.done(): future.result() # The run failed before submitting its first plan. return future async def _run(self, start: Callable[[], Awaitable[str]], started: Future[None] | None = None) -> OptimizationResult: """Claim the runner, submit the first plan with ``start``, and run the loop, releasing the runner at the end.""" self._claim() try: item_uid = await start() if started is not None: started.set_result(None) return await self._run_loop(item_uid) finally: self._running = False async def _start_run(self, iterations: int, num_points: int, checkpoint_interval: int | None) -> str: """Check the environment and submit the first plan of a run, returning its queue item UID.""" await asyncio.to_thread(self._client.check_environment) self._state = _OptimizationState( max_iterations=iterations, num_points=num_points, checkpoint_interval=checkpoint_interval ) self._continuous = True suggestions = await asyncio.to_thread(self._problem.optimizer.suggest, num_points) plan = self._build_plan(suggestions) logger.info(f"Submitting iteration {self._state.current_iteration}/{self._state.max_iterations}") return await asyncio.to_thread(self._client.submit_plan, plan) async def _start_submit(self, suggestions: Sequence[Mapping]) -> str: """Check the environment and submit a plan for manual suggestions, returning its queue item UID.""" await asyncio.to_thread(self._client.check_environment) self._state = _OptimizationState(max_iterations=1, num_points=len(suggestions)) self._continuous = False # Ensure all suggestions have an ID_KEY or register them with the optimizer if not isinstance(self.optimization_problem.optimizer, CanRegisterSuggestions) and any( ID_KEY not in suggestion for suggestion in suggestions ): raise ValueError( f"All suggestions must contain an '{ID_KEY}' key to later match with the outcomes or your optimizer must " "implement the `bluesky_adaptive.protocols.CanRegisterSuggestions` protocol. Please review your optimizer " f"implementation. Got suggestions: {suggestions}" ) elif isinstance(self.optimization_problem.optimizer, CanRegisterSuggestions): suggestions = await asyncio.to_thread(self.optimization_problem.optimizer.register_suggestions, suggestions) plan = self._build_plan(suggestions) logger.info("Submitting manually specified suggestion(s)") return await asyncio.to_thread(self._client.submit_plan, plan) def _try_register_failures(self, suggestions: Sequence[Mapping]) -> None: """Notify a TrialFaultAware optimizer of failed suggestions, if supported.""" if suggestions and isinstance(self._problem.optimizer, TrialFaultAware): try: self._problem.optimizer.register_failures(suggestions) except Exception: # Logged, not raised: the original error must still propagate to the caller. logger.exception("Failed to register trial failures with the optimizer") def _build_plan(self, suggestions: Sequence[Mapping]) -> BPlan: """Build the plan to submit and update current state.""" if self._state is None: raise RuntimeError("_build_plan() called before run() or submit_suggestions()") _validate_suggestions(suggestions) self._state.current_iteration += 1 self._state.current_suggestions = suggestions # Build metadata md: dict[str, Any] = {"blop_suggestions": self._state.current_suggestions} return BPlan( self._plan_name, self._state.current_suggestions, list(self._problem.actuators), list(self._problem.sensors), md=md, **(self._problem.acquisition_plan_kwargs or {}), ) async def _run_loop(self, item_uid: str) -> OptimizationResult: """Evaluate, ingest, and resubmit until done, registering the in-flight suggestions as failed on error.""" state = self._state if state is None: raise RuntimeError("_run_loop() called before run() or submit_suggestions()") try: next_item_uid: str | None = item_uid while next_item_uid is not None: next_item_uid = await self._process_acquisition(next_item_uid) except Exception: logger.exception("Unhandled exception in the optimization loop. Optimization has been stopped.") # Register failures before re-raising so a caller that observes the failure sees the trials failed. await asyncio.to_thread(self._try_register_failures, state.current_suggestions) raise return state.build_result() async def _process_acquisition(self, item_uid: str) -> str | None: """ Evaluate and ingest one acquisition. Returns the next submitted queue item UID, or ``None`` when the loop is done. """ if self._state is None: raise RuntimeError("_process_acquisition called before run()") suggestions = self._state.current_suggestions # Evaluate the results; the evaluation function waits for its data. logger.info(f"Evaluating queue item {item_uid}") outcomes = await asyncio.to_thread(self._problem.evaluation_function, uid=item_uid, suggestions=suggestions) _validate_outcomes(outcomes, suggestions) logger.info(f"Evaluated {len(outcomes)} outcomes") # Ingest into optimizer await asyncio.to_thread(self._problem.optimizer.ingest, outcomes) self._state.uids.append(item_uid) # Ingested suggestions are no longer in flight. self._state.current_suggestions = [] # Checkpoint optimizer, if applicable await asyncio.to_thread( _maybe_checkpoint, self._problem.optimizer, self._state.checkpoint_interval, iteration=self._state.current_iteration, ) # Check if done if not self._continuous or self._state.current_iteration >= self._state.max_iterations: logger.info(f"Optimization complete after {self._state.current_iteration} iterations") return None # Continue: get next suggestions and submit suggestions = await asyncio.to_thread(self._problem.optimizer.suggest, self._state.num_points) plan = self._build_plan(suggestions) logger.info(f"Submitting iteration {self._state.current_iteration}/{self._state.max_iterations}") return await asyncio.to_thread(self._client.submit_plan, plan)