Skip to content

service restart has failed #51

Description

@jackeyPeng

再次启动报错:
C:\Users\jacky\soft\OpenOPC>uv run opc ui
15:31:50 | INFO | opc.engine - Initializing OPC Engine...
15:31:50 | INFO | opc.layer5_memory.skill_library - Loaded 9 skills
15:31:50 | INFO | opc.layer5_memory.skill_library - Loaded 9 skills
15:31:50 | INFO | opc.layer3_agent.adapters.registry - External agent claude_code: not found
15:31:50 | INFO | opc.layer3_agent.adapters.registry - External agent cursor: not found
15:31:50 | INFO | opc.layer3_agent.adapters.registry - External agent codex: not found
15:31:50 | INFO | opc.layer3_agent.adapters.registry - External agent opencode: not found
15:31:50 | INFO | opc.layer5_memory.skill_library - Loaded 9 skills
15:31:50 | INFO | opc.layer2_organization.org_engine - OrgEngine initialized with 11 roles
15:31:50 | ERROR | opc.engine - Startup reconcile failed for project default; aborting initialization
Traceback (most recent call last):

File "", line 198, in _run_module_as_main
File "", line 88, in _run_code

File "C:\Users\jacky\soft\OpenOPC.venv\Scripts\opc.exe_main_.py", line 10, in

File "C:\Users\jacky\soft\OpenOPC\opc\cli\app.py", line 8597, in main
app()
└ <typer.main.Typer object at 0x00000262438596A0>

File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer\main.py", line 1137, in call
return get_command(self)(*args, **kwargs)
│ │ │ └ {}
│ │ └ ()
│ └ <typer.main.Typer object at 0x00000262438596A0>
└ <function get_command at 0x00000262424CDB20>
File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer_click\core.py", line 807, in call
return self.main(*args, **kwargs)
│ │ │ └ {}
│ │ └ ()
│ └ <function TyperGroup.main at 0x00000262424CD580>

File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer\core.py", line 1203, in main
return _main(
└ <function _main at 0x00000262424BF6A0>
File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer\core.py", line 189, in _main
rv = self.invoke(ctx)
│ │ └ <typer._click.core.Context object at 0x0000026240240710>
│ └ <function TyperGroup.invoke at 0x00000262424CD120>

File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer\core.py", line 1125, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
│ │ │ │ └ <typer._click.core.Context object at 0x0000026243AC2F90>
│ │ │ └ <function Command.invoke at 0x00000262423F2C00>
│ │ └
│ └ <typer._click.core.Context object at 0x0000026243AC2F90>
└ <function TyperGroup.invoke.._process_result at 0x0000026243AA7740>
File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer_click\core.py", line 746, in invoke
return ctx.invoke(self.callback, **ctx.params)
│ │ │ │ │ └ {'port': 8765, 'host': '0.0.0.0', 'project': None, 'rebuild': False}
│ │ │ │ └ <typer._click.core.Context object at 0x0000026243AC2F90>
│ │ │ └ <function register_cli..ui at 0x0000026243A711C0>
│ │ └
│ └ <function Context.invoke at 0x00000262423F1F80>
└ <typer._click.core.Context object at 0x0000026243AC2F90>
File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer_click\core.py", line 489, in invoke
return callback(*args, **kwargs)
│ │ └ {'port': 8765, 'host': '0.0.0.0', 'project': None, 'rebuild': False}
│ └ ()
└ <function register_cli..ui at 0x0000026243A711C0>
File "C:\Users\jacky\soft\OpenOPC.venv\Lib\site-packages\typer\main.py", line 1524, in wrapper
return callback(**use_params)
│ └ {'port': 8765, 'host': '0.0.0.0', 'project': None, 'rebuild': False}
└ <function register_cli..ui at 0x0000026243A6B880>

File "C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui_init_.py", line 172, in ui
run_server(host=host, port=port, config=config, project_id=project)
│ │ │ │ └ None
│ │ │ └ OPCConfig(system=SystemConfig(opc_home='', default_channel='cli', log_level='INFO', max_agent_iterations=50, context_compress...
│ │ └ 8765
│ └ '0.0.0.0'
└ <function run_server at 0x000002624C7EA5C0>

File "C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\server.py", line 302, in run_server
asyncio.run(_start())
│ │ └ <function run_server.._start at 0x0000026243ECE3E0>
│ └ <function run at 0x0000026241CDB560>
└ <module 'asyncio' from 'C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\__ini...

File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py", line 195, in run
return runner.run(main)
│ │ └ <coroutine object run_server.._start at 0x000002624C77DCF0>
│ └ <function Runner.run at 0x0000026241DA9BC0>
└ <asyncio.runners.Runner object at 0x0000026245243BC0>
File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py", line 118, in run
return self._loop.run_until_complete(task)
│ │ │ └ <Task pending name='Task-1' coro=<run_server.._start() running at C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\s...
│ │ └ <function BaseEventLoop.run_until_complete at 0x0000026241DA77E0>
│ └
└ <asyncio.runners.Runner object at 0x0000026245243BC0>
File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\base_events.py", line 678, in run_until_complete
self.run_forever()
│ └ <function ProactorEventLoop.run_forever at 0x000002624226B7E0>

File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\windows_events.py", line 322, in run_forever
super().run_forever()
File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\base_events.py", line 645, in run_forever
self._run_once()
│ └ <function BaseEventLoop._run_once at 0x0000026241DA9580>

File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\base_events.py", line 1999, in _run_once
handle._run()
│ └ <function Handle._run at 0x0000026241CC7880>
└ <Handle Task.task_wakeup()>
File "C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\events.py", line 88, in _run
self._context.run(self._callback, *self._args)
│ │ │ │ │ └ <member '_args' of 'Handle' objects>
│ │ │ │ └ <Handle Task.task_wakeup()>
│ │ │ └ <member '_callback' of 'Handle' objects>
│ │ └ <Handle Task.task_wakeup()>
│ └ <member '_context' of 'Handle' objects>
└ <Handle Task.task_wakeup()>

File "C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\server.py", line 286, in _start
app = await create_app(config=config, project_id=project_id)
│ │ └ None
│ └ OPCConfig(system=SystemConfig(opc_home='', default_channel='cli', log_level='INFO', max_agent_iterations=50, context_compress...
└ <function create_app at 0x000002624C7D8A40>

File "C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\server.py", line 130, in create_app
await engine.initialize()
│ └ <function OPCEngine.initialize at 0x000002624C676200>
└ <opc.engine.OPCEngine object at 0x000002624C4FBE60>

File "C:\Users\jacky\soft\OpenOPC\opc\engine.py", line 973, in initialize
reconciled = await self._reconcile_interrupted_project_tasks()
│ └ <function OPCEngine._reconcile_interrupted_project_tasks at 0x000002624C694040>
└ <opc.engine.OPCEngine object at 0x000002624C4FBE60>

File "C:\Users\jacky\soft\OpenOPC\opc\engine.py", line 9092, in _reconcile_interrupted_project_tasks
await settle_claims(
└ <bound method OPCStore.settle_stale_delegation_run_claims_for_controller of <opc.database.store.OPCStore object at 0x00000262...

File "C:\Users\jacky\soft\OpenOPC\opc\database\store.py", line 19438, in settle_stale_delegation_run_claims_for_controller
raise RuntimeError(

RuntimeError: controller takeover found a mixed linked Task/WorkItem attempt envelope for 'cd6756ad-6ad0-4472-a711-612c291fcdac'
╭───────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────╮
│ C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui_init_.py:172 in ui │
│ │
│ 169 │ │ │
│ 170 │ │ from opc.plugins.office_ui.server import run_server │
│ 171 │ │ │
│ ❱ 172 │ │ run_server(host=host, port=port, config=config, project_id=project) │
│ 173 │
│ │
│ C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\server.py:302 in run_server │
│ │
│ 299 │ │ │ await runner.cleanup() │
│ 300 │ │
│ 301 │ try: │
│ ❱ 302 │ │ asyncio.run(_start()) │
│ 303 │ except KeyboardInterrupt: │
│ 304 │ │ terminal_status("Shutting down Office UI", kind="warning") │
│ 305 │ except SystemExit as exc: │
│ │
│ C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py:195 in run │
│ │
│ 192 │ │ │ "asyncio.run() cannot be called from a running event loop") │
│ 193 │ │
│ 194 │ with Runner(debug=debug, loop_factory=loop_factory) as runner: │
│ ❱ 195 │ │ return runner.run(main) │
│ 196 │
│ 197 │
│ 198 def _cancel_all_tasks(loop): │
│ │
│ C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\runners.py:118 in run │
│ │
│ 115 │ │ │
│ 116 │ │ self._interrupt_count = 0 │
│ 117 │ │ try: │
│ ❱ 118 │ │ │ return self._loop.run_until_complete(task) │
│ 119 │ │ except exceptions.CancelledError: │
│ 120 │ │ │ if self._interrupt_count > 0: │
│ 121 │ │ │ │ uncancel = getattr(task, "uncancel", None) │
│ │
│ C:\Users\jacky\AppData\Roaming\uv\python\cpython-3.12-windows-x86_64-none\Lib\asyncio\base_events.py:691 in │
│ run_until_complete │
│ │
│ 688 │ │ if not future.done(): │
│ 689 │ │ │ raise RuntimeError('Event loop stopped before Future completed.') │
│ 690 │ │ │
│ ❱ 691 │ │ return future.result() │
│ 692 │ │
│ 693 │ def stop(self): │
│ 694 │ │ """Stop running the event loop. │
│ │
│ C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\server.py:286 in _start │
│ │
│ 283 │ """Create and run the office-UI server (blocking).""" │
│ 284 │ │
│ 285 │ async def _start() -> None: │
│ ❱ 286 │ │ app = await create_app(config=config, project_id=project_id) │
│ 287 │ │ runner = aiohttp.web.AppRunner(app) │
│ 288 │ │ await runner.setup() │
│ 289 │ │ site = aiohttp.web.TCPSite(runner, host, port) │
│ │
│ C:\Users\jacky\soft\OpenOPC\opc\plugins\office_ui\server.py:130 in create_app │
│ │
│ 127 │ event_adapter = EventAdapter() │
│ 128 │ │
│ 129 │ # ── Initialize engine (this starts all OPC layers) ──────────────── │
│ ❱ 130 │ await engine.initialize() │
│ 131 │ │
│ 132 │ # ── WSHandler ───────────────────────────────────────────────────── │
│ 133 │ ws_handler = WSHandler(engine, agent_store, chat_store, event_adapter) │
│ │
│ C:\Users\jacky\soft\OpenOPC\opc\engine.py:973 in initialize │
│ │
│ 970 │ │ reconciled = 0 │
│ 971 │ │ if self._run_startup_reconcile: │
│ 972 │ │ │ try: │
│ ❱ 973 │ │ │ │ reconciled = await self._reconcile_interrupted_project_tasks() │
│ 974 │ │ │ except InvalidPhaseTransition: │
│ 975 │ │ │ │ logger.opt(exception=True).error( │
│ 976 │ │ │ │ │ "Startup reconcile hit an invalid work-item phase transition for project {}; aborting │
│ initialization", │
│ │
│ C:\Users\jacky\soft\OpenOPC\opc\engine.py:9092 in _reconcile_interrupted_project_tasks │
│ │
│ 9089 │ │ │ │ │ │ │ None, │
│ 9090 │ │ │ │ │ │ ) │
│ 9091 │ │ │ │ │ │ if callable(settle_claims): │
│ ❱ 9092 │ │ │ │ │ │ │ await settle_claims( │
│ 9093 │ │ │ │ │ │ │ │ run_id, │
│ 9094 │ │ │ │ │ │ │ │ project_id=project_id, │
│ 9095 │ │ │ │ │ │ │ │ root_session_id=parent_session_id, │
│ │
│ C:\Users\jacky\soft\OpenOPC\opc\database\store.py:19438 in settle_stale_delegation_run_claims_for_controller │
│ │
│ 19435 │ │ │ │ │ │ │ or linked_task_generation │
│ 19436 │ │ │ │ │ │ │ != prior_work_item_generation │
│ 19437 │ │ │ │ │ │ ): │
│ ❱ 19438 │ │ │ │ │ │ │ raise RuntimeError( │
│ 19439 │ │ │ │ │ │ │ │ "controller takeover found a mixed linked " │
│ 19440 │ │ │ │ │ │ │ │ f"Task/WorkItem attempt envelope for " │
│ 19441 │ │ │ │ │ │ │ │ f"{clean_work_item_id!r}" │
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
RuntimeError: controller takeover found a mixed linked Task/WorkItem attempt envelope for

问题所在
报错不是代码 bug,而是上一次运行留下的残留数据,把启动时的"中断清理"(startup reconcile)卡死了。

具体链路:
之前那次运行其实已经失败了——数据库里存的角色 chair-man 的工作记忆里写着:

LLM stream failed: litellm.BadRequestError:
LLM Provider NOT provided ... model=deepseek-v4-pro-0813
也就是那一次的 LLM 配置没设 provider,导致运行直接失败。

那次失败的运行留下了一个没释放干净的 work item(cd6756ad-...)。它 phase=failed、attempt_settled=true,已经终态了,但 claimed_by_role_runtime_session_id / claimed_by_seat_id 这两个"认领"字段没被清空。

a/C:\Users\jacky\soft\OpenOPC\opc\database\store.py → b/C:\Users\jacky\soft\OpenOPC\opc\database\store.py
@@ -19417,6 +19417,26 @@
)
or ""
).strip()

  •                    # A linked Task may legitimately still be "parked":
    
  •                    # no controller owner token, generation 0, and no
    
  •                    # claimed attempt yet — the same envelope the
    
  •                    # company-work-item gate flow accepts as
    
  •                    # `task_parked_envelope`.  The preceding controller
    
  •                    # created it but never stamped it before it died.
    
  •                    # Adopt such a Task instead of rejecting the
    
  •                    # takeover, so startup reconcile does not get stuck
    
  •                    # on a benign mixed envelope.
    
  •                    task_parked_envelope = bool(
    
  •                        not linked_task_owner
    
  •                        and linked_task_generation == 0
    
  •                        and linked_task_attempt == 0
    
  •                    )
    
  •                    task_envelope_matches = bool(
    
  •                        linked_task_attempt == attempt_seq
    
  •                        and linked_task_owner == prior_work_item_owner
    
  •                        and linked_task_generation
    
  •                        == prior_work_item_generation
    
  •                    )
                       if (
                           not linked_task_id
                           or linked_task_project_id != clean_project_id
    

@@ -19428,18 +19448,22 @@
).strip()
!= clean_run_id
or attempt_seq <= 0

  •                        or linked_task_attempt != attempt_seq
                           or not prior_work_item_owner
    
  •                        or linked_task_owner != prior_work_item_owner
                           or prior_work_item_generation <= 0
    
  •                        or linked_task_generation
    
  •                        != prior_work_item_generation
    
  •                        or not (
    
  •                            task_envelope_matches
    
  •                            or task_parked_envelope
    
  •                        )
                       ):
                           raise RuntimeError(
                               "controller takeover found a mixed linked "
                               f"Task/WorkItem attempt envelope for "
                               f"{clean_work_item_id!r}"
                           )
    
  •                    if task_parked_envelope:
    
  •                        linked_task_metadata[
    
  •                            "claimed_work_item_attempt_seq"
    
  •                        ] = attempt_seq
                       linked_task_metadata[
                           "company_run_controller_owner_token"
                       ] = clean_owner_token
    

重启时引擎发现这条"陈旧认领",去接管它,核对它和关联 Task 是否是一对匹配的"凭据"(company_run_controller_owner_token / lease_generation / claimed_work_item_attempt_seq 必须一一对应)。结果那条 Task 处于"停靠"状态(三个字段全是空的),和 work item 对不上 → RuntimeError: mixed linked Task/WorkItem attempt envelope → 直接中止启动。

  1. 那次运行之所以失败,是 LLM 配置问题:LLM Provider NOT provided,用的模型名是 deepseek-v4-pro-0813(跟你现在这个 Hermes 会话同一个模型)。如果 OpenOPC 里你还没配好这个模型的 provider,下次跑公司运行还会失败——跟这次的启动崩溃是两码事,但值得顺手查一下配置。
  2. 有复发可能:settle_stale_delegation_run_claims_for_controller 这段接管校验(store.py:19420)没有考虑 Task 处于"停靠"(无凭据)状态的合法情况,代码里别处(gate 审批流程)是显式允许这种状态的。如果哪次运行又是在 work item 被认领但 Task 还没盖章时被强关,还可能再触发同样的崩溃。

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions