Allow to live update attributes without server restart

api/attributes/attributes.py

@@ -5,14 +5,13 @@
 
 from ayon_server.api.dependencies import AttributeName, CurrentUser
 from ayon_server.api.responses import EmptyResponse
-from ayon_server.api.system import require_server_restart
 from ayon_server.attributes.models import (
     AttributeModel,
     AttributePatchModel,
     AttributePutModel,
 )
 from ayon_server.attributes.validate_attribute_data import validate_attribute_data
-from ayon_server.entities import ProjectEntity
+from ayon_server.events import EventStream
 from ayon_server.exceptions import ForbiddenException, NotFoundException
 from ayon_server.lib.postgres import Postgres
 from ayon_server.types import OPModel
@@ -39,8 +38,7 @@ class SetAttributeListModel(GetAttributeListModel):
 async def save_attribute(attribute: AttributeModel) -> None:
     """Save attribute configuration to the database.
 
-    Additionally performs validation of the attribute data and updates
-    the enumerator in the running instance.
+    Additionally performs validation of the attribute data.
     """
     query = """
     INSERT INTO attributes
@@ -60,25 +58,6 @@ async def save_attribute(attribute: AttributeModel) -> None:
         attribute.data.dict(exclude_none=True),
     )
 
-    # TODO: The following code does not support horizontal scaling!!
-    # Notify other instances instead and reload the attribute library
-
-    if (enum := attribute.data.enum) is not None:
-        for name, field in ProjectEntity.model.attrib_model.__fields__.items():
-            if name != attribute.name:
-                continue
-
-            field_enum = field.field_info.extra.get("enum")
-            if field_enum is None:
-                continue
-            field_enum.clear()
-            field_enum.extend(enum)
-
-        for name, field in ProjectEntity.model.attrib_model.__fields__.items():
-            if name != attribute.name:
-                continue
-            field_enum = field.field_info.extra.get("enum")
-
 
 async def list_raw_attributes() -> list[dict[str, Any]]:
     """Return a list of attributes as they are stored in the DB"""
@@ -153,7 +132,7 @@ async def set_attribute_list(
     for attr in new_attributes:
         await save_attribute(attr)
 
-    await require_server_restart()
+    await EventStream.dispatch("server.attributes_updated")
     return EmptyResponse()
 
 
@@ -180,9 +159,7 @@ async def set_attribute_config(
         raise ForbiddenException("Only administrators are allowed to modify attributes")
     attribute = AttributeModel(name=attribute_name, **payload.dict())
     await save_attribute(attribute)
-    await require_server_restart(
-        None, "Restart the server to apply the attribute changes."
-    )
+    await EventStream.dispatch("server.attributes_updated")
     return EmptyResponse()
 
 
@@ -197,8 +174,6 @@ async def patch_attribute_config(
     patch_payload = payload.dict(exclude_unset=True)
     patch_data = patch_payload.pop("data", {})
 
-    requires_restart = False
-
     if "scope" in patch_payload or any(
         k in patch_data
         for k in (
@@ -216,8 +191,6 @@ async def patch_attribute_config(
             "inherit",
         )
     ):
-        requires_restart = True
-
         if not user.is_admin:
             raise ForbiddenException(
                 "Only administrators are allowed to modify attribute configuration"
@@ -236,10 +209,7 @@ async def patch_attribute_config(
 
     await save_attribute(attribute)
 
-    if requires_restart:
-        await require_server_restart(
-            None, "Restart the server to apply the attribute changes."
-        )
+    await EventStream.dispatch("server.attributes_updated")
     return EmptyResponse()
 
 
@@ -251,7 +221,5 @@ async def delete_attribute(
         raise ForbiddenException("Only administrators are allowed to delete attributes")
 
     await remove_attribute(attribute_name)
-    await require_server_restart(
-        None, "Restart the server to apply the attribute changes."
-    )
+    await EventStream.dispatch("server.attributes_updated")
     return EmptyResponse()

ayon_server/api/lifespan.py

@@ -8,6 +8,7 @@
 import semver
 
 from ayon_server.addons import AddonLibrary
+from ayon_server.entities.core import attribute_library
 from ayon_server.api.frontend import init_frontend
 from ayon_server.api.messaging import messaging
 from ayon_server.api.static import addon_static_router
@@ -110,6 +111,7 @@ async def lifespan(app: "FastAPI"):
         f.write(str(os.getpid()))
 
     await ayon_init()
+    EventStream.subscribe("server.attributes_updated", attribute_library.reload_handler, True)
     await load_access_groups()
     await CloudUtils.clear_cloud_info_cache()
 

ayon_server/entities/core/attrib.py

@@ -2,7 +2,7 @@
 import collections
 import functools
 import threading
-from typing import Any
+from typing import Any, Callable
 
 from ayon_server.lib.postgres import Postgres
 from ayon_server.logging import logger
@@ -29,6 +29,8 @@ def __init__(self) -> None:
         # in the same format as the attributes endpoint
         self.info_data: list[Any] = []
 
+        self._invalidation_callbacks: list[Callable[[], None]] = []
+
         # We need to load attribute data in a separate thread
         # with a separate event loop, because the main event loop
         # is already running and we cannot run another one
@@ -146,5 +148,58 @@ def by_name_scoped(self, entity_type: str, name: str) -> dict[str, Any]:
                 return attr
         raise KeyError(f"Attribute {name} not found for entity type {entity_type}")
 
+    def register_invalidation_callback(self, callback: Callable[[], None]) -> None:
+        """Register a callback to be called when attributes are reloaded."""
+        self._invalidation_callbacks.append(callback)
+
+    async def reload(self) -> None:
+        """Reload attributes from the database and invalidate all cached models.
+
+        Fetches fresh data from the database, updates the in-memory attribute
+        lists in-place (preserving list object identity so ModelSet.attributes
+        references remain valid), and invalidates all cached Pydantic models
+        so they are regenerated on next access.
+        """
+        query = "SELECT * FROM public.attributes ORDER BY position"
+        result = await Postgres.fetch(query)
+
+        # Build new data in a temporary structure first to minimize
+        # the window of inconsistency
+        new_data: collections.defaultdict[str, list[Any]] = collections.defaultdict(list)
+        new_info_data: list[Any] = []
+
+        for row in result:
+            new_info_data.append(row)
+            for scope in row["scope"]:
+                attrd = {"name": row["name"], **row["data"]}
+                if (scope != "project") and ("default" in attrd):
+                    del attrd["default"]
+                new_data[scope].append(attrd)
+
+        # Update self.data in-place to preserve list object identity.
+        # ModelSet instances hold direct references to these list objects,
+        # so we must mutate them rather than replace them.
+        all_scopes = set(self.data.keys()) | set(new_data.keys())
+        for scope in all_scopes:
+            self.data[scope].clear()
+            self.data[scope].extend(new_data.get(scope, []))
+
+        self.info_data = new_info_data
+
+        # Clear functools caches since the underlying data has changed
+        AttributeLibrary.inheritable_attributes.cache_clear()
+        AttributeLibrary.by_name.cache_clear()
+        AttributeLibrary.by_name_scoped.cache_clear()
+
+        # Invalidate all registered ModelSet Pydantic model caches
+        for callback in self._invalidation_callbacks:
+            callback()
+
+        logger.info("Attribute library reloaded")
+
+    async def reload_handler(self, event: Any = None) -> None:
+        """Event handler adapter for reload(), for use with EventStream.subscribe."""
+        await self.reload()
+
 
 attribute_library = AttributeLibrary()

ayon_server/entities/models/__init__.py

@@ -16,6 +16,7 @@
     version_fields,
     workfile_fields,
 )
+from ayon_server.entities.core.attrib import attribute_library
 from ayon_server.entities.models.generator import generate_model
 from ayon_server.types import (
     ENTITY_ID_EXAMPLE,
@@ -85,6 +86,50 @@ def __init__(
         self._patch_model: type[BaseModel] | None = None
         self._attrib_model: type[BaseModel] | None = None
 
+        attribute_library.register_invalidation_callback(self.invalidate)
+
+    def invalidate(self) -> None:
+        """Update the attrib model's fields in-place after an attribute reload.
+
+        FastAPI evaluates route type annotations (e.g. `post_data:
+        FolderEntity.model.patch_model`) once at import time and holds the
+        resulting Pydantic class for the lifetime of the process.  Creating
+        a brand-new class on reload would be invisible to those routes.
+
+        Instead we mutate the *existing* `_attrib_model` class in-place:
+        - Replace its `__fields__` dict so Pydantic's validator picks up
+          added / removed / changed attributes on the very next request.
+        - Clear `__schema_cache__` on all four models so the OpenAPI schema
+          is regenerated correctly on the next `/openapi.json` request.
+
+        If the attrib model has not been built yet (still None) the fresh
+        build from the updated `self.attributes` list will happen lazily on
+        the next access, so no action is needed.
+        """
+        if self._attrib_model is None:
+            return
+
+        # Build a temporary model to obtain the updated ModelField objects.
+        new_attrib_model = generate_model(
+            f"{self.entity_name.capitalize()}AttribModel",
+            self.attributes,
+            AttribModelConfig,
+        )
+
+        # Swap fields in-place on the class FastAPI already holds a reference to.
+        self._attrib_model.__fields__.clear()
+        self._attrib_model.__fields__.update(new_attrib_model.__fields__)
+
+        # Clear Pydantic's cached JSON schemas so OpenAPI reflects the changes.
+        for model in (
+            self._attrib_model,
+            self._model,
+            self._post_model,
+            self._patch_model,
+        ):
+            if model is not None:
+                model.__schema_cache__.clear()
+
     @property
     def attrib_model(self) -> type[BaseModel]:
         """Return the attribute model."""

ayon_server/events/default_hooks.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING
 
+import aiocache
+
 from ayon_server.lib.redis import Redis
 from ayon_server.logging import logger
 
@@ -16,8 +18,19 @@ async def clear_settings_cache(event: "EventModel"):
     await Redis.delete_ns("all-settings")
 
 
+async def clear_attribute_info_cache(event: "EventModel"):
+    """Clear the in-memory aiocache for the /info attributes response.
+
+    Called on all nodes via global hook so each instance flushes its own
+    local cache immediately when attributes are updated.
+    """
+    logger.trace("Clearing attribute info cache")
+    await aiocache.caches.get("default").clear()
+
+
 DEFAULT_HOOKS: list[tuple[str, HandlerType, bool]] = [
     ("settings.changed", clear_settings_cache, False),
     ("bundle.created", clear_settings_cache, False),
     ("bundle.updated", clear_settings_cache, False),
+    ("server.attributes_updated", clear_attribute_info_cache, True),
 ]

ayon_server/graphql/__init__.py

@@ -226,3 +226,77 @@ def process_errors(
     graphql_ide=None,
     context_getter=graphql_get_context,
 )
+
+
+def rebuild_graphql_schema() -> None:
+    """Rebuild the Strawberry GraphQL schema after attribute changes.
+
+    Strawberry compiles the GraphQL schema once at startup. Each entity's
+    XxxAttribType is a Strawberry type whose fields were read from the Pydantic
+    attrib model at decoration time. When attributes change we must:
+
+      1. Regenerate the fields on each attrib type in-place (same class object,
+         so existing return-type annotations on node resolvers remain valid).
+      2. Rebuild the compiled schema so graphql-core validates queries against
+         the updated type defThe initions.
+    """
+    from strawberry.experimental.pydantic import type as pydantic_type_decorator
+
+    from ayon_server.entities import (
+        FolderEntity,
+        ProductEntity,
+        ProjectEntity,
+        RepresentationEntity,
+        TaskEntity,
+        UserEntity,
+        VersionEntity,
+        WorkfileEntity,
+    )
+    from ayon_server.graphql.nodes import folder as folder_mod
+    from ayon_server.graphql.nodes import product as product_mod
+    from ayon_server.graphql.nodes import project as project_mod
+    from ayon_server.graphql.nodes import representation as representation_mod
+    from ayon_server.graphql.nodes import task as task_mod
+    from ayon_server.graphql.nodes import user as user_mod
+    from ayon_server.graphql.nodes import version as version_mod
+    from ayon_server.graphql.nodes import workfile as workfile_mod
+
+    pairs = [
+        (FolderEntity, folder_mod.FolderAttribType),
+        (TaskEntity, task_mod.TaskAttribType),
+        (ProductEntity, product_mod.ProductAttribType),
+        (VersionEntity, version_mod.VersionAttribType),
+        (WorkfileEntity, workfile_mod.WorkfileAttribType),
+        (RepresentationEntity, representation_mod.RepresentationAttribType),
+        (UserEntity, user_mod.UserAttribType),
+        (ProjectEntity, project_mod.ProjectAttribType),
+    ]
+
+    for entity_cls, existing_attrib_type in pairs:
+        temp = type(existing_attrib_type.__name__, (), {})
+        new_type = pydantic_type_decorator(
+            model=entity_cls.model.attrib_model, all_fields=True
+        )(temp)
+        existing_attrib_type.__strawberry_definition__.fields[:] = (
+            new_type.__strawberry_definition__.fields
+        )
+        # Strawberry types are dataclasses; __init__, __repr__, __eq__ are
+        # generated from fields at decoration time and must be replaced too so
+        # that constructing XxxAttribType(**attrib_dict) accepts new fields.
+        existing_attrib_type.__dataclass_fields__ = dict(
+            new_type.__dataclass_fields__
+        )
+        existing_attrib_type.__init__ = new_type.__init__
+        existing_attrib_type.__repr__ = new_type.__repr__
+        existing_attrib_type.__eq__ = new_type.__eq__
+
+    router.schema = AyonSchema(query=Query)
+    logger.info("GraphQL schema rebuilt after attribute update")
+
+
+# Register the GraphQL schema rebuild as an attribute invalidation callback so
+# it fires whenever attribute_library.reload() is called (which happens on every
+# server.attributes_updated event, including from other server instances via Redis).
+from ayon_server.entities.core.attrib import attribute_library  # noqa: E402
+
+attribute_library.register_invalidation_callback(rebuild_graphql_schema)