🎨 Improves webserver's exception handling to enhance diagnoses of catalog's client errors (#7817)

Author: pcrespov

services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client_service.py

@@ -21,13 +21,15 @@
 from models_library.services_types import ServiceKey, ServiceVersion
 from models_library.users import UserID
 from pydantic import TypeAdapter
-from servicelib.aiohttp import status
 from servicelib.aiohttp.client_session import get_client_session
 from servicelib.rest_constants import X_PRODUCT_NAME_HEADER
+from simcore_service_webserver.catalog.errors import (
+    CatalogConnectionError,
+    CatalogResponseError,
+)
 from yarl import URL
 
 from .._meta import api_version_prefix
-from ._constants import MSG_CATALOG_SERVICE_NOT_FOUND, MSG_CATALOG_SERVICE_UNAVAILABLE
 from .settings import CatalogSettings, get_plugin_settings
 
 _logger = logging.getLogger(__name__)
@@ -51,16 +53,17 @@ def _handle_client_exceptions(app: web.Application) -> Iterator[ClientSession]:
         yield session
 
     except ClientResponseError as err:
-        if err.status == status.HTTP_404_NOT_FOUND:
-            raise web.HTTPNotFound(text=MSG_CATALOG_SERVICE_NOT_FOUND) from err
-        raise web.HTTPServiceUnavailable(
-            reason=MSG_CATALOG_SERVICE_UNAVAILABLE
+        raise CatalogResponseError(
+            status=err.status,
+            message=err.message,
+            headers=err.headers,
+            request_info=err.request_info,
         ) from err
 
     except (TimeoutError, ClientConnectionError) as err:
-        _logger.debug("Request to catalog service failed: %s", err)
-        raise web.HTTPServiceUnavailable(
-            reason=MSG_CATALOG_SERVICE_UNAVAILABLE
+        raise CatalogConnectionError(
+            message=str(err),
+            request_info=getattr(err, "request_info", None),
         ) from err
 
 

services/web/server/src/simcore_service_webserver/catalog/_constants.py

@@ -1,7 +1,12 @@
 from typing import Final
 
-MSG_CATALOG_SERVICE_UNAVAILABLE: Final[
-    str
-] = "Currently catalog service is unavailable, please try again later"
+from ..constants import MSG_TRY_AGAIN_OR_SUPPORT
+
+MSG_CATALOG_SERVICE_UNAVAILABLE: Final[str] = (
+    # Most likely the director service is down or misconfigured so the user is asked to try again later.
+    "This service is temporarily unavailable. The incident was logged and will be investigated. "
+    + MSG_TRY_AGAIN_OR_SUPPORT
+)
+
 
 MSG_CATALOG_SERVICE_NOT_FOUND: Final[str] = "Not Found"

services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py

@@ -1,26 +1,96 @@
 """Defines the different exceptions that may arise in the catalog subpackage"""
 
+import logging
+
+from aiohttp import web
+from common_library.error_codes import create_error_code
+from models_library.rest_error import ErrorGet
 from servicelib.aiohttp import status
+from servicelib.logging_errors import create_troubleshotting_log_kwargs
+from servicelib.rabbitmq._errors import RemoteMethodNotRegisteredError
 from servicelib.rabbitmq.rpc_interfaces.catalog.errors import (
     CatalogForbiddenError,
     CatalogItemNotFoundError,
 )
 
 from ..exception_handling import (
+    ExceptionHandlersMap,
     ExceptionToHttpErrorMap,
     HttpErrorInfo,
+    create_error_context_from_request,
+    create_error_response,
     exception_handling_decorator,
     to_exceptions_handlers_map,
 )
 from ..resource_usage.errors import DefaultPricingPlanNotFoundError
-from .errors import DefaultPricingUnitForServiceNotFoundError
+from ._constants import MSG_CATALOG_SERVICE_NOT_FOUND, MSG_CATALOG_SERVICE_UNAVAILABLE
+from .errors import (
+    CatalogConnectionError,
+    CatalogResponseError,
+    DefaultPricingUnitForServiceNotFoundError,
+)
 
 # mypy: disable-error-code=truthy-function
 assert CatalogForbiddenError  # nosec
 assert CatalogItemNotFoundError  # nosec
 
 
+_logger = logging.getLogger(__name__)
+
+
+async def _handler_catalog_client_errors(
+    request: web.Request, exception: Exception
+) -> web.Response:
+
+    assert isinstance(  # nosec
+        exception, CatalogResponseError | CatalogConnectionError
+    ), f"check mapping, got {exception=}"
+
+    if (
+        isinstance(exception, CatalogResponseError)
+        and exception.status == status.HTTP_404_NOT_FOUND
+    ):
+        error = ErrorGet(
+            status=status.HTTP_404_NOT_FOUND,
+            message=MSG_CATALOG_SERVICE_NOT_FOUND,
+        )
+
+    else:
+        # NOTE: The remaining errors are mapped to 503
+        status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+        user_msg = MSG_CATALOG_SERVICE_UNAVAILABLE
+
+        # Log for further investigation
+        oec = create_error_code(exception)
+        _logger.exception(
+            **create_troubleshotting_log_kwargs(
+                user_msg,
+                error=exception,
+                error_code=oec,
+                error_context={
+                    **create_error_context_from_request(request),
+                    "error_code": oec,
+                },
+            )
+        )
+        error = ErrorGet.model_construct(
+            message=user_msg,
+            support_id=oec,
+            status=status_code,
+        )
+
+    return create_error_response(error, status_code=error.status)
+
+
 _TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = {
+    RemoteMethodNotRegisteredError: HttpErrorInfo(
+        status.HTTP_503_SERVICE_UNAVAILABLE,
+        MSG_CATALOG_SERVICE_UNAVAILABLE,
+    ),
+    CatalogForbiddenError: HttpErrorInfo(
+        status.HTTP_403_FORBIDDEN,
+        "Forbidden catalog access",
+    ),
     CatalogItemNotFoundError: HttpErrorInfo(
         status.HTTP_404_NOT_FOUND,
         "Catalog item not found",
@@ -32,13 +102,17 @@
     DefaultPricingUnitForServiceNotFoundError: HttpErrorInfo(
         status.HTTP_404_NOT_FOUND, "Default pricing unit not found"
     ),
-    CatalogForbiddenError: HttpErrorInfo(
-        status.HTTP_403_FORBIDDEN, "Forbidden catalog access"
-    ),
 }
 
+
+_exceptions_handlers_map: ExceptionHandlersMap = {
+    CatalogResponseError: _handler_catalog_client_errors,
+    CatalogConnectionError: _handler_catalog_client_errors,
+}
+_exceptions_handlers_map.update(to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP))
+
 handle_plugin_requests_exceptions = exception_handling_decorator(
-    to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP)
+    _exceptions_handlers_map
 )
 
 

services/web/server/src/simcore_service_webserver/catalog/errors.py

@@ -1,5 +1,3 @@
-"""Defines the different exceptions that may arise in the catalog subpackage"""
-
 from ..errors import WebServerBaseError
 
 
@@ -23,3 +21,14 @@ def __init__(self, *, service_key: str, service_version: str, **ctxs):
         super().__init__(**ctxs)
         self.service_key = service_key
         self.service_version = service_version
+
+
+class CatalogResponseError(BaseCatalogError):
+    msg_template = "Catalog response with error status {status} and message '{message}'"
+    status: int
+    message: str
+
+
+class CatalogConnectionError(BaseCatalogError):
+    msg_template = "Catalog connection or timeout error: {message}"
+    message: str