feat: unify LLM settings (default model, multi-provider, models, keys) and fix oauth2 mysql persistenr bug (#181)

Co-authored-by: nishenghao.nsh <nishenghao.nsh@oceanbase.com>

Author: niiish32x

packages/derisk-app/src/derisk_app/app.py

@@ -173,26 +173,24 @@ def _sync_oauth2_config_from_db():
     """
     try:
         from derisk_app.config_storage.oauth2_db_storage import get_oauth2_db_storage
-        from derisk.configs.model_config import Config
+        from derisk_core.config import ConfigManager, OAuth2Config
 
         db_storage = get_oauth2_db_storage()
         # Load with actual secrets for runtime use
         db_oauth2 = db_storage.load_with_secrets()
 
         if db_oauth2 is not None:
             # Update the runtime config with database values
-            cfg = Config()
-            if hasattr(cfg, "oauth2"):
-                from derisk_core.config import OAuth2Config
-
-                # Convert dict to OAuth2Config
-                oauth2_config = OAuth2Config(
-                    enabled=db_oauth2.get("enabled", False),
-                    providers=db_oauth2.get("providers", []),
-                    admin_users=db_oauth2.get("admin_users", []),
-                )
-                cfg.oauth2 = oauth2_config
-                logger.info("OAuth2 config loaded from database (secrets loaded for runtime)")
+            cfg = ConfigManager.get()
+            oauth2_config = OAuth2Config(
+                enabled=db_oauth2.get("enabled", False),
+                providers=db_oauth2.get("providers", []),
+                admin_users=db_oauth2.get("admin_users", []),
+            )
+            cfg.oauth2 = oauth2_config
+            logger.info(
+                "OAuth2 config loaded from database (secrets loaded for runtime)"
+            )
         else:
             logger.info("No OAuth2 config in database, using file config")
     except Exception as e:

packages/derisk-app/src/derisk_app/base.py

@@ -88,6 +88,127 @@ def _initialize_db_storage(param: ServiceConfig, system_app: SystemApp):
         try_to_create_db=not disable_alembic_upgrade,
         system_app=system_app,
     )
+    _migrate_mysql_chat_tables_utf8mb4(db_url)
+
+
+def _migrate_mysql_chat_tables_utf8mb4(db_url: str) -> None:
+    """Ensure chat history tables use utf8mb4 (fixes DataError 1366 on emoji).
+
+    New installs get correct DDL from assets/schema/derisk.sql. Legacy MySQL
+    databases may still have utf8/utf8mb3 tables; migrate them automatically.
+    """
+    if "mysql" not in (db_url or "").lower():
+        return
+    try:
+        from sqlalchemy import text
+        from derisk.storage.metadata.db_manager import db
+
+        if not db.is_initialized:
+            return
+
+        engine = db.engine
+        with engine.connect() as conn:
+            tables = conn.execute(
+                text(
+                    """
+                    SELECT TABLE_NAME, TABLE_COLLATION
+                    FROM information_schema.TABLES
+                    WHERE TABLE_SCHEMA = DATABASE()
+                      AND TABLE_NAME IN ('chat_history', 'chat_history_message')
+                    """
+                )
+            ).fetchall()
+
+            if not tables:
+                return
+
+            def _collation_ok(collation: object) -> bool:
+                return str(collation or "").lower().startswith("utf8mb4")
+
+            bad_tables = [
+                name for name, coll in tables if not _collation_ok(coll)
+            ]
+
+            msg_detail_charset = None
+            row = conn.execute(
+                text(
+                    """
+                    SELECT CHARACTER_SET_NAME
+                    FROM information_schema.COLUMNS
+                    WHERE TABLE_SCHEMA = DATABASE()
+                      AND TABLE_NAME = 'chat_history_message'
+                      AND COLUMN_NAME = 'message_detail'
+                    """
+                )
+            ).fetchone()
+            if row:
+                msg_detail_charset = (row[0] or "").lower()
+
+            bad_column = msg_detail_charset and msg_detail_charset != "utf8mb4"
+
+            if not bad_tables and not bad_column:
+                return
+
+            schema = conn.execute(text("SELECT DATABASE()")).scalar()
+            if not schema:
+                return
+
+        # DDL auto-commits; use begin() for a clean transaction boundary per statement.
+        migrated = False
+        with engine.begin() as conn:
+            db_coll = conn.execute(
+                text(
+                    """
+                    SELECT DEFAULT_COLLATION_NAME
+                    FROM information_schema.SCHEMATA
+                    WHERE SCHEMA_NAME = :schema
+                    """
+                ),
+                {"schema": schema},
+            ).scalar()
+            if db_coll and not str(db_coll).lower().startswith("utf8mb4"):
+                conn.execute(
+                    text(
+                        "ALTER DATABASE `{schema}` CHARACTER SET utf8mb4 "
+                        "COLLATE utf8mb4_unicode_ci".format(schema=schema)
+                    )
+                )
+                migrated = True
+
+            if "chat_history" in bad_tables:
+                conn.execute(
+                    text(
+                        "ALTER TABLE `chat_history` CONVERT TO CHARACTER SET utf8mb4 "
+                        "COLLATE utf8mb4_unicode_ci"
+                    )
+                )
+                migrated = True
+            if "chat_history_message" in bad_tables:
+                conn.execute(
+                    text(
+                        "ALTER TABLE `chat_history_message` "
+                        "CONVERT TO CHARACTER SET utf8mb4 "
+                        "COLLATE utf8mb4_unicode_ci"
+                    )
+                )
+                migrated = True
+            elif bad_column:
+                conn.execute(
+                    text(
+                        "ALTER TABLE `chat_history_message` "
+                        "MODIFY COLUMN `message_detail` LONGTEXT "
+                        "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL "
+                        "COMMENT 'Message details, json format'"
+                    )
+                )
+                migrated = True
+
+        if migrated:
+            logger.info(
+                "MySQL chat_history / chat_history_message migrated to utf8mb4."
+            )
+    except Exception as e:
+        logger.error("MySQL utf8mb4 migration for chat tables failed: %s", e)
 
 
 def _add_missing_columns_sqlite(db):

packages/derisk-app/src/derisk_app/config_storage/oauth2_db_storage.py

@@ -8,10 +8,8 @@
 import logging
 from typing import Any, Dict, List, Optional
 
-from cryptography.fernet import Fernet
 from sqlalchemy import Column, DateTime, Integer, String, Text
 
-from derisk._private.config import Config
 from derisk.storage.metadata import BaseDao, Model
 
 logger = logging.getLogger(__name__)
@@ -182,28 +180,37 @@ def _mask_providers_for_display(
                 provider["client_secret"] = "****"
         return masked
 
-    def get_config(self, config_key: str = "global", mask_secrets: bool = True) -> Optional[Dict[str, Any]]:
+    def get_config(
+        self, config_key: str = "global", mask_secrets: bool = True
+    ) -> Optional[Dict[str, Any]]:
         """Get OAuth2 config from database.
 
         Args:
             config_key: Configuration key (default: global)
             mask_secrets: If True, mask client_secret in providers for display
         """
-        entity = self.get_by_key(config_key)
-        if not entity:
-            return None
+        with self.session() as session:
+            entity = (
+                session.query(OAuth2ConfigEntity)
+                .filter(OAuth2ConfigEntity.config_key == config_key)
+                .first()
+            )
+            if not entity:
+                return None
+
+            # Read ORM attributes before the session closes to avoid
+            # detached-instance access during JSON conversion.
+            enabled = bool(entity.enabled)
+            admin_users_json = entity.admin_users_json or "[]"
+            providers_json = entity.providers_json or "[]"
 
         try:
-            admin_users = (
-                json.loads(entity.admin_users_json)
-                if entity.admin_users_json
-                else []
-            )
+            admin_users = json.loads(admin_users_json) if admin_users_json else []
         except json.JSONDecodeError:
             admin_users = []
 
         try:
-            providers = json.loads(entity.providers_json or "[]")
+            providers = json.loads(providers_json)
         except json.JSONDecodeError:
             providers = []
 
@@ -212,12 +219,14 @@ def get_config(self, config_key: str = "global", mask_secrets: bool = True) -> O
             providers = self._mask_providers_for_display(providers)
 
         return {
-            "enabled": bool(entity.enabled),
+            "enabled": enabled,
             "providers": providers,
             "admin_users": admin_users,
         }
 
-    def get_config_with_secrets(self, config_key: str = "global") -> Optional[Dict[str, Any]]:
+    def get_config_with_secrets(
+        self, config_key: str = "global"
+    ) -> Optional[Dict[str, Any]]:
         """Get OAuth2 config with actual secrets (for internal use only)."""
         return self.get_config(config_key, mask_secrets=False)
 
@@ -242,7 +251,9 @@ def load_with_secrets(self) -> Optional[Dict[str, Any]]:
         """Load OAuth2 config with actual secrets (for internal use only)."""
         return self.dao.get_config_with_secrets("global")
 
-    def save(self, enabled: bool, providers: List[Dict], admin_users: List[str]) -> bool:
+    def save(
+        self, enabled: bool, providers: List[Dict], admin_users: List[str]
+    ) -> bool:
         """Save OAuth2 config to database."""
         try:
             self.dao.save_or_update(enabled, providers, admin_users, "global")