fix: implement fetch_public_url function for secure URL downloads in sandbox

.gitignore

@@ -192,4 +192,5 @@ apps/models_provider/impl/tencent_model_provider/model/stt.py
 tmp/
 config.yml
 .SANDBOX_BANNED_HOSTS
-copilot-instructions.md
+copilot-instructions.md
+.ai/

apps/oss/serializers/file.py

@@ -9,7 +9,6 @@
 from common.constants.authentication_type import AuthenticationType
 from common.database_model_manage.database_model_manage import DatabaseModelManage
 from common.exception.app_exception import AppApiException, AppUnauthorizedFailed, NotFound404
-from common.utils.common import common_convert_value
 from django.db.models import QuerySet
 from django.http import HttpResponse
 from django.utils.translation import gettext
@@ -22,6 +21,7 @@
 )
 from knowledge.models import Document, File, FileSourceType, Knowledge, PublicFileAccess
 from maxkb.const import CONFIG
+from oss.url_fetch import FETCH_URL_CODE
 from rest_framework import serializers
 from system_manage.models import WorkspaceUserResourcePermission
 from system_manage.models.resource_mapping import ResourceMapping, ResourceType
@@ -367,34 +367,10 @@ def get_url_content(url, application_id: str):
         file_limit = application.file_upload_setting.get('fileLimit') * 1024 * 1024
     try:
         from common.utils.tool_code import ToolExecutor
-        response = ToolExecutor().exec_code(
-            """
-    def get_url_content(url):
-        import requests
-        requests.packages.urllib3.disable_warnings()
-        response = requests.get(url, verify=False, allow_redirects=False)
-        content_type = response.headers.get('Content-Type', '')
-        if 'text' in content_type or 'json' in content_type:
-            content = response.text
-        else:
-            import base64
-            content = base64.b64encode(response.content).decode('utf-8')
-        return {
-            "status_code": response.status_code,
-            "Content-Type": content_type,
-            "Content-Length": response.headers.get('Content-Length', 0),
-            "content": content,
-        }
-    """,
-            {"url": url}
+        return ToolExecutor().exec_code(
+            FETCH_URL_CODE,
+            {"url": url, "file_limit": file_limit},
+            function_name="fetch_url",
         )
     except Exception as e:
         raise AppApiException(500, str(e))
-    if int(response.get('Content-Length')) > file_limit:
-        raise AppApiException(500, _('File size exceeds limit'))
-    return {
-        'status_code': response.get('status_code'),
-        'Content-Type': response.get('Content-Type'),
-        'Content-Length': response.get('Content-Length'),
-        'content': response.get('content'),
-    }

apps/oss/url_fetch.py

@@ -0,0 +1,85 @@
+# coding=utf-8
+
+# Run only in ToolExecutor's dedicated subprocess, never in the web process:
+# the resolver is temporarily wrapped for this download. Keep the source here
+# so execution does not depend on inspect.getsource in packaged releases.
+FETCH_URL_CODE = r'''
+def fetch_url(url, file_limit):
+    import base64
+    import ipaddress
+    import socket
+    import time
+    from urllib.parse import urlsplit
+
+    import requests
+
+    file_limit = int(file_limit)
+    if file_limit <= 0:
+        raise ValueError("Invalid file size limit")
+    if not isinstance(url, str) or not url or any(ord(c) <= 32 or ord(c) == 127 or c == "\\" for c in url):
+        raise ValueError("Invalid URL")
+    parsed = urlsplit(url)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname:
+        raise ValueError("Only HTTP and HTTPS URLs are allowed")
+    if parsed.username is not None or parsed.password is not None or "%" in parsed.hostname:
+        raise ValueError("URL credentials and scoped or encoded hostnames are not allowed")
+    if parsed.port == 0:
+        raise ValueError("Invalid URL port")
+
+    resolve = socket.getaddrinfo
+
+    def resolve_public(*args, **kwargs):
+        addresses = resolve(*args, **kwargs)
+        if not addresses:
+            raise ValueError("Failed to resolve URL hostname")
+        for family, _, _, _, address in addresses:
+            if family not in (socket.AF_INET, socket.AF_INET6):
+                raise ValueError("Unsupported address family")
+            ip = ipaddress.ip_address(address[0])
+            if not ip.is_global or ip.is_multicast or ip.is_reserved or (
+                ip.version == 6 and (ip.is_site_local or ip.ipv4_mapped or ip.sixtofour or ip.teredo)
+            ):
+                raise ValueError("Access to non-public IP addresses is blocked")
+        return addresses
+
+    # Validate the addresses requests/urllib3 will actually connect to, not a
+    # separate DNS lookup followed by an unchecked second resolution.
+    deadline = time.monotonic() + 30
+    socket.getaddrinfo = resolve_public
+    try:
+        with requests.Session() as session:
+            session.trust_env = False
+            # requests otherwise buffers redirect bodies even when redirects are disabled.
+            session.resolve_redirects = lambda *args, **kwargs: iter(())
+            with session.get(url, stream=True, timeout=(5, 10), allow_redirects=False) as response:
+                length = response.headers.get("Content-Length")
+                if length is not None:
+                    if not length.isascii() or not length.isdecimal():
+                        raise ValueError("Invalid Content-Length")
+                    if int(length) > file_limit:
+                        raise ValueError("File size exceeds limit")
+                body = bytearray()
+                while True:
+                    if time.monotonic() >= deadline:
+                        raise TimeoutError("URL download timed out")
+                    chunk = response.raw.read1(min(64 * 1024, file_limit - len(body) + 1), decode_content=True)
+                    if not chunk:
+                        break
+                    if len(body) + len(chunk) > file_limit:
+                        raise ValueError("File size exceeds limit")
+                    body.extend(chunk)
+                content_type = response.headers.get("Content-Type", "")
+                if "text" in content_type.lower() or "json" in content_type.lower():
+                    response._content = bytes(body)
+                    content = response.text
+                else:
+                    content = base64.b64encode(body).decode("ascii")
+                return {
+                    "status_code": response.status_code,
+                    "Content-Type": content_type,
+                    "Content-Length": len(body),
+                    "content": content,
+                }
+    finally:
+        socket.getaddrinfo = resolve
+'''