From 03c93f6fd940e66945f3fba55b7cbf4047576505 Mon Sep 17 00:00:00 2001 From: CaptainB Date: Sun, 20 Sep 2026 10:36:00 +0800 Subject: [PATCH] fix: implement fetch_public_url function for secure URL downloads in sandbox --- .gitignore | 3 +- apps/oss/serializers/file.py | 35 ++------- apps/oss/url_fetch.py | 144 +++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 30 deletions(-) create mode 100644 apps/oss/url_fetch.py diff --git a/.gitignore b/.gitignore index 0ac1c2149be..32f4aaf3ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -192,4 +192,5 @@ apps/models_provider/impl/tencent_model_provider/model/stt.py tmp/ config.yml .SANDBOX_BANNED_HOSTS -copilot-instructions.md \ No newline at end of file +copilot-instructions.md +.ai/ \ No newline at end of file diff --git a/apps/oss/serializers/file.py b/apps/oss/serializers/file.py index 24994eacde7..0042cf3a6d2 100644 --- a/apps/oss/serializers/file.py +++ b/apps/oss/serializers/file.py @@ -1,4 +1,5 @@ # coding=utf-8 +import inspect import re import urllib @@ -9,7 +10,6 @@ from common.constants.authentication_type import AuthenticationType from common.database_model_manage.database_model_manage import DatabaseModelManage from common.exception.app_exception import AppApiException, AppUnauthorizedFailed, NotFound404 -from common.utils.common import common_convert_value from django.db.models import QuerySet from django.http import HttpResponse from django.utils.translation import gettext @@ -22,6 +22,7 @@ ) from knowledge.models import Document, File, FileSourceType, Knowledge, PublicFileAccess from maxkb.const import CONFIG +from oss.url_fetch import fetch_public_url from rest_framework import serializers from system_manage.models import WorkspaceUserResourcePermission from system_manage.models.resource_mapping import ResourceMapping, ResourceType @@ -367,34 +368,10 @@ def get_url_content(url, application_id: str): file_limit = application.file_upload_setting.get('fileLimit') * 1024 * 1024 try: from common.utils.tool_code import ToolExecutor - response = ToolExecutor().exec_code( - """ - def get_url_content(url): - import requests - requests.packages.urllib3.disable_warnings() - response = requests.get(url, verify=False, allow_redirects=False) - content_type = response.headers.get('Content-Type', '') - if 'text' in content_type or 'json' in content_type: - content = response.text - else: - import base64 - content = base64.b64encode(response.content).decode('utf-8') - return { - "status_code": response.status_code, - "Content-Type": content_type, - "Content-Length": response.headers.get('Content-Length', 0), - "content": content, - } - """, - {"url": url} + return ToolExecutor().exec_code( + inspect.getsource(fetch_public_url), + {"url": url, "file_limit": file_limit}, + function_name="fetch_public_url", ) except Exception as e: raise AppApiException(500, str(e)) - if int(response.get('Content-Length')) > file_limit: - raise AppApiException(500, _('File size exceeds limit')) - return { - 'status_code': response.get('status_code'), - 'Content-Type': response.get('Content-Type'), - 'Content-Length': response.get('Content-Length'), - 'content': response.get('content'), - } diff --git a/apps/oss/url_fetch.py b/apps/oss/url_fetch.py new file mode 100644 index 00000000000..5727c5f1462 --- /dev/null +++ b/apps/oss/url_fetch.py @@ -0,0 +1,144 @@ +# coding=utf-8 + + +def fetch_public_url(url, file_limit): + """Download a public HTTP(S) URL inside ToolExecutor's sandbox. + + Keep this function self-contained: its source is sent to the sandbox, which + cannot import application modules. Resolve once and connect to the checked + socket addresses, while preserving the hostname for HTTP and TLS. + """ + import base64 + import ipaddress + import socket + import time + from urllib.parse import urlsplit + + import requests + from urllib3 import HTTPConnectionPool, HTTPSConnectionPool + from urllib3.connection import HTTPConnection, HTTPSConnection + from urllib3.exceptions import NewConnectionError + from urllib3.util import Timeout + + file_limit = int(file_limit) + if file_limit <= 0: + raise ValueError("Invalid file size limit") + if not isinstance(url, str) or not url or any(ord(c) <= 32 or ord(c) == 127 or c == "\\" for c in url): + raise ValueError("Invalid URL") + parsed = urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise ValueError("Only HTTP and HTTPS URLs are allowed") + if parsed.username is not None or parsed.password is not None or "%" in parsed.hostname: + raise ValueError("URL credentials and scoped or encoded hostnames are not allowed") + if parsed.port is not None and parsed.port == 0: + raise ValueError("Invalid URL port") + + # Use the HTTP client's normalized hostname (including IDNA), not a second + # parser's interpretation of the original URL. + prepared = requests.Request("GET", url).prepare() + parsed = urlsplit(prepared.url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + addresses = socket.getaddrinfo(parsed.hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + if not addresses: + raise ValueError("Failed to resolve URL hostname") + for family, _, _, _, address in addresses: + if family not in (socket.AF_INET, socket.AF_INET6): + raise ValueError("Unsupported address family") + ip = ipaddress.ip_address(address[0]) + if not ip.is_global or ip.is_multicast or ip.is_reserved: + raise ValueError("Access to non-public IP addresses is blocked") + if isinstance(ip, ipaddress.IPv6Address) and ( + ip.ipv4_mapped is not None + or ip.sixtofour is not None + or ip.teredo is not None + or ip in ipaddress.ip_network("64:ff9b::/96") + or ip in ipaddress.ip_network("64:ff9b:1::/48") + ): + raise ValueError("Access to IPv6 transition addresses is blocked") + + deadline = time.monotonic() + 30 + + class PinnedConnection: + def _new_conn(self): + last_error = None + for family, socktype, proto, _, address in addresses: + if time.monotonic() >= deadline: + raise TimeoutError("URL download timed out") + sock = socket.socket(family, socktype, proto) + try: + sock.settimeout(min(self.timeout, max(0.001, deadline - time.monotonic()))) + for option in self.socket_options or (): + sock.setsockopt(*option) + # Do not pass the hostname to create_connection/getaddrinfo + # again: DNS may have changed since validation. + sock.connect(address) + return sock + except OSError as exc: + last_error = exc + sock.close() + raise NewConnectionError(self, "Could not connect to URL") from last_error + + class PinnedHTTPConnection(PinnedConnection, HTTPConnection): + pass + + class PinnedHTTPSConnection(PinnedConnection, HTTPSConnection): + pass + + if parsed.scheme == "https": + pool = HTTPSConnectionPool(parsed.hostname, port, cert_reqs="CERT_REQUIRED", ca_certs=requests.certs.where()) + pool.ConnectionCls = PinnedHTTPSConnection + else: + pool = HTTPConnectionPool(parsed.hostname, port) + pool.ConnectionCls = PinnedHTTPConnection + + response = None + try: + # Direct pools do not inherit HTTP_PROXY/HTTPS_PROXY or .netrc credentials. + response = pool.urlopen( + "GET", + prepared.path_url, + headers={"Host": parsed.netloc, "Accept-Encoding": "identity"}, + redirect=False, + retries=False, + preload_content=False, + timeout=Timeout(connect=5, read=10), + ) + content_length = response.headers.get("Content-Length") + if content_length is not None: + if not content_length.isascii() or not content_length.isdecimal(): + raise ValueError("Invalid Content-Length") + if int(content_length) > file_limit: + raise ValueError("File size exceeds limit") + + # Count decoded bytes as they arrive, including chunked/compressed bodies + # and responses without Content-Length. Never trust a header as the limit. + body = bytearray() + while True: + if time.monotonic() >= deadline: + raise TimeoutError("URL download timed out") + chunk = response.read1(min(64 * 1024, file_limit - len(body) + 1), decode_content=True) + if not chunk: + break + if len(body) + len(chunk) > file_limit: + raise ValueError("File size exceeds limit") + body.extend(chunk) + + content_type = response.headers.get("Content-Type", "") + if "text" in content_type.lower() or "json" in content_type.lower(): + # Preserve requests' charset detection and text/binary response format. + decoded = requests.Response() + decoded._content = bytes(body) + decoded.encoding = requests.utils.get_encoding_from_headers(response.headers) + content = decoded.text + else: + content = base64.b64encode(body).decode("ascii") + return { + "status_code": response.status, + "Content-Type": content_type, + "Content-Length": len(body), + "content": content, + } + finally: + if response is not None: + response.close() + pool.close()