diff --git a/CMakeLists.txt b/CMakeLists.txt index 97a98889..350139ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -282,6 +282,7 @@ add_library(moqx_core STATIC src/admin/BuiltinRoutes.cpp src/admin/CachePurgeHandler.cpp src/admin/ConfigHandler.cpp + src/admin/ConnectionLogsHandler.cpp src/admin/MetricsHandler.cpp src/admin/TrackMetricsHandler.cpp src/admin/StateHandler.cpp diff --git a/src/admin/ConnectionLogsHandler.cpp b/src/admin/ConnectionLogsHandler.cpp new file mode 100644 index 00000000..59b4edf9 --- /dev/null +++ b/src/admin/ConnectionLogsHandler.cpp @@ -0,0 +1,242 @@ +/* + * Copyright (c) OpenMOQ contributors. + * This source code is licensed under the Apache 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "admin/ConnectionLogsHandler.h" + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "admin/AdminResponse.h" +#include "admin/AdminServer.h" + +namespace openmoq::moqx::admin { + +namespace { + +constexpr size_t kMaxDownloadBytes = 512ULL * 1024 * 1024; // 512 MB hard cap +constexpr size_t kChunkSize = 64 * 1024; + +// Normalize a raw connection ID string: +// - strip 0x/0X prefix +// - lowercase hex digits +// - validate hex-only, 1–40 chars +std::optional normalizeConnectionId(std::string_view raw) { + if (raw.size() >= 2 && raw[0] == '0' && (raw[1] == 'x' || raw[1] == 'X')) { + raw.remove_prefix(2); + } + std::string result; + result.reserve(raw.size()); + for (char c : raw) { + if (!std::isxdigit(static_cast(c))) + return std::nullopt; + result += static_cast(std::tolower(static_cast(c))); + } + if (result.empty() || result.size() > 40) + return std::nullopt; + return result; +} + +// Blocking; must run off the event-loop thread. Returns nullptr if the file +// cannot be opened, is not a regular file, is empty, or exceeds maxBytes. +// maxBytes is a policy limit on what a client may pull, not a memory bound: +// the body is streamed, so it never lands in the process whole. +std::unique_ptr openLogFile(const std::string& path, size_t maxBytes) { + std::unique_ptr file; + try { + file = std::make_unique(path, O_RDONLY); + } catch (const std::exception&) { + return nullptr; + } + + struct stat st{}; + if (::fstat(file->fd(), &st) != 0 || !S_ISREG(st.st_mode)) + return nullptr; + const auto size = static_cast(st.st_size); + if (size == 0 || size > maxBytes) + return nullptr; + + return file; +} + +// Blocking; must run off the event-loop thread. Returns a zero-length buffer +// at EOF, nullptr on read error. +std::unique_ptr readChunk(int fd) { + auto buf = folly::IOBuf::create(kChunkSize); + const auto rc = folly::readNoInt(fd, buf->writableTail(), kChunkSize); + if (rc < 0) + return nullptr; + buf->append(static_cast(rc)); + return buf; +} + +// Runs on the admin event base. Every resumption point must re-check +// cancelToken: downstream is destroyed as soon as cancellation fires. +folly::coro::Task streamLogFile( + std::string filePath, + std::string fileName, + proxygen::ResponseHandler* downstream, + folly::CancellationToken cancelToken +) { + if (cancelToken.isCancellationRequested()) + co_return; + + // Open on the global CPU pool: open(2)/fstat(2) block. + auto openResult = + co_await folly::coro::co_awaitTry(folly::via(folly::getGlobalCPUExecutor(), [&filePath] { + return openLogFile(filePath, kMaxDownloadBytes); + })); + if (openResult.hasException()) { + XLOG(ERR) << "ConnectionLogsHandler: file open threw: " << openResult.exception().what(); + if (!cancelToken.isCancellationRequested()) { + sendError(downstream, 500, "internal error\n"); + } + co_return; + } + + if (cancelToken.isCancellationRequested()) + co_return; + + std::unique_ptr file = std::move(openResult.value()); + if (!file) { + sendError(downstream, 404, "log file not found or exceeds size limit\n"); + co_return; + } + + // No Content-Length: the log is still being appended to while we read it, so + // a length from fstat(2) would be stale by EOF. The body streams chunked. + proxygen::ResponseBuilder(downstream) + .status(200, proxygen::HTTPMessage::getDefaultReason(200)) + .header("Content-Type", "application/json") + .header("Content-Disposition", "attachment; filename=\"" + fileName + "\"") + .send(); + + // Read on the CPU pool, send from the event-loop thread: each disk read + // overlaps the network write of the chunk before it. + for (;;) { + auto chunkResult = co_await folly::coro::co_awaitTry( + folly::via(folly::getGlobalCPUExecutor(), [fd = file->fd()] { return readChunk(fd); }) + ); + + if (cancelToken.isCancellationRequested()) + co_return; + + if (chunkResult.hasException() || !chunkResult.value()) { + // Headers are already out, so the only way left to signal failure is to + // tear the response down. + XLOG(ERR) << "ConnectionLogsHandler: read failed for " << filePath; + downstream->sendAbort(); + co_return; + } + + auto chunk = std::move(chunkResult.value()); + if (chunk->empty()) { + proxygen::ResponseBuilder(downstream).sendWithEOM(); + co_return; + } + proxygen::ResponseBuilder(downstream).body(std::move(chunk)).send(); + } +} + +} // namespace + +void registerConnectionLogsRoutes( + AdminServer& adminServer, + const std::optional& logging +) { + std::string mlogDir, qlogDir; + if (logging) { + if (logging->mlog && !logging->mlog->dir.empty()) { + mlogDir = logging->mlog->dir; + } + if (logging->qlog && !logging->qlog->dir.empty()) { + qlogDir = logging->qlog->dir; + } + } + + // ── GET /logs?connection_id=&type=mlog|qlog + // + // Path is constructed directly as {dir}/{normalized_cid}.{ext} + adminServer.addRoute( + "GET", + "/logs", + [mlogDir = std::move(mlogDir), qlogDir = std::move(qlogDir)]( + std::unique_ptr req, + std::unique_ptr /*body*/, + proxygen::ResponseHandler* downstream, + folly::CancellationToken cancelToken + ) { + // Resolve type → directory, file extension, Content-Type. + const auto& typeStr = req->getQueryParam("type"); + const std::string* dir = nullptr; + const char* ext = nullptr; + if (typeStr == "mlog") { + dir = &mlogDir; + ext = ".mlog"; + } else if (typeStr == "qlog") { + dir = &qlogDir; + ext = ".qlog"; + } else { + sendError(downstream, 400, "type must be 'mlog' or 'qlog'\n"); + return; + } + + if (dir->empty()) { + sendError(downstream, 503, "that log type is not configured\n"); + return; + } + + const auto& rawCid = req->getQueryParam("connection_id"); + if (rawCid.empty()) { + sendError(downstream, 400, "missing connection_id\n"); + return; + } + + auto normCid = normalizeConnectionId(rawCid); + if (!normCid) { + sendError(downstream, 400, "invalid connection_id\n"); + return; + } + + // {dir}/{normalizedCid}.{ext} + auto filePath = *dir + "/" + *normCid + ext; + auto fileName = *normCid + ext; + + auto* evb = folly::EventBaseManager::get()->getEventBase(); + folly::coro::co_withCancellation( + cancelToken, + folly::coro::co_withExecutor( + evb, + streamLogFile( + std::move(filePath), + std::move(fileName), + downstream, + std::move(cancelToken) + ) + ) + ) + .start(); + } + ); +} + +} // namespace openmoq::moqx::admin diff --git a/src/admin/ConnectionLogsHandler.h b/src/admin/ConnectionLogsHandler.h new file mode 100644 index 00000000..542bb1f4 --- /dev/null +++ b/src/admin/ConnectionLogsHandler.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) OpenMOQ contributors. + * This source code is licensed under the Apache 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +#include "config/Config.h" + +namespace openmoq::moqx::admin { + +class AdminServer; + +// Registers GET /logs on the admin server. +// +// GET /logs?connection_id=&type=mlog|qlog +// Resolves the file path as {log_dir}/{normalized_cid}.{ext} and streams +// the file directly. No index or disk scan is required — files written +// after startup are immediately reachable. +// Responds 400 for missing/invalid params, 503 if the requested type is +// not configured, 404 if the file does not exist. +void registerConnectionLogsRoutes( + AdminServer& adminServer, + const std::optional& logging +); + +} // namespace openmoq::moqx::admin diff --git a/src/config/ConfigResolver.cpp b/src/config/ConfigResolver.cpp index 98130e67..d3c98ef1 100644 --- a/src/config/ConfigResolver.cpp +++ b/src/config/ConfigResolver.cpp @@ -1240,7 +1240,11 @@ folly::Expected resolveConfig(const ParsedConfig& c if (!mlogConfig.dir.empty()) { std::error_code ec; const auto st = std::filesystem::status(mlogConfig.dir, ec); - if (ec) { + // A missing directory is not an error here: LogSetup creates it + // (via create_directories) before logging starts. Only reject + // genuine access failures (e.g. permission denied on a parent + // directory). + if (ec && st.type() != std::filesystem::file_type::not_found) { return folly::makeUnexpected( "Failed to access mlog directory '" + mlogConfig.dir + "': " + ec.message() ); diff --git a/src/main.cpp b/src/main.cpp index 9dd15e6d..7f261729 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,6 +10,7 @@ #include "admin/BuiltinRoutes.h" #include "admin/CachePurgeHandler.h" #include "admin/ConfigHandler.h" +#include "admin/ConnectionLogsHandler.h" #include "admin/MetricsHandler.h" #include "admin/StateHandler.h" #include "admin/TrackMetricsHandler.h" @@ -204,6 +205,7 @@ int main(int argc, char* argv[]) { } admin::registerTrackMetricsRoute(adminServer, context, trackLimits); admin::registerConfigRoute(adminServer, std::make_shared(config)); + admin::registerConnectionLogsRoutes(adminServer, config.logging); // === 8. Start serving === for (auto& server : servers) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 65ea2767..086ae16a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -244,6 +244,10 @@ add_test( NAME admin_config_endpoint COMMAND bash ${PROJECT_SOURCE_DIR}/test/test_admin_config.sh $ ) +add_test( + NAME admin_connection_logs_endpoint + COMMAND bash ${PROJECT_SOURCE_DIR}/test/test_admin_connection_logs.sh $ +) add_test( NAME admin_cache_purge_concurrency_test COMMAND bash ${PROJECT_SOURCE_DIR}/test/test_admin_cache_purge_race.sh $ diff --git a/test/test_admin_connection_logs.sh b/test/test_admin_connection_logs.sh new file mode 100755 index 00000000..fb30a4e8 --- /dev/null +++ b/test/test_admin_connection_logs.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -euo pipefail + +BINARY="${1:-$(dirname "$0")/../build/moqx}" +# shellcheck source=test_ports.sh +source "$(dirname "$0")/test_ports.sh" +LISTEN_PORT=$TEST_ADMIN_CONNECTION_LOGS_LISTEN +ADMIN_PORT=$TEST_ADMIN_CONNECTION_LOGS_ADMIN +LOGS_URL="http://localhost:${ADMIN_PORT}/logs" +INFO_URL="http://localhost:${ADMIN_PORT}/info" + +if [[ ! -x "$BINARY" ]]; then + echo "ERROR: binary not found or not executable: $BINARY" >&2 + exit 1 +fi + +TMPDIR=$(mktemp -d) +MOQX_PID="" +cleanup() { + if [[ -n "${MOQX_PID:-}" ]]; then + kill "$MOQX_PID" 2>/dev/null || true + wait "$MOQX_PID" 2>/dev/null || true + fi + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +# Set up fake log directories +mkdir -p "$TMPDIR/mlog" +mkdir -p "$TMPDIR/qlog" + +# Add logging config to the test config +"$(dirname "$0")/make_test_config.sh" "$LISTEN_PORT" "$ADMIN_PORT" > "$TMPDIR/config.yaml" +cat <> "$TMPDIR/config.yaml" +logging: + mlog: + dir: "$TMPDIR/mlog" + qlog: + dir: "$TMPDIR/qlog" +EOF + +# Create a fake mlog file +echo '{"fake":"mlog"}' > "$TMPDIR/mlog/abcdef123456.mlog" + +# A second one past the 64 KB read chunk, so the streaming loop spans chunks +BIG_MLOG="$TMPDIR/mlog/beef00000001.mlog" +for i in $(seq 1 20000); do + echo "{\"line\":$i}" +done > "$BIG_MLOG" + +# Start moqx with the generated config in the background. +"$BINARY" --config="$TMPDIR/config.yaml" & +MOQX_PID=$! + +# Wait for readiness +for i in $(seq 1 100); do + HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "$INFO_URL" 2>/dev/null || echo "000") + if [[ "$HTTP_CODE" == "200" ]]; then + break + fi + sleep 0.1 + if [[ $i -eq 100 ]]; then + echo "ERROR: admin /info endpoint did not become ready in time" >&2 + exit 1 + fi +done + +echo "Running tests..." + +# Test 1: Missing params returns 400 +HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "${LOGS_URL}" 2>/dev/null || true) +if [[ "$HTTP_CODE" != "400" ]]; then + echo "FAIL: expected HTTP 400 for missing params, got $HTTP_CODE" >&2 + exit 1 +fi + +# Test 2: Invalid type returns 400 +HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "${LOGS_URL}?type=invalid&connection_id=123" 2>/dev/null || true) +if [[ "$HTTP_CODE" != "400" ]]; then + echo "FAIL: expected HTTP 400 for invalid type, got $HTTP_CODE" >&2 + exit 1 +fi + +# Test 3: Valid type but missing file returns 404 +HTTP_CODE=$(curl -sw "%{http_code}" -o /dev/null "${LOGS_URL}?type=mlog&connection_id=abcd" 2>/dev/null || true) +if [[ "$HTTP_CODE" != "404" ]]; then + echo "FAIL: expected HTTP 404 for missing file, got $HTTP_CODE" >&2 + exit 1 +fi + +# Test 4: Valid file returns 200 and correct content +HEADERS_FILE=$(mktemp) +trap 'rm -f "$HEADERS_FILE"' RETURN +HTTP_CODE=$(curl -sw "%{http_code}" -D "$HEADERS_FILE" -o /tmp/logs_response.txt "${LOGS_URL}?type=mlog&connection_id=abcdef123456" 2>/dev/null || true) + +if [[ "$HTTP_CODE" != "200" ]]; then + echo "FAIL: expected HTTP 200 for existing mlog, got $HTTP_CODE" >&2 + exit 1 +fi + +HEADERS=$(cat "$HEADERS_FILE") +RESPONSE=$(cat /tmp/logs_response.txt) +rm -f /tmp/logs_response.txt + +if ! grep -qi 'content-type:.*application/json' <<<"$HEADERS"; then + echo "FAIL: expected application/json content type" >&2 + echo "Got headers: $HEADERS" >&2 + exit 1 +fi + +# The body is streamed, so the length is not known when headers go out. +if grep -qi '^content-length:' <<<"$HEADERS"; then + echo "FAIL: expected a streamed response, got Content-Length" >&2 + echo "Got headers: $HEADERS" >&2 + exit 1 +fi + +if ! grep -q '{"fake":"mlog"}' <<<"$RESPONSE"; then + echo "FAIL: response body did not match expected mlog content" >&2 + exit 1 +fi + +# Test 5: A multi-chunk file arrives byte-for-byte +BIG_RESPONSE="$TMPDIR/big_response.mlog" +HTTP_CODE=$(curl -sw "%{http_code}" -o "$BIG_RESPONSE" "${LOGS_URL}?type=mlog&connection_id=beef00000001" 2>/dev/null || true) + +if [[ "$HTTP_CODE" != "200" ]]; then + echo "FAIL: expected HTTP 200 for large mlog, got $HTTP_CODE" >&2 + exit 1 +fi + +if ! cmp -s "$BIG_MLOG" "$BIG_RESPONSE"; then + echo "FAIL: large mlog body differs from the file on disk" >&2 + exit 1 +fi + +echo "PASS" diff --git a/test/test_ports.sh b/test/test_ports.sh index 2bd9458f..84f5489a 100644 --- a/test/test_ports.sh +++ b/test/test_ports.sh @@ -34,6 +34,10 @@ TEST_CACHE_PURGE_ADMIN=9667 TEST_ADMIN_CONFIG_LISTEN=9668 TEST_ADMIN_CONFIG_ADMIN=9669 +# test_admin_connection_logs.sh +TEST_ADMIN_CONNECTION_LOGS_LISTEN=9670 +TEST_ADMIN_CONNECTION_LOGS_ADMIN=9671 + # test_relay_chain.sh (two relay instances) TEST_RELAY_CHAIN_UPSTREAM=19668 TEST_RELAY_CHAIN_UPSTREAM_ADMIN=19669