diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index bd767833326..fc4e73b893e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -404,7 +404,7 @@ jobs: run: sudo apt-get install -o Acquire::Retries=5 libtool-bin libapr1-dev libaprutil1-dev liblua5.3-dev libbrotli-dev libcurl4-openssl-dev - libnghttp2-dev libjansson-dev libpcre2-dev + libnghttp2-dev libjansson-dev libmagic-dev libpcre2-dev ${{ ! ( env.SKIP_TESTING || env.NO_TEST_FRAMEWORK ) && 'cpanminus perl-doc' || '' }} ${{ ! env.SKIP_TESTING && 'gdb' || '' }} libsasl2-dev curl pipx ${{ matrix.pkgs }} check diff --git a/changes-entries/mime-libmagic.txt b/changes-entries/mime-libmagic.txt new file mode 100644 index 00000000000..28a927de27b --- /dev/null +++ b/changes-entries/mime-libmagic.txt @@ -0,0 +1,2 @@ + *) mod_mime_libmagic: New module determining the MIME type of a file + from its contents using libmagic(3). [Joe Orton] diff --git a/docs/manual/mod/allmodules.xml b/docs/manual/mod/allmodules.xml index d001c25c799..d2c35728fb1 100644 --- a/docs/manual/mod/allmodules.xml +++ b/docs/manual/mod/allmodules.xml @@ -81,6 +81,7 @@ mod_md.xml mod_mime.xml mod_mime_magic.xml + mod_mime_libmagic.xml mod_negotiation.xml mod_nw_ssl.xml mod_policy.xml diff --git a/docs/manual/mod/mod_mime_libmagic.xml b/docs/manual/mod/mod_mime_libmagic.xml new file mode 100644 index 00000000000..09ed4c0c9e7 --- /dev/null +++ b/docs/manual/mod/mod_mime_libmagic.xml @@ -0,0 +1,187 @@ + + + + + + + + + +mod_mime_libmagic +Determines the MIME type of a file from its contents + using libmagic +Extension +mod_mime_libmagic.c +mime_libmagic_module +Available in Apache 2.5.1 and later + + +

This module determines the MIME + type of a file by examining its contents, in the same + way the file(1) command does. It is intended as a + "second line of defense" for files which + mod_mime cannot type from their extension: it + runs after mod_mime and does nothing if a type + has already been set.

+ +

Unlike mod_mime_magic, which has its own + copy of the file(1) engine and its own magic + database, this module uses the system's libmagic(3) + library and magic database from the file package, + so the types it reports are those which file --mime-type + would report. The module is built only if libmagic is found at + configure time, and is only active in a server or virtual host + where MimeLibmagic + is On.

+ +

Where libmagic reports application/octet-stream + the module sets no type, leaving the response without a + Content-Type header, as mod_mime_magic + does for unrecognised content. Directories are typed as + httpd/unix-directory and empty files as + text/plain.

+ +

The module does not look inside compressed files and never + sets a Content-Encoding: a gzip file is reported as + application/gzip. This differs from + mod_mime_magic, which reports the compressed + file's encoding.

+ + +mod_mime +mod_mime_magic + +
Performance and threading + +

A libmagic handle is created, loaded with the magic database + and destroyed for every request examined, so the module is safe + to use with any MPM. Loading a compiled database + (magic.mgc) maps the file into memory rather than + parsing it, and the mapped pages are shared between all server + processes, so this is inexpensive. If the configured database + is a text file rather than a compiled one, it is parsed on every + request.

+ +

Only the first MimeLibmagicBytes bytes of a file are read and + examined, whatever its size.

+ +

Examining text content in particular can allocate a few + megabytes of memory temporarily. With a threaded MPM and a C + library which keeps a separate heap per thread, each server + thread retains its peak allocation, so the resident size of a + busy child process can grow by that much per thread before + levelling off. This is not a leak.

+
+ + +MimeLibmagic +Enable MIME type determination from file contents +MimeLibmagic On|Off +MimeLibmagic Off +server configvirtual host + + + +

The MimeLibmagic directive enables the + module for a server or virtual host. Files which + mod_mime leaves untyped, typically those without + a recognised extension, then have their contents examined by + libmagic.

+ + Example + +MimeLibmagic On + + + + + + +MimeLibmagicFile +The magic database to use +MimeLibmagicFile file-path +The default database of the installed libmagic +server configvirtual host + + + +

The MimeLibmagicFile directive names + the magic database, in place of the default compiled into + libmagic (typically /usr/share/misc/magic, for which + the compiled magic.mgc alongside it is loaded). + The database must be in the format described in + magic(5); the file distributed with + mod_mime_magic is not suitable. A relative path + is taken relative to the ServerRoot. If the database cannot be loaded, an + error is logged for each request examined and the file is left + untyped.

+ +

Note that the MAGIC environment variable, if set + when the server is started, overrides the library default, but + not a database given with this directive.

+ + Example + +MimeLibmagicFile /usr/share/misc/magic.mgc + + + + + + +MimeLibmagicBytes +Number of bytes of a file examined +MimeLibmagicBytes bytes +MimeLibmagicBytes 16384 +server configvirtual host + + + +

The MimeLibmagicBytes directive sets + how many bytes from the start of a file are read and examined. + Larger values allow magic rules which look deeper into a file to + match, at the cost of reading more of every examined file. The + maximum is 1048576.

+ + + + +MimeLibmagicCharset +Add a charset parameter to text types +MimeLibmagicCharset On|Off +MimeLibmagicCharset Off +server configvirtual host + + + +

When MimeLibmagicCharset is + On, the character encoding libmagic detects (such + as us-ascii, utf-8 or + iso-8859-1) is added as a charset + parameter to text/* types, giving for example + text/plain; charset=utf-8. Other types are not + affected. As with AddDefaultCharset, the detected charset should be + trusted only where the content is known to be encoded + consistently.

+ + + + diff --git a/modules/metadata/config.m4 b/modules/metadata/config.m4 index 25d1aece65f..e794861d900 100644 --- a/modules/metadata/config.m4 +++ b/modules/metadata/config.m4 @@ -6,6 +6,23 @@ APACHE_MODPATH_INIT(metadata) APACHE_MODULE(env, clearing/setting of ENV vars, , , yes) APACHE_MODULE(mime_magic, automagically determining MIME type) + +APACHE_MODULE(mime_libmagic, determining MIME type using libmagic, , , most, [ + ap_libmagic_found=no + if test -n "$PKGCONFIG"; then + AC_MSG_CHECKING([for libmagic via pkg-config]) + if $PKGCONFIG --exists libmagic; then + ap_libmagic_found=yes + APR_ADDTO(MOD_CFLAGS, [`$PKGCONFIG --cflags libmagic`]) + APR_ADDTO(MOD_MIME_LIBMAGIC_LDADD, [`$PKGCONFIG --libs libmagic`]) + fi + AC_MSG_RESULT($ap_libmagic_found) + fi + if test "$ap_libmagic_found" = "no"; then + AC_MSG_WARN([libmagic not found]) + enable_mime_libmagic=no + fi +]) APACHE_MODULE(cern_meta, CERN-type meta files, , , no) APACHE_MODULE(expires, Expires header control, , , most) APACHE_MODULE(headers, HTTP header control, , , yes) diff --git a/modules/metadata/mod_mime_libmagic.c b/modules/metadata/mod_mime_libmagic.c new file mode 100644 index 00000000000..fd8a74eb1d3 --- /dev/null +++ b/modules/metadata/mod_mime_libmagic.c @@ -0,0 +1,335 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * mod_mime_libmagic: determine the Content-Type of a file from its + * contents using libmagic(3) from the "file" package. A fallback for + * files which mod_mime cannot type by extension, in the style of + * mod_mime_magic, but using the system magic database and matching + * engine rather than a private copy. + * + * A magic_t handle is opened, loaded and closed for each request: all + * libmagic's mutable state lives in the handle, so this is safe from + * any number of threads, and loading a compiled (.mgc) database is an + * mmap() of a file already in the page cache. + */ + +#include "apr.h" +#include "apr_strings.h" +#include "apr_lib.h" + +#include + +#include "ap_config.h" +#include "httpd.h" +#include "http_config.h" +#include "http_core.h" +#include "http_log.h" +#include "http_protocol.h" +#include "http_request.h" + +#include + +module AP_MODULE_DECLARE_DATA mime_libmagic_module; + +#define UNSET (-1) +#define DEFAULT_BYTES 16384 +#define MAX_BYTES 1048576 + +typedef struct { + int enabled; /* MimeLibmagic On|Off */ + const char *magicfile; /* MimeLibmagicFile, or the library default */ + apr_int64_t bytes; /* MimeLibmagicBytes */ + int charset; /* MimeLibmagicCharset On|Off */ +} libmagic_server_conf; + +static void *create_server_conf(apr_pool_t *p, server_rec *s) +{ + libmagic_server_conf *conf = apr_pcalloc(p, sizeof *conf); + + conf->enabled = UNSET; + conf->magicfile = NULL; + conf->bytes = UNSET; + conf->charset = UNSET; + return conf; +} + +static void *merge_server_conf(apr_pool_t *p, void *basev, void *addv) +{ + libmagic_server_conf *base = basev, *add = addv; + libmagic_server_conf *conf = apr_palloc(p, sizeof *conf); + + conf->enabled = add->enabled != UNSET ? add->enabled : base->enabled; + conf->magicfile = add->magicfile ? add->magicfile : base->magicfile; + conf->bytes = add->bytes != UNSET ? add->bytes : base->bytes; + conf->charset = add->charset != UNSET ? add->charset : base->charset; + return conf; +} + +static const char *set_enabled(cmd_parms *cmd, void *dummy, int flag) +{ + libmagic_server_conf *conf = + ap_get_module_config(cmd->server->module_config, &mime_libmagic_module); + + conf->enabled = flag; + return NULL; +} + +static const char *set_magicfile(cmd_parms *cmd, void *dummy, const char *arg) +{ + libmagic_server_conf *conf = + ap_get_module_config(cmd->server->module_config, &mime_libmagic_module); + + conf->magicfile = ap_server_root_relative(cmd->pool, arg); + if (!conf->magicfile) { + return apr_pstrcat(cmd->pool, "Invalid MimeLibmagicFile path ", arg, NULL); + } + return NULL; +} + +static const char *set_bytes(cmd_parms *cmd, void *dummy, const char *arg) +{ + libmagic_server_conf *conf = + ap_get_module_config(cmd->server->module_config, &mime_libmagic_module); + char *end; + apr_int64_t n = apr_strtoi64(arg, &end, 10); + + if (*arg == '\0' || *end != '\0' || n < 1 || n > MAX_BYTES) { + return "MimeLibmagicBytes must be between 1 and " + APR_STRINGIFY(MAX_BYTES); + } + conf->bytes = n; + return NULL; +} + +static const char *set_charset(cmd_parms *cmd, void *dummy, int flag) +{ + libmagic_server_conf *conf = + ap_get_module_config(cmd->server->module_config, &mime_libmagic_module); + + conf->charset = flag; + return NULL; +} + +/* Open a handle with flags and load the database at path. Returns + * NULL after logging the failure. */ +static magic_t open_magic(request_rec *r, int flags, const char *path) +{ + magic_t ms = magic_open(flags); + + if (!ms) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, APR_FROM_OS_ERROR(errno), r, + APLOGNO() "libmagic: could not open handle"); + return NULL; + } + + if (magic_load(ms, path) != 0) { + const char *err = magic_error(ms); + + ap_log_rerror(APLOG_MARK, APLOG_ERR, APR_FROM_OS_ERROR(magic_errno(ms)), + r, APLOGNO() "libmagic: could not load magic " + "database %s: %s", path, err ? err : "unknown error"); + magic_close(ms); + return NULL; + } + + return ms; +} + +/* Set the content type from libmagic's result string, either "type" + * or "type; charset=encoding". Returns OK or DECLINED. */ +static int set_type(request_rec *r, const char *result, int charset_on) +{ + char *type = apr_pstrdup(r->pool, result); + char *charset = NULL; + const char *slash, *p; + char *sep; + + sep = strchr(type, ';'); + if (sep) { + *sep++ = '\0'; + while (*sep == ' ') { + sep++; + } + if (strncmp(sep, "charset=", 8) == 0) { + charset = sep + 8; + } + } + + /* Some matchers append a note, e.g. "application/x-executable, no + * program header": keep the type. */ + type[strcspn(type, ", \t")] = '\0'; + + /* the type must be of the form token "/" token */ + slash = ap_scan_http_token(type); + if (slash == type || *slash != '/' + || (p = ap_scan_http_token(slash + 1)) == slash + 1 || *p != '\0') { + ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, r, APLOGNO() + "libmagic: ignoring invalid type '%s' for %s", + result, r->filename); + return DECLINED; + } + ap_content_type_tolower(type); + + /* Unknown binary content is left untyped, as mod_mime_magic does. */ + if (strcmp(type, "application/octet-stream") == 0 + || strcmp(type, "application/x-empty") == 0) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, + "libmagic: no type for %s (%s)", r->filename, result); + return DECLINED; + } + + if (charset_on && charset && strncmp(type, "text/", 5) == 0 + && strcmp(charset, "binary") != 0 + && strcmp(charset, "unknown-8bit") != 0 + && *charset && *ap_scan_http_token(charset) == '\0') { + type = apr_pstrcat(r->pool, type, "; charset=", charset, NULL); + } + + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, + "libmagic: %s is '%s' (%s)", r->filename, type, result); + ap_set_content_type_ex(r, type, 1); + return OK; +} + +static int libmagic_find_ct(request_rec *r) +{ + const libmagic_server_conf *conf; + apr_file_t *fd; + apr_status_t rv; + apr_size_t nbytes, bytes; + magic_t ms; + const char *result, *path; + char *buf; + int ret, flags; + + /* the file has to exist */ + if (r->finfo.filetype == APR_NOFILE || !r->filename) { + return DECLINED; + } + + /* was someone else already here? */ + if (r->content_type) { + return DECLINED; + } + + conf = ap_get_module_config(r->server->module_config, &mime_libmagic_module); + if (conf->enabled != 1) { + return DECLINED; + } + + switch (r->finfo.filetype) { + case APR_DIR: + ap_set_content_type_ex(r, DIR_MAGIC_TYPE, 1); + return OK; + case APR_CHR: + case APR_BLK: + case APR_PIPE: + case APR_SOCK: + ap_set_content_type_ex(r, "application/octet-stream", 1); + return OK; + case APR_REG: + break; + default: + /* a broken symlink, or something exotic: leave it to the handler */ + return DECLINED; + } + + rv = apr_file_open(&fd, r->filename, APR_READ | APR_BINARY, + APR_OS_DEFAULT, r->pool); + if (rv != APR_SUCCESS) { + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, rv, r, + "libmagic: could not open %s", r->filename); + return DECLINED; + } + + bytes = conf->bytes != UNSET ? conf->bytes : DEFAULT_BYTES; + buf = apr_palloc(r->pool, bytes); + rv = apr_file_read_full(fd, buf, bytes, &nbytes); + apr_file_close(fd); + if (rv != APR_SUCCESS && !APR_STATUS_IS_EOF(rv)) { + ap_log_rerror(APLOG_MARK, APLOG_WARNING, rv, r, APLOGNO() + "libmagic: could not read %s", r->filename); + return DECLINED; + } + + if (nbytes == 0) { + ap_set_content_type_ex(r, "text/plain", 1); + return OK; + } + + /* With no MimeLibmagicFile, the library's default database; action + * 1 means the user's ~/.magic is not consulted, unlike + * magic_load(ms, NULL). */ + path = conf->magicfile ? conf->magicfile : magic_getpath(NULL, 1); + /* Never look inside compressed files: that path spawns external + * decompressors from the calling thread. */ + flags = MAGIC_MIME_TYPE | MAGIC_NO_CHECK_COMPRESS + | (conf->charset == 1 ? MAGIC_MIME_ENCODING : 0); + ms = open_magic(r, flags, path); + if (!ms) { + return DECLINED; + } + + result = magic_buffer(ms, buf, nbytes); + if (!result) { + const char *err = magic_error(ms); + + ap_log_rerror(APLOG_MARK, APLOG_WARNING, APR_FROM_OS_ERROR(magic_errno(ms)), + r, APLOGNO() "libmagic: could not determine type " + "of %s: %s", r->filename, err ? err : "unknown error"); + ret = DECLINED; + } + else { + /* result points into the handle, so must be used before close */ + ret = set_type(r, result, conf->charset == 1); + } + + magic_close(ms); + + return ret; +} + +static const command_rec libmagic_cmds[] = { + AP_INIT_FLAG("MimeLibmagic", set_enabled, NULL, RSRC_CONF, + "Enable content type detection using libmagic"), + AP_INIT_TAKE1("MimeLibmagicFile", set_magicfile, NULL, RSRC_CONF, + "Path to the magic database, in place of the library default"), + AP_INIT_TAKE1("MimeLibmagicBytes", set_bytes, NULL, RSRC_CONF, + "Number of bytes of a file examined"), + AP_INIT_FLAG("MimeLibmagicCharset", set_charset, NULL, RSRC_CONF, + "Add a charset parameter to text types"), + {NULL} +}; + +static void register_hooks(apr_pool_t *p) +{ + static const char * const aszPre[] = { "mod_mime.c", "mod_mime_magic.c", NULL }; + + /* run after mod_mime (and mod_mime_magic, if loaded) */ + ap_hook_type_checker(libmagic_find_ct, aszPre, NULL, APR_HOOK_MIDDLE); +} + +AP_DECLARE_MODULE(mime_libmagic) = +{ + STANDARD20_MODULE_STUFF, + NULL, /* dir config creator */ + NULL, /* dir merger --- default is to override */ + create_server_conf, /* server config */ + merge_server_conf, /* merge server config */ + libmagic_cmds, /* command apr_table_t */ + register_hooks /* register hooks */ +}; diff --git a/modules/metadata/mod_mime_magic.c b/modules/metadata/mod_mime_magic.c index c15d11b7ff7..341e42323bb 100644 --- a/modules/metadata/mod_mime_magic.c +++ b/modules/metadata/mod_mime_magic.c @@ -1802,16 +1802,18 @@ static int mconvert(request_rec *r, union VALUETYPE *p, struct magic *m) return 1; case BELONG: case BEDATE: - p->l = (long) - ((p->hl[0] << 24) | (p->hl[1] << 16) | (p->hl[2] << 8) | (p->hl[3])); + p->l = (long) (apr_int32_t) + (((apr_uint32_t) p->hl[0] << 24) | ((apr_uint32_t) p->hl[1] << 16) + | (p->hl[2] << 8) | (p->hl[3])); return 1; case LESHORT: p->h = (short) ((p->hs[1] << 8) | (p->hs[0])); return 1; case LELONG: case LEDATE: - p->l = (long) - ((p->hl[3] << 24) | (p->hl[2] << 16) | (p->hl[1] << 8) | (p->hl[0])); + p->l = (long) (apr_int32_t) + (((apr_uint32_t) p->hl[3] << 24) | ((apr_uint32_t) p->hl[2] << 16) + | (p->hl[1] << 8) | (p->hl[0])); return 1; default: ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(01538) diff --git a/test/modules/metadata/__init__.py b/test/modules/metadata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/modules/metadata/env.py b/test/modules/metadata/env.py index 038d478968a..30126fcb7f9 100644 --- a/test/modules/metadata/env.py +++ b/test/modules/metadata/env.py @@ -1,6 +1,8 @@ import inspect import logging import os +import re +import subprocess from pyhttpd.env import HttpdTestEnv, HttpdTestSetup @@ -13,13 +15,25 @@ def __init__(self, env: 'HttpdTestEnv'): super().__init__(env=env) self.add_source_dir(os.path.dirname(inspect.getfile(MetadataTestSetup))) self.add_modules(["mime", "mime_magic"]) + # mod_mime_libmagic needs libmagic at build time, so it must not + # be a hard requirement; its tests skip when it is absent. + self.add_optional_modules(["mime_libmagic"]) class MetadataTestEnv(HttpdTestEnv): def __init__(self, pytestconfig=None): super().__init__(pytestconfig=pytestconfig) - self.add_httpd_log_modules(["mime_magic", "core"]) + self.add_httpd_log_modules(["mime_magic", "mime_libmagic", "core"]) def setup_httpd(self, setup: HttpdTestSetup = None): super().setup_httpd(setup=MetadataTestSetup(env=self)) + + @property + def has_libmagic_module(self) -> bool: + """Whether mod_mime_libmagic is available, as a DSO or built in.""" + if os.path.isfile(os.path.join(self.libexec_dir, 'mod_mime_libmagic.so')): + return True + p = subprocess.run([self.httpd_bin, '-l'], capture_output=True, + text=True) + return re.search(r'^\s+mod_mime_libmagic\.c$', p.stdout, re.M) is not None diff --git a/test/modules/metadata/samples.py b/test/modules/metadata/samples.py new file mode 100644 index 00000000000..310cee310f0 --- /dev/null +++ b/test/modules/metadata/samples.py @@ -0,0 +1,49 @@ +# Sample file contents for content type detection tests. Each is +# served without an extension, so mod_mime sets no type and the magic +# modules have to derive one from the content. mod_mime_magic only +# applies its magic rules to files of at least 64 bytes, so samples +# meant to match a rule are padded past that. +import gzip +import os + +SAMPLES = { + "html": b"\nhello\n\n", + "html-doctype": b"\nt" + b"hi\n", + # (no commas: libmagic 5.45 calls lines with a consistent number + # of commas text/csv) + "text": b"hello world this is plain ascii text\n" + b"with a second line of a different length\n" * 2, + "text-utf8": "café naïve résumé\n".encode() * 4, + "text-latin1": b"caf\xe9 na\xefve\n" * 4, + "csrc": b"#include \n\nint main(int argc, char **argv)\n{\n" + b" printf(\"hi\\n\");\n return 0;\n}\n", + "json": b'{"a": 1, "b": [1, 2, 3], "c": {"d": "e", "f": "some more text"}}\n', + "xml": b"\n1234\n", + "rfc822": b"From: a@example.com\nTo: b@example.com\nSubject: hi there\n\n" + b"the body of the message\n", + "shell": b"#!/bin/sh\necho hi\necho there\necho this is a shell script\n" + b"exit 0\n", + "png": b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + b"\x00\x00\x00\x10\x00\x00\x00\x10\x08\x06\x00\x00\x00\x1f\xf3\xffa" + + b"\x00" * 64, + "gif": b"GIF89a\x10\x00\x10\x00\x80\x00\x00" + b"\x00" * 64, + "pdf": b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n1 0 obj\n<< /Type /Catalog >>\nendobj\n", + "gzip": gzip.compress(bytes(range(32, 127)) * 4, mtime=0), + "elf": b"\x7fELF\x02\x01\x01\x00" + b"\x00" * 8 + + b"\x02\x00\x3e\x00\x01\x00\x00\x00" + b"\x00" * 100, + # deterministic bytes which libmagic reports as application/octet-stream + "binary": bytes((i * 7919) % 256 for i in range(4096)), + "empty": b"", +} + + +def write_samples(doc_dir): + """Write every sample into doc_dir, plus png.txt (PNG content + with a .txt extension).""" + os.makedirs(doc_dir, exist_ok=True) + for name, content in SAMPLES.items(): + with open(os.path.join(doc_dir, name), "wb") as f: + f.write(content) + with open(os.path.join(doc_dir, "png.txt"), "wb") as f: + f.write(SAMPLES["png"]) diff --git a/test/modules/metadata/test_002_mime_libmagic.py b/test/modules/metadata/test_002_mime_libmagic.py new file mode 100644 index 00000000000..8ba335fb7cf --- /dev/null +++ b/test/modules/metadata/test_002_mime_libmagic.py @@ -0,0 +1,89 @@ +import os +import pytest + +from pyhttpd.conf import HttpdConf +from .samples import write_samples + + +def get_type(env, path): + """The Content-Type header of GET path on test1, or None.""" + r = env.curl_get(env.mkurl("http", "test1", path)) + assert r.response, "no response: server may have crashed" + assert r.response["status"] == 200 + assert "content-encoding" not in r.response["header"] + return r.response["header"].get("content-type") + + +class TestMimeLibmagic: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + if not env.has_libmagic_module: + pytest.skip("mod_mime_libmagic is not built") + write_samples(os.path.join(env.server_dir, "htdocs", "test1", "libmagic")) + conf = HttpdConf(env, extras={ + 'base': "MimeLibmagic On", + }) + conf.add_vhost_test1() + conf.install() + assert env.apache_restart() == 0 + + @pytest.mark.parametrize(["name", "ctype"], [ + ("html", "text/html"), + ("html-doctype", "text/html"), + ("text", "text/plain"), + ("text-utf8", "text/plain"), + ("text-latin1", "text/plain"), + ("csrc", "text/x-c"), + ("json", "application/json"), + ("xml", "text/xml"), + ("rfc822", "message/rfc822"), + ("shell", "text/x-shellscript"), + ("png", "image/png"), + ("gif", "image/gif"), + ("pdf", "application/pdf"), + # no decompression support, the compressed file itself is typed + ("gzip", "application/gzip"), + # libmagic appends ", no program header" to the type + ("elf", "application/x-executable"), + # application/octet-stream from libmagic leaves the type unset + ("binary", None), + ("empty", "text/plain"), + ]) + def test_metadata_002_01_types(self, env, name, ctype): + assert get_type(env, f"/libmagic/{name}") == ctype + + # mod_mime's extension mapping wins over the content + def test_metadata_002_02_extension(self, env): + assert get_type(env, "/libmagic/png.txt") == "text/plain" + + +class TestMimeLibmagicCharset: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + if not env.has_libmagic_module: + pytest.skip("mod_mime_libmagic is not built") + write_samples(os.path.join(env.server_dir, "htdocs", "test1", "libmagic")) + conf = HttpdConf(env, extras={ + 'base': """ + MimeLibmagic On + MimeLibmagicCharset On + """, + }) + conf.add_vhost_test1() + conf.install() + assert env.apache_restart() == 0 + + @pytest.mark.parametrize(["name", "ctype"], [ + ("html", "text/html; charset=us-ascii"), + ("text", "text/plain; charset=us-ascii"), + ("text-utf8", "text/plain; charset=utf-8"), + ("text-latin1", "text/plain; charset=iso-8859-1"), + # only text types get a charset + ("json", "application/json"), + ("png", "image/png"), + ("binary", None), + ]) + def test_metadata_002_10_charset(self, env, name, ctype): + assert get_type(env, f"/libmagic/{name}") == ctype diff --git a/test/modules/metadata/test_003_compare.py b/test/modules/metadata/test_003_compare.py new file mode 100644 index 00000000000..cdc48c44f9b --- /dev/null +++ b/test/modules/metadata/test_003_compare.py @@ -0,0 +1,85 @@ +import logging +import os +import pytest + +from pyhttpd.conf import HttpdConf +from .samples import SAMPLES, write_samples + +log = logging.getLogger(__name__) + +# (sample, mod_mime_magic type, mod_mime_magic encoding, mod_mime_libmagic type) +# mod_mime_magic uses the stock conf/magic and its token-based text +# detection, which knows nothing of most of these, so many are untyped. +EXPECTED = [ + ("html", "text/html", None, "text/html"), + ("html-doctype", None, None, "text/html"), + ("text", None, None, "text/plain"), + ("text-utf8", None, None, "text/plain"), + ("text-latin1", None, None, "text/plain"), + ("csrc", "text/plain", None, "text/x-c"), + ("json", None, None, "application/json"), + ("xml", "text/xml", None, "text/xml"), + ("rfc822", "message/rfc822", "7bit", "message/rfc822"), + ("shell", None, None, "text/x-shellscript"), + ("png", "image/png", None, "image/png"), + ("gif", "image/gif", None, "image/gif"), + ("pdf", None, None, "application/pdf"), + ("gzip", "application/octet-stream", "x-gzip", "application/gzip"), + ("elf", None, None, "application/x-executable"), + ("binary", None, None, None), + ("empty", "text/plain", None, "text/plain"), + ("png.txt", "text/plain", None, "text/plain"), +] + + +def get_headers(env, host, path): + r = env.curl_get(env.mkurl("http", host, path)) + assert r.response, "no response: server may have crashed" + assert r.response["status"] == 200, r.response + h = r.response["header"] + return h.get("content-type"), h.get("content-encoding") + + +class TestCompare: + """The same files served by mod_mime_magic (vhost test1) and by + mod_mime_libmagic (vhost test2).""" + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + if not env.has_libmagic_module: + pytest.skip("mod_mime_libmagic is not built") + for vhost in ["test1", "test2"]: + write_samples(os.path.join(env.server_dir, "htdocs", vhost, "cmp")) + # No MimeMagicFile in the main server, so test2 inherits no + # magic rules and only mod_mime_libmagic runs there. + conf = HttpdConf(env, extras={ + f"test1.{env.http_tld}": f'MimeMagicFile "{env.prefix}/conf/magic"', + f"test2.{env.http_tld}": "MimeLibmagic On", + }) + conf.add_vhost_test1() + conf.add_vhost_test2() + conf.install() + assert env.apache_restart() == 0 + + @pytest.mark.parametrize(["name", "mm_type", "mm_enc", "lm_type"], EXPECTED) + def test_metadata_003_01_mime_magic(self, env, name, mm_type, mm_enc, lm_type): + assert get_headers(env, "test1", f"/cmp/{name}") == (mm_type, mm_enc) + + @pytest.mark.parametrize(["name", "mm_type", "mm_enc", "lm_type"], EXPECTED) + def test_metadata_003_02_libmagic(self, env, name, mm_type, mm_enc, lm_type): + assert get_headers(env, "test2", f"/cmp/{name}") == (lm_type, None) + + # Log a side-by-side table of what the two modules actually report. + def test_metadata_003_03_report(self, env): + rows = [("sample", "mod_mime_magic", "mod_mime_libmagic")] + for name in list(SAMPLES) + ["png.txt"]: + mm = get_headers(env, "test1", f"/cmp/{name}") + lm = get_headers(env, "test2", f"/cmp/{name}") + fmt = lambda t: f"{t[0]}" + (f" ({t[1]})" if t[1] else "") + rows.append((name, fmt(mm), fmt(lm))) + widths = [max(len(r[i]) for r in rows) for i in range(3)] + table = "\n".join(" ".join(c.ljust(widths[i]) for i, c in enumerate(r)) + for r in rows) + log.info("content types reported:\n%s", table) + with open(os.path.join(env.gen_dir, "mime-compare.txt"), "w") as f: + f.write(table + "\n")