diff --git a/docs/modules/ROOT/pages/advanced/setting-limits.adoc b/docs/modules/ROOT/pages/advanced/setting-limits.adoc index bb0f0588f0..4ca3c3d36c 100644 --- a/docs/modules/ROOT/pages/advanced/setting-limits.adoc +++ b/docs/modules/ROOT/pages/advanced/setting-limits.adoc @@ -52,7 +52,8 @@ This is the same configuration tested in `AllLimitsTest.java`: "zipBombRatio": 100 }, "timeout-limits": { - "taskTimeoutMillis": 60000 + "totalTaskTimeoutMillis": 3600000, + "progressTimeoutMillis": 60000 }, "standard-metadata-limiter-factory": { "maxTotalBytes": 1048576, @@ -243,7 +244,9 @@ See test: `tika-serialization/src/test/java/org/apache/tika/config/OutputLimitsT == Timeout Limits -The `TimeoutLimits` class controls time-based limits for parsing operations. +The `TimeoutLimits` class controls time-based limits for parsing operations. Tika 4.x +uses a *two-tier* timeout: one bound on total wall-clock time, and one on time elapsed +since the parser last reported progress. === Configuration Options @@ -251,11 +254,52 @@ The `TimeoutLimits` class controls time-based limits for parsing operations. |=== |Setting |Default |Description -|`taskTimeoutMillis` -|60000 (1 minute) -|Maximum time in milliseconds for a parse operation to complete. +|`totalTaskTimeoutMillis` +|3600000 (1 hour) +|Maximum wall-clock time in milliseconds for the entire parse task. + +|`progressTimeoutMillis` +|120000 (2 minutes) +|Maximum time in milliseconds since the parser last reported progress. Catches +infinite loops and hung processes. |=== +[IMPORTANT] +==== +Which bound actually applies depends on whether the parser reports progress. +A parser that never calls `TikaProgressTracker.update()` never advances the +timer, so it effectively gets `progressTimeoutMillis` as its total timeout — +matching the single-timeout behavior of earlier versions. + +In practice only long-running parsers report progress: `TesseractOCRParser`, +`Tess4JParser`, `ExternalParser`, `GDALParser`, the VLM and image-embedding +parsers, and `StringsParser`. Everything else — including container parsing and +embedded-document recursion — does not. + +Those parsers report progress *after* each external-process invocation +completes, not while one is running. So a document that needs many OCR calls +can extend well past `progressTimeoutMillis`, because each finished page resets +the timer — but a *single* call that runs longer than `progressTimeoutMillis` +is still cut short. + +Because of this, `progressTimeoutMillis` also caps how long any single external +process may run. Parsers that spawn processes size their own timeout via +`TimeoutLimits.getProcessTimeoutMillis(context, ...)`, which never allows a +value beyond `progressTimeoutMillis`, so the process is stopped just before the +progress watchdog would fire. + +The shipped defaults are aligned: `progressTimeoutMillis` is 120 seconds and +the bundled process-spawning parsers (OCR, strings, inference) each default to +a 120-second per-process timeout, so those defaults are reachable. **If you +raise a per-process timeout above 120 seconds, raise `progressTimeoutMillis` +to match** — raising the parser's own timeout alone has no effect. + +For most documents — anything without one of the parsers above in the chain — +the effective ceiling is `progressTimeoutMillis`, not `totalTaskTimeoutMillis`. +Lower `totalTaskTimeoutMillis` if you need a hard ceiling on OCR-heavy or +external-process work regardless of progress. +==== + === JSON Configuration [source,json] @@ -263,7 +307,8 @@ The `TimeoutLimits` class controls time-based limits for parsing operations. { "parse-context": { "timeout-limits": { - "taskTimeoutMillis": 120000 + "totalTaskTimeoutMillis": 7200000, + "progressTimeoutMillis": 120000 } } } @@ -275,7 +320,7 @@ Configuration file: `tika-serialization/src/test/resources/configs/timeout-limit [source,java] ---- -TimeoutLimits limits = new TimeoutLimits(120000); +TimeoutLimits limits = new TimeoutLimits(7200000, 120000); context.set(TimeoutLimits.class, limits); // Helper method diff --git a/docs/modules/ROOT/pages/developers/serialization.adoc b/docs/modules/ROOT/pages/developers/serialization.adoc index 122589731d..d8477966b3 100644 --- a/docs/modules/ROOT/pages/developers/serialization.adoc +++ b/docs/modules/ROOT/pages/developers/serialization.adoc @@ -241,6 +241,26 @@ The serialization system implements a security allowlist: This prevents attacks where malicious JSON specifies dangerous classes for instantiation. +[IMPORTANT] +==== +The allowlist governs *which components may be instantiated* from JSON. It does +not restrict *how an already-loaded component may be configured*. + +Self-configuring components — which includes every `Parser`, since `Parser` +extends `SelfConfiguring` — are skipped by the wire-block scan +(`ParseContextDeserializer.assertNoBlockedComponents`): their config subtree is +passed through to the component unexamined. So while a request cannot bind a new +`Parser` from the wire, a request carrying +`{"parse-context": {"pdf-parser": {"ocr": {"strategy": "OCR_AND_TEXT_EXTRACTION"}}}}` +will reach `PDFParser` and take effect. + +That is why per-request configuration is gated separately by +`allowPerRequestConfig`, which is off by default. Treat "the caller may supply +per-request config" as equivalent to "the caller may set any parser option, +including options that spawn external processes such as OCR" — not as something +the allowlist constrains. +==== + [source,java] ---- // This will FAIL - class not registered diff --git a/docs/modules/ROOT/pages/migration-to-4x/design-notes-4x.adoc b/docs/modules/ROOT/pages/migration-to-4x/design-notes-4x.adoc index 7913fafefb..f6053f808d 100644 --- a/docs/modules/ROOT/pages/migration-to-4x/design-notes-4x.adoc +++ b/docs/modules/ROOT/pages/migration-to-4x/design-notes-4x.adoc @@ -74,6 +74,14 @@ may be bound from the wire; `Parser`, `Detector`, `Renderer`, and similar are blocked before anything is constructed. See xref:developers/serialization.adoc[Serialization and Configuration]. +Note the boundary: the allowlist blocks *binding a component* from the wire, not +*configuring one that is already loaded*. Self-configuring components — every +`Parser` among them — have their config subtree passed through unscanned, so a +per-request config can still set parser options (including ones that spawn +external processes, such as OCR). This is why `allowPerRequestConfig` is a +separate gate and is off by default; the allowlist alone does not make +per-request configuration safe to expose. + === Implementation Challenges * Converted code to true Java beans with matching getters/setters diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc index fad6b193e4..3eda1ecd1d 100644 --- a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc +++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc @@ -135,6 +135,31 @@ for error text should check `tk:exception:container-exception` (full-object endpoints) or the `422` body (`/meta/\{field}`, populated only when `returnStackTrace=true`). +Two changes to the returned metadata come with this, neither of which produces an +error: + +* **`/meta` no longer returns a `language` field.** Language detection previously ran + inline on this endpoint via a dedicated content handler that buffered text solely to + detect the language, which meant holding the document text twice to populate one + field. That handler was removed. `/meta` deliberately parses with the `ignore` + content handler, so there is no text for a language detector to work from. ++ +**Migration:** configure a language-detection metadata filter +(`charsoup-metadata-filter`, `optimaize`, or `opennlp`) and use `/rmeta` or +`/tika/json`, which capture content. The detected value arrives as +`tk:detected-language`, with `tk:detected-language-confidence`. Note that these +filters read `tk:content`, so they are no-ops on `/meta` and on any endpoint +configured with the `ignore` handler. + +* **`/meta` now sets `tk:exception:embedded-depth-limit-reached` on any document + with embedded content.** `/meta` suppresses embedded parsing by setting an embedded + depth limit of `0`, and reaching a limit is recorded. The previous implementation + suppressed embedded documents by a different mechanism that recorded nothing. The + flag is expected on this endpoint and does not indicate a truncated result. ++ +**Migration:** clients that alert on the presence of any `tk:exception:*` key should +exclude this one for `/meta`. + === Accept Header Routing Removed The `/tika` endpoint no longer routes based on `Accept` headers. Use explicit paths instead: @@ -157,7 +182,24 @@ The following `TikaServerConfig` options have been removed: === `/pipes` and `/async` Require `allowPipes`; Per-Request Config Requires `allowPerRequestConfig` -Previously these endpoints (and per-request parser configuration) were enabled simply by listing endpoints under `server.endpoints`. The capabilities are now split into two default-`false` flags in the `server` section: +This replaces the `enableUnsecureFeatures` flag that alpha-1 briefly used, and before +that, enabling these capabilities simply by listing endpoints under `server.endpoints`. +`enableUnsecureFeatures` no longer exists: a config that still carries it fails to start +with an "Unrecognized field" error naming the key, rather than silently ignoring it. +The single flag has been split into two, so that granting batch/fetcher access and +granting per-request parser configuration are separate decisions: + +|=== +|Was |Now + +|`enableUnsecureFeatures: true` (to use `/pipes` or `/async`) +|`allowPipes: true` + +|`enableUnsecureFeatures: true` (to send per-request config) +|`allowPerRequestConfig: true` +|=== + +The capabilities are two default-`false` flags in the `server` section: * `allowPipes` gates the `/pipes` and `/async` endpoints, which drive process-isolated batch parsing through your fetchers and emitters. Selecting either without `allowPipes` causes the server to refuse to start with a clear error. * `allowPerRequestConfig` gates per-request parser configuration: the `/config` family of endpoints and the multipart `config` part. When off, such requests are rejected with 403. @@ -189,7 +231,7 @@ All tika-server configurations must now include a `pipes` section and a `file-sy "fetchers": { "file-system-fetcher": { "file-system-fetcher": { - "allowAbsolutePaths": true + "basePath": "/path/to/your/input" } } }, @@ -205,6 +247,23 @@ All tika-server configurations must now include a `pipes` section and a `file-sy } ---- +[IMPORTANT] +==== +Set `basePath` to a directory that contains only the documents you intend the +server to read. It is the filesystem sandbox: the fetcher rejects any fetch key +that resolves outside it, including absolute paths and `../` traversal, and +re-checks after resolving symlinks. + +Setting `allowAbsolutePaths` instead of `basePath` turns that sandbox off +entirely — fetch keys are then used as raw absolute paths, so any caller who can +reach `/pipes` can read any file the server process can read. The matching +emitter setting is worse: it grants arbitrary file *write*. `allowAbsolutePaths` +is not a relaxation of `basePath`; it is what you get when there is no +`basePath` at all, and it is a no-op when `basePath` is set. Use it only if you +genuinely intend an unsandboxed fetcher and have restricted access to the server +by other means. +==== + [IMPORTANT] ==== `numClients` is not boilerplate to copy unchanged from this example. In 3.x, diff --git a/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc b/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc index 034a5d7b93..2a13b5d068 100644 --- a/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc +++ b/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc @@ -81,8 +81,8 @@ The outer key (`fsf`) is the fetcher ID — referenced by `pipesIterator.fetcher |Field |Default |Description |`basePath` -|_required_ -|Base directory for fetch operations. Fetch keys are resolved relative to this path. +|_none_ +|Base directory for fetch operations. Fetch keys are resolved relative to this path and must stay inside it. Not technically required, but omitting it disables containment entirely — see `allowAbsolutePaths` below and <>. |`extractFileSystemMetadata` |`false` @@ -90,7 +90,7 @@ The outer key (`fsf`) is the fetcher ID — referenced by `pipesIterator.fetcher |`allowAbsolutePaths` |`false` -|When `true`, fetch keys may be absolute paths and `basePath` may be omitted. Use sparingly — see <>. +|Permission to run *without* a `basePath`. It is not a relaxation of `basePath` — see <>. |=== [#file-system-emitter] @@ -253,6 +253,7 @@ Tradeoffs: [#security-notes] == Security Notes -* **`basePath` is a sandbox boundary.** The fetcher and emitter reject fetch/emit keys that resolve outside `basePath`. Do not set `allowAbsolutePaths=true` unless the source of fetch keys is fully trusted — an attacker-controlled fetch key could otherwise read arbitrary files. -* **Symlinks are followed.** A symlink under `basePath` pointing outside `basePath` may still be readable. If you need strict containment, do not allow symlinks in your input tree. +* **`basePath` is the sandbox boundary, and it is the only one.** With `basePath` set, the fetcher and emitter reject any key that resolves outside it, including absolute paths and `../` traversal. `allowAbsolutePaths` has no effect in this state. +* **Without `basePath` there is no containment at all.** The key is used as a raw absolute path, and the containment checks are skipped entirely. `allowAbsolutePaths=true` is how you assert that you intend this; it is a switch between two states, not a dial that loosens `basePath`. For the fetcher this means any file the process can read; for the emitter, any file it can write. Use it only when fetch/emit keys come from a fully trusted source and access to the service is restricted by other means. +* **Symlink containment differs between fetcher and emitter.** The fetcher re-checks with `toRealPath()`, so a symlink under `basePath` pointing outside it is rejected. The emitter does not: it checks only the normalized path, so a symlink already present under its `basePath` can be written through. Do not rely on symlinks being contained on the emit side. * **Output directories are created automatically.** The emitter creates intermediate directories as needed. Make sure the process's umask is appropriate for the data being written. diff --git a/docs/modules/ROOT/pages/pipes/plugins/http.adoc b/docs/modules/ROOT/pages/pipes/plugins/http.adoc index d60a006243..9acab6fbce 100644 --- a/docs/modules/ROOT/pages/pipes/plugins/http.adoc +++ b/docs/modules/ROOT/pages/pipes/plugins/http.adoc @@ -92,7 +92,7 @@ include::example$pipes-http-fetcher.json[] |`maxRedirects` |`0` -|Maximum number of redirects to follow. `0` means follow none. +|Maximum number of redirects to follow. `0` means follow none. Not applied to range requests — see <>. |`maxSpoolSize` |`-1` @@ -123,6 +123,38 @@ include::example$pipes-http-fetcher.json[] |Base64-encoded private key for asymmetric (RSA/ECDSA) JWT signing. Mutually exclusive with `jwtSecret`. |=== +[#security-notes] +== Security Notes + +This fetcher makes the server issue HTTP requests to a URL supplied as the fetch key. +That is a server-side request forgery primitive by design, and it is not constrained by +this plugin. Treat the source of fetch keys as fully trusted, and restrict access to any +endpoint that can reach it (`/pipes`, `/async`). + +* **The fetch key is used as the URL with no validation.** It is passed straight to + `new HttpGet(fetchKey)`. There is no scheme allowlist, no host denylist, and no check + against loopback, link-local, or RFC1918 addresses. A fetch key of + `http://169.254.169.254/...` reaches a cloud metadata endpoint like any other URL. The + resolved address is recorded in metadata *after* the fetch, not consulted before it. + Schemes other than `http`/`https` fail only because no other scheme is registered in + the connection manager — that is a side effect of the transport setup, not a check. + +* **TLS certificates are not verified, and this is not configurable here.** The + underlying client defaults to `verifySsl=false`, which installs an accept-everything + trust strategy and `NoopHostnameVerifier`. `HttpFetcherConfig` exposes no `verifySsl` + setting, so an http-fetcher config cannot turn verification on. Do not use this fetcher + to retrieve anything whose authenticity matters over an untrusted network. + +* **`maxRedirects` does not apply to range requests.** The main `fetch` builds a + `RequestConfig` from `maxRedirects`; the `startRange`/`endRange` overload sets no + request config at all and therefore uses the client's defaults (redirects enabled). + A `maxRedirects: 0` setting does not stop redirects on a range fetch. + +* **The redirect host allowlist is currently inert.** `CustomRedirectStrategy` will + refuse a redirect to a host outside `allowedHostsForRedirect`, but that set is never + populated from any configuration path, and the check is skipped when the set is empty. + Do not rely on it to contain redirects. + [#notes] == Notes diff --git a/docs/modules/ROOT/pages/pipes/timeouts.adoc b/docs/modules/ROOT/pages/pipes/timeouts.adoc index 1cdddd7746..ddbaee8dee 100644 --- a/docs/modules/ROOT/pages/pipes/timeouts.adoc +++ b/docs/modules/ROOT/pages/pipes/timeouts.adoc @@ -23,7 +23,9 @@ Tika Pipes uses a two-tier timeout system to handle both long-running tasks and * **`progressTimeoutMillis`** -- Maximum time between progress updates. If no progress is reported within this interval, the task is considered stalled and killed. - Default: `60000` (1 minute). + Default: `120000` (2 minutes). + This also caps how long any single external process (OCR, `ExternalParser`, VLM) + may run, since those parsers report progress only once a process completes. * **`totalTaskTimeoutMillis`** -- Maximum wall-clock time for an entire task. Even if the parser is making progress, the task is killed after this time. diff --git a/docs/modules/ROOT/pages/security.adoc b/docs/modules/ROOT/pages/security.adoc index 044c221a3d..c2b043225b 100644 --- a/docs/modules/ROOT/pages/security.adoc +++ b/docs/modules/ROOT/pages/security.adoc @@ -56,8 +56,9 @@ and `allowComponentManagement` — the latter lets clients add, modify, and dele and read back stored configs, which can contain secrets — are off by default. Run it only behind network controls and, ideally, mutual TLS. See xref:using-tika/grpc/index.adoc[Tika gRPC]. -For the upgrade from the former `enableUnsecureFeatures` flag, see -xref:migration-to-4x/migrating-tika-server-4x.adoc[Migrating tika-server to 4.x]. +For the upgrade from the former `enableUnsecureFeatures` flag, which is now split into +`allowPipes` and `allowPerRequestConfig`, see +xref:migration-to-4x/migrating-tika-server-4x.adoc#_pipes_and_async_require_allowpipes_per_request_config_requires_allowperrequestconfig[Migrating tika-server to 4.x]. == Known Vulnerabilities diff --git a/tika-core/src/main/java/org/apache/tika/config/TimeoutLimits.java b/tika-core/src/main/java/org/apache/tika/config/TimeoutLimits.java index 34fca0a13c..36e880c7ba 100644 --- a/tika-core/src/main/java/org/apache/tika/config/TimeoutLimits.java +++ b/tika-core/src/main/java/org/apache/tika/config/TimeoutLimits.java @@ -28,7 +28,7 @@ *
  • {@code totalTaskTimeoutMillis} — bounds entire task wall-clock time * (default: 3,600,000 ms = 1 hour)
    • *
  • {@code progressTimeoutMillis} — bounds time since the last progress update; - * catches infinite loops and hung processes (default: 60,000 ms = 1 minute)
    • + * catches infinite loops and hung processes (default: 120,000 ms = 2 minutes) * *

    * Parsers that never call {@link TikaProgressTracker#update()} effectively get @@ -56,7 +56,14 @@ public class TimeoutLimits implements Serializable { private static final long serialVersionUID = 2L; public static final long DEFAULT_TOTAL_TASK_TIMEOUT_MILLIS = 3_600_000L; - public static final long DEFAULT_PROGRESS_TIMEOUT_MILLIS = 60_000L; + + /** + * Also caps how long a single external process may run (see + * {@link #getProcessTimeoutMillis(ParseContext, long)}), so this must not be + * shorter than the per-process timeouts the bundled process-spawning parsers + * default to, or those defaults become unreachable. + */ + public static final long DEFAULT_PROGRESS_TIMEOUT_MILLIS = 120_000L; private long totalTaskTimeoutMillis = DEFAULT_TOTAL_TASK_TIMEOUT_MILLIS; private long progressTimeoutMillis = DEFAULT_PROGRESS_TIMEOUT_MILLIS; @@ -133,13 +140,19 @@ public static TimeoutLimits get(ParseContext context) { /** * Returns the per-process timeout to use for external process execution. *

    - * This checks for {@link TimeoutLimits} in the ParseContext and returns - * {@code max(0, progressTimeoutMillis - 100)} to give the monitoring loop - * a small window to detect the timeout before the process itself times out. - * Falls back to {@code defaultMs} if no TimeoutLimits is found. + * External processes must not outlive the progress watchdog: a parser only + * reports progress once its process has finished, so a process allowed to run + * past {@code progressTimeoutMillis} would be killed as a hang. This caps the + * caller's timeout at {@code progressTimeoutMillis - 100}, leaving the monitoring + * loop a small window to observe the process exit first. + *

    + * The cap is a ceiling, not a replacement: a caller asking for less than the cap + * keeps its own shorter value. Falls back to {@code defaultMs} when no + * TimeoutLimits is in the context. * * @param context the ParseContext (may be null) - * @param defaultMs default timeout if no TimeoutLimits in context + * @param defaultMs the caller's configured timeout; also used if no TimeoutLimits + * is in the context * @return timeout in milliseconds for external process execution */ public static long getProcessTimeoutMillis(ParseContext context, long defaultMs) { @@ -150,7 +163,7 @@ public static long getProcessTimeoutMillis(ParseContext context, long defaultMs) if (limits == null) { return defaultMs; } - return Math.max(0, limits.progressTimeoutMillis - 100); + return Math.max(0, Math.min(defaultMs, limits.progressTimeoutMillis - 100)); } @Override diff --git a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/VLMOCRConfig.java b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/VLMOCRConfig.java index 671851596d..0f683dec99 100644 --- a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/VLMOCRConfig.java +++ b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/VLMOCRConfig.java @@ -65,7 +65,7 @@ public class VLMOCRConfig implements Serializable { * HTTP timeout in seconds for the chat completions request. * VLM inference can be slow; default is generous. */ - private int timeoutSeconds = 300; + private int timeoutSeconds = 120; /** Optional API key for authenticated endpoints. Empty means no auth. */ private String apiKey = ""; diff --git a/tika-serialization/src/test/java/org/apache/tika/config/TimeoutLimitsTest.java b/tika-serialization/src/test/java/org/apache/tika/config/TimeoutLimitsTest.java index d82f040fad..5d05fd7eca 100644 --- a/tika-serialization/src/test/java/org/apache/tika/config/TimeoutLimitsTest.java +++ b/tika-serialization/src/test/java/org/apache/tika/config/TimeoutLimitsTest.java @@ -55,7 +55,7 @@ public void testLoadIntoParseContext() throws Exception { public void testDefaults() { TimeoutLimits limits = new TimeoutLimits(); assertEquals(TimeoutLimits.DEFAULT_PROGRESS_TIMEOUT_MILLIS, limits.getProgressTimeoutMillis()); - assertEquals(60000, limits.getProgressTimeoutMillis()); + assertEquals(120000, limits.getProgressTimeoutMillis()); assertEquals(TimeoutLimits.DEFAULT_TOTAL_TASK_TIMEOUT_MILLIS, limits.getTotalTaskTimeoutMillis()); assertEquals(3600000, limits.getTotalTaskTimeoutMillis()); } @@ -91,10 +91,15 @@ public void testGetProcessTimeoutMillis() { ParseContext context = new ParseContext(); assertEquals(5000, TimeoutLimits.getProcessTimeoutMillis(context, 5000)); - // Test with context that has TimeoutLimits + // progressTimeoutMillis is a ceiling, not a replacement: a caller asking for + // less than the cap keeps its own shorter value TimeoutLimits limits = new TimeoutLimits(3600000, 60000); context.set(TimeoutLimits.class, limits); - assertEquals(59900, TimeoutLimits.getProcessTimeoutMillis(context, 5000)); + assertEquals(5000, TimeoutLimits.getProcessTimeoutMillis(context, 5000)); + + // a caller asking for more than the cap is capped just under it, so the + // process exits before the progress watchdog fires + assertEquals(59900, TimeoutLimits.getProcessTimeoutMillis(context, 300000)); // Test with very small progress timeout TimeoutLimits smallLimits = new TimeoutLimits(3600000, 50); diff --git a/tika-server/README.md b/tika-server/README.md index 3cb3676ee9..591a421754 100644 --- a/tika-server/README.md +++ b/tika-server/README.md @@ -26,23 +26,32 @@ Running $ java -jar tika-server/target/tika-server.jar --help usage: tikaserver -?,--help this help message - -h,--host host name (default = localhost) - -l,--log request URI log level ('debug' or 'info') + -c,--config tika-config file + -h,--host host name (default = localhost, use * for all) + -i,--id id to use for the server in the status endpoint and logging -p,--port listen port (default = 9998) - -s,--includeStack whether or not to return a stack trace - if there is an exception during 'parse' ``` +Everything beyond host, port and id is configured in the tika-config JSON file +passed with `-c`, not on the command line. + Running via Docker ------------------ Assuming you have Docker installed, you can use a prebuilt image: -`docker run -d -p 9998:9998 apache/tika` +`docker run -d -p 127.0.0.1:9998:9998 apache/tika` This will load Apache Tika Server and expose its interface on: `http://localhost:9998` +Note the `127.0.0.1:` prefix. Unlike the jar, which binds `localhost` by default, +the Docker images start the server with `-h 0.0.0.0`, so publishing the port +without an explicit interface exposes it on every interface of the host. +tika-server performs no authentication and parses untrusted files; only expose it +on a trusted, access-controlled network. See the +[Tika Security Model](https://tika.apache.org/security-model.html). + You may also be interested in the https://github.com/apache/tika-docker project which provides prebuilt Docker images. @@ -64,14 +73,17 @@ Usage ----- Usage examples from command line with `curl` utility: -* Extract plain text: +* Extract XHTML: `curl -T price.xls http://localhost:9998/tika` -* Extract text with mime-type hint: +* Extract plain text: +`curl -T price.xls http://localhost:9998/tika/text` + +* Extract XHTML with mime-type hint: `curl -v -H "Content-type: application/vnd.openxmlformats-officedocument.wordprocessingml.document" -T document.docx http://localhost:9998/tika` * Get all document attachments as ZIP-file: -`curl -v -T Doc1_ole.doc http://localhost:9998/unpacker > /var/tmp/x.zip` +`curl -v -T Doc1_ole.doc http://localhost:9998/unpack > /var/tmp/x.zip` * Extract metadata to CSV format: `curl -T price.xls http://localhost:9998/meta` @@ -84,6 +96,9 @@ HTTP Return Codes ----------------- `200` - Ok `204` - No content (for example when we are unpacking file without attachments) +`403` - Forbidden (per-request configuration was supplied but `allowPerRequestConfig` is off) `415` - Unknown file type `422` - Unparsable document of known type (password protected documents and unsupported versions like Biff5 Excel) +`429` - Too many requests (all forked workers were busy for longer than `maxWaitForClientMillis`; retry with backoff) `500` - Internal error +`503` - Service unavailable (the forked worker hit a timeout, ran out of memory, or crashed) diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java index ef4edd21d3..498d98ec60 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java @@ -47,7 +47,12 @@ */ @Path("/") public class TikaWelcome { - private static final String DOCS_URL = "https://wiki.apache.org/tika/TikaJAXRS"; + private static final String DOCS_URL = + "https://cwiki.apache.org/confluence/display/TIKA/TikaJAXRS"; + + /** Matches {@code {name : regex}} in a JAX-RS path, capturing the parameter name. */ + private static final Pattern PATH_TEMPLATE_REGEX = + Pattern.compile("\\{\\s*(\\w+)\\s*:[^}]*}"); private static final Map, String> HTTP_METHODS = new HashMap<>(); @@ -164,13 +169,21 @@ public String getWelcomeHTML() { h.append("

      \n"); for (Endpoint e : identifyEndpoints()) { + String displayPath = simplifyPathTemplate(e.path); h.append("
    • "); h.append(e.httpMethod); - h.append(" "); - h.append(e.path); - h.append("
      "); + h.append(" "); + // Only linkify concrete paths; one with a {param} is not fetchable as written. + if (displayPath.indexOf('{') < 0) { + h.append(""); + h.append(displayPath); + h.append(""); + } else { + h.append(displayPath); + } + h.append("
      "); h.append("Class: "); h.append(e.className); h.append("
      Method: "); @@ -213,6 +226,17 @@ public String getWelcomePlain() { return text.toString(); } + /** + * Strips the regex from a JAX-RS path template so {@code /rmeta/{handler : (\w+)?}} + * renders as {@code /rmeta/{handler}}. + */ + static String simplifyPathTemplate(String path) { + if (path == null || path.indexOf('{') < 0) { + return path; + } + return PATH_TEMPLATE_REGEX.matcher(path).replaceAll("{$1}"); + } + protected static class Endpoint { public final String className; public final String methodName; diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/ConfigExamplesTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/ConfigExamplesTest.java index 43dd1391e3..7d8fb58495 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/ConfigExamplesTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/ConfigExamplesTest.java @@ -49,6 +49,12 @@ private void loadAndValidate(String resourceName) throws Exception { Files.writeString(configFile, json, StandardCharsets.UTF_8); TikaLoader loader = TikaLoader.load(configFile); assertNotNull(loader, "TikaLoader should not be null for: " + resourceName); + // TikaLoader.load only validates top-level keys. Deserialize the server + // section too -- otherwise an example carrying a removed server key passes + // here and still fails to start a server (FAIL_ON_UNKNOWN_PROPERTIES). + TikaServerConfig serverConfig = + loader.getConfig().deserialize("server", TikaServerConfig.class); + assertNotNull(serverConfig, "'server' section should deserialize for: " + resourceName); } } diff --git a/tika-server/tika-server-core/src/test/resources/config-examples/server-basic.json b/tika-server/tika-server-core/src/test/resources/config-examples/server-basic.json index af5f016a34..044303b17c 100644 --- a/tika-server/tika-server-core/src/test/resources/config-examples/server-basic.json +++ b/tika-server/tika-server-core/src/test/resources/config-examples/server-basic.json @@ -2,10 +2,15 @@ "server": { "port": 9998, "host": "localhost", - "taskTimeoutMillis": 300000, "allowPipes": false, "allowPerRequestConfig": false }, + "parse-context": { + "timeout-limits": { + "totalTaskTimeoutMillis": 3600000, + "progressTimeoutMillis": 120000 + } + }, "parsers": [ { "default-parser": {} diff --git a/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json b/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json index c093108a52..dc1bc58b80 100644 --- a/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json +++ b/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json @@ -1,9 +1,7 @@ { "server": { "port": 9998, - "host": "0.0.0.0", - "taskTimeoutMillis": 600000, - "returnStackTrace": true + "host": "localhost" }, "parsers": [ { diff --git a/tika-server/tika-server-standard/src/test/resources/configs/cxf-test-base-template.json b/tika-server/tika-server-standard/src/test/resources/configs/cxf-test-base-template.json index f0bfba8eb0..664b2599e6 100644 --- a/tika-server/tika-server-standard/src/test/resources/configs/cxf-test-base-template.json +++ b/tika-server/tika-server-standard/src/test/resources/configs/cxf-test-base-template.json @@ -25,8 +25,6 @@ }, "server": { "port": 9999, - "taskTimeoutMillis": "TIMEOUT_MILLIS", - "taskPulseMillis": 100, "allowPipes": true, "allowPerRequestConfig": true, "endpoints": [